skeleton.sh 10 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312
  1. # Function leap days calculates the number of leap days (29th of Februrary) in
  2. # a time intervall between two dates.
  3. #
  4. # Usage leap_days START_DATE END_DATE
  5. #TODO: simplify this
  6. function leap_days()
  7. {
  8. local ld=0
  9. local frstYYYY=$(date -ud "$1" +%Y)
  10. local lastYYYY=$(date -ud "$2" +%Y)
  11. set +e
  12. # Check first year for leap day between start and end date
  13. $(date -ud "${frstYYYY}-02-29" > /dev/null 2>&1) \
  14. && (( $(date -ud "$1" +%s) < $(date -ud "${frstYYYY}-03-01" +%s) )) \
  15. && (( $(date -ud "$2" +%s) > $(date -ud "${lastYYYY}-02-28" +%s) )) \
  16. && (( ld++ ))
  17. # Check intermediate years for leap day
  18. for (( y=(( ${frstYYYY}+1 )); y<=(( ${lastYYYY}-1 )); y++ ))
  19. do
  20. $(date -ud "$y-02-29" > /dev/null 2>&1) && (( ld++ ))
  21. done
  22. # Check last year (if different from first year) for leap day between start
  23. # and end date
  24. (( $lastYYYY > $frstYYYY )) \
  25. && $(date -ud "${lastYYYY}-02-29" > /dev/null 2>&1) \
  26. && (( $(date -ud "$1" +%s) < $(date -ud "${frstYYYY}-03-01" +%s) )) \
  27. && (( $(date -ud "$2" +%s) > $(date -ud "${lastYYYY}-02-28" +%s) )) \
  28. && (( ld++ ))
  29. set -e
  30. echo "$ld"
  31. }
  32. [[ $@ == *verbose* ]] && set -x
  33. module load ${module_list:?}
  34. export LD_LIBRARY_PATH=${LD_LIBRARY_PATH:+${LD_LIBRARY_PATH}:}"${extralibs_list}"
  35. #
  36. # Create run directory
  37. # Copy executables
  38. # Copy XIOS XML files into run directory
  39. # Copy namelist files into run directory
  40. #
  41. if [ ! -d ${run_dir:?} ]
  42. then
  43. mkdir -p ${run_dir}
  44. if $special_restart
  45. then
  46. rsync -av --delete ${run_dir}/../${special_restart_from}/ --exclude log --exclude output --exclude restart --exclude="${special_restart_from}_*" --exclude="ocean*" --exclude="restart_*" --exclude="debug.*" --exclude="output.*" ${run_dir}
  47. cp -f ${nem_exe_file} ${run_dir}
  48. cp -f ${xio_exe_file} ${run_dir}
  49. special_year=${special_restart_date:0:4}
  50. sed -i "/$special_year/q" ${run_dir}/${info_file}
  51. . ${run_dir}/${info_file}
  52. special_restart_leg=$(printf %03d $((leg_number+1)))
  53. # PUT HERE THE INSTRUCTIONS TO COPY THE restart files
  54. cd ${run_dir}/../../archive/${special_restart_from}/restart/${special_restart_leg}
  55. for f in *.nc; do
  56. nf=${exp_name}${f:4}
  57. cp $f ${run_dir}/$nf
  58. done
  59. cd -
  60. cd ${run_dir}
  61. for f in ${exp_name}_????????_restart_???_????.nc; do
  62. nf=${f:14}
  63. ln -s $f $nf
  64. done
  65. cd -
  66. fi
  67. fi
  68. cd ${start_dir}
  69. cp -u xios_config/*xml "${run_dir}"
  70. cp -u namelists/* "${run_dir}"
  71. cd "${run_dir}"
  72. cp -u "${nem_exe_file:?}" .
  73. cp -u "${xio_exe_file:?}" .
  74. # Write fake file for previous fresh water budget adjustment (nn_fwb==2 in namelist)
  75. [[ ! -f EMPave_old.dat ]] && echo " 0 0.0000000000000000E+00 0.0000000000000000E+00" > EMPave_old.dat
  76. #TODO: Enquiry about this
  77. #
  78. # Link data files into run directory
  79. # Attention: no space in filenames allowed!
  80. # (No protection against word splitting by design)
  81. #
  82. for file in "${ic_files[@]}"; do
  83. [[ ! -e ${file#*> } ]] && ln -sf $(sed 's/ *=> */ /' <<< "${ini_data_dir}/${ic_subdir}/${nem_grid}/$file")
  84. done
  85. for file in "${forcing_files[@]}"; do
  86. [[ ! -e ${file#*> } || "$file" == \** ]] && ln -sf $(sed 's/ *=> */ /' <<< "${ini_data_dir}/${forcing_subdir}/${nem_forcing_set}/$file")
  87. done
  88. for file in "${shared_files[@]}"; do
  89. [[ ! -e ${file#*> } ]] && ln -sf $(sed 's/ *=> */ /' <<< "${shared_dir}/$file")
  90. done
  91. #
  92. # Normalize date formats
  93. #
  94. run_start_date=$(date -uR -d "${run_start_date}")
  95. run_end_date="${run_start_date} + ${run_duration:?}"
  96. run_end_date=$(date -uR -d "${run_end_date}")
  97. run_start_epoch=$(date -u -d"${run_start_date}" +%s)
  98. run_end_epoch=$(date -u -d"${run_end_date}" +%s)
  99. for (( ; run_num_legs>0 ; run_num_legs-- ))
  100. do
  101. #
  102. # Initialize variables (using restart file if it exists)
  103. # Variables ending in '_date' are in RFC2822
  104. # Variables ending in '_epoch' are in nb of seconds since 1970-01-01
  105. #
  106. [[ -r "${info_file:?}" ]] && source "${info_file:?}"
  107. leg_start_date=${leg_end_date:-$run_start_date}
  108. leg_number=$((${leg_number:=0}+1))
  109. leg_start_epoch=$(date -u -d "${leg_start_date}" +%s)
  110. leg_end_epoch=$(date -u -d "${leg_start_date:?} + ${rst_freq:=$run_duration}" +%s)
  111. leg_end_date=$(date -uR -d@"${leg_end_epoch}")
  112. leg_length_sec=$(( leg_end_epoch - leg_start_epoch ))
  113. leg_start_sec=$(( leg_start_epoch - run_start_epoch ))
  114. leg_end_sec=$(( leg_end_epoch - run_start_epoch ))
  115. leg_start_date_yyyymmdd=$(date -u -d "${leg_start_date}" +%Y%m%d) # FIXME appears unused
  116. # Correct for leap days because NEMO standalone uses no-leap calendar
  117. leg_length_sec=$(( leg_length_sec - $(leap_days "${leg_start_date}" "${leg_end_date}")*24*3600 ))
  118. leg_start_sec=$(( leg_start_sec - $(leap_days "${run_start_date}" "${leg_start_date}")*24*3600 ))
  119. leg_end_sec=$(( leg_end_sec - $(leap_days "${run_start_date}" "${leg_end_date}")*24*3600 ))
  120. (( leg_number > 1 )) && leg_is_restart=true || leg_is_restart=false
  121. #
  122. # Compute leg end-date and trim if necessary
  123. #
  124. (( leg_end_epoch > run_end_epoch )) && leg_end_date=${run_end_epoch}
  125. #
  126. # Check whether there is some work left to do
  127. #
  128. if (( leg_start_epoch >= run_end_epoch ))
  129. then
  130. echo "Leg start date equal to or after end of simulation."
  131. echo "Nothing left to do. Cleaning and exiting."
  132. for (( n=0 ; n<nem_numproc ; n++ ))
  133. do
  134. np=$(printf %04d ${n})
  135. rm -f "${run_dir}/restart_oce_${np}.nc"
  136. rm -f "${run_dir}/restart_ice_${np}.nc"
  137. done
  138. exit 0
  139. fi
  140. #
  141. # Update namelist
  142. #
  143. source build_namelist_cfg.sh > namelist_cfg
  144. #
  145. # Link the restart files
  146. #
  147. ns=$(printf %08d $(( leg_start_sec / nem_time_step_sec - nem_restart_offset )))
  148. if ((leg_start_sec > 0 )); then
  149. for (( n=0 ; n<nem_numproc ; n++ ))
  150. do
  151. np=$(printf %04d ${n})
  152. [[ -f "${exp_name:?}_${ns}_restart_oce_${np}.nc" ]] || { echo "Error: restart file not found." ; exit 2 ; }
  153. ln -fs "${exp_name:?}_${ns}_restart_oce_${np}.nc" "restart_oce_${np}.nc"
  154. [[ -f "${exp_name:?}_${ns}_restart_ice_${np}.nc" ]] || { echo "Error: restart file not found." ; exit 2 ; }
  155. ln -fs "${exp_name:?}_${ns}_restart_ice_${np}.nc" "restart_ice_${np}.nc"
  156. done
  157. fi
  158. # Stop here is preponly was specified
  159. [[ $@ == *preponly* ]] && exit 0
  160. #
  161. # Run nemo
  162. #
  163. time_begin=$(date +%s)
  164. mpirun -np "${xio_numproc:?}" "${xio_exe_file:?}" : -np "${nem_numproc:?}" "${nem_exe_file:?}"
  165. time_end=$(date +%s)
  166. #############################
  167. formatted_leg_number=$(printf %03d $((leg_number)))
  168. #
  169. # Move NEMO output files to archive directory
  170. #
  171. outdir="${archive_dir:?}/output/${formatted_leg_number}"
  172. mkdir -p "${outdir}"
  173. shopt -s nullglob
  174. for v in grid_U grid_V grid_W grid_T icemod icemoa SBC SBC_scalar scalar
  175. do
  176. for f in ${exp_name}_??_????????_????????_${v}_????.nc
  177. do
  178. mv "$f" "$outdir/"
  179. done
  180. for f in ${exp_name}_??_????????_????????_${v}.nc
  181. do
  182. mv "$f" "$outdir/"
  183. done
  184. for f in ${exp_name}_??_${v}.nc
  185. do
  186. mv "$f" "$outdir/"
  187. done
  188. done
  189. #
  190. # Move NEMO restart files from previous run to archive directory
  191. #
  192. outdir="$archive_dir/restart/${formatted_leg_number}"
  193. mkdir -p "${outdir}"
  194. for f in ${exp_name}_${ns}_restart_???_????.nc
  195. do
  196. [ -f "$f" ] && mv "$f" "${outdir}"
  197. done
  198. #
  199. # Move log files to archive directory
  200. #
  201. outdir="$archive_dir/log/${formatted_leg_number}"
  202. mkdir -p "${outdir}"
  203. for f in ocean.output time.step solver.stat ; do mv "${f}" "${outdir}"; done
  204. #
  205. #
  206. ##############################
  207. shopt -u nullglob
  208. #
  209. # Write checkpoint control file
  210. # TODO: enquiry why 0 -1 +2 rather than 2- 1
  211. #
  212. tr=$(date -d "0 -$time_begin sec + $time_end sec" +%T)
  213. current_date=$(date +'%F %T')
  214. {
  215. echo "#"
  216. echo "# Finished leg at ${current_date} after ${tr} (hh:mm:ss)"
  217. echo "leg_number=${leg_number}"
  218. echo "leg_start_date=\"${leg_start_date}\""
  219. echo "leg_end_date=\"${leg_end_date}\""
  220. } | tee -a "${info_file}"
  221. special_restart=false
  222. done
  223. #
  224. # Move back to submission directory
  225. #
  226. cd - >/dev/null
  227. #
  228. # Check whether there is some work left to do in a further job
  229. #
  230. [[ $@ == *noresubmit* ]] && exit 0
  231. if (( leg_end_epoch < run_end_epoch )) ; then
  232. echo "Leg end earlier than end of simulation."
  233. echo "Submitting another job."
  234. #[[ $@ == *local* ]] && exec $0 $@ || scontrol requeue $SLURM_JOB_ID #TODO: factorize this
  235. if [[ "$@" == *"local"* ]] ; then
  236. exec "$0" "$@"
  237. elif [ "${submit_command}" == *sbatch* ] ; then
  238. sbatch $0 $@ | awk '{print $4}' >> ${run_dir}/.coral_jobs
  239. elif [[ $USER == "vsc"* ]]; then
  240. # Retrieve credits
  241. logfiletmp=`echo ${PBS_O_WORKDIR-$PWD} | rev | cut -d'/' -f1 | rev`
  242. logfiletmp2=`echo $logfiletmp | cut -d'_' -f1`
  243. logfile=`ls ${PBS_O_WORKDIR-$PWD}"/"$logfiletmp2*.o*`
  244. credits=`grep "account string:" $logfile | sed 's/^.*: //'`
  245. if [ -z "$credits" ]; then
  246. credits="laerocloud"
  247. fi
  248. script_name=`cat ${PBS_O_WORKDIR-$PWD}"/"script_name`
  249. run_path=`cat ${PBS_O_WORKDIR-$PWD}"/"address`
  250. ssh login2 "cd $run_path; qsub -A $credits $script_name | tee -a coral_jobs"
  251. sleep 2
  252. jobid=`cat coral_jobs`
  253. rm -f coral_jobs
  254. jobid=${jobid%%.*}
  255. echo "${jobid}" >> "${run_dir}"/.coral_jobs ;
  256. else
  257. "qsub" -v PBS_OPTIONS="$@" "$0" | tee -a coral_jobs
  258. sleep 2
  259. jobid=`cat coral_jobs`
  260. rm -f coral_jobs
  261. jobid=${jobid%.*}
  262. echo "${jobid}" >> "${run_dir}"/.coral_jobs
  263. fi
  264. else
  265. if [[ $USER == "vsc"* ]]; then
  266. run_path=`cat ${PBS_O_WORKDIR-$PWD}"/"address`
  267. rm -f $run_path/script_name
  268. rm -f $run_path/address
  269. fi
  270. echo "Nothing left to do. Cleaning and exiting." # FIXME Factorize this (we have two exit points)
  271. for (( n=0 ; n<nem_numproc ; n++ ))
  272. do
  273. np=$(printf %04d ${n})
  274. rm -f "restart_oce_${np}.nc"
  275. rm -f "restart_ice_${np}.nc"
  276. done
  277. fi
  278. exit 0