ece-ifs+nemo.sh.tmpl 30 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788
  1. #!/usr/bin/env bash
  2. #PBS -q large
  3. #PBS -r y
  4. #PBS -W group_list=ecearth
  5. #PBS -l walltime=23:50:00
  6. #PBS -l select=16:ncpus=24:mem=63000mb:mpiprocs=24:ompthreads=1
  7. #PBS -M pierre-yves.barriat@uclouvain.be
  8. #PBS -m abe
  9. exec > ${PBS_O_WORKDIR-$PWD}/${PBS_JOBNAME-"local"}_${PBS_JOBID-"id"}.log
  10. echo "------------------ Job Info --------------------"
  11. echo "jobid : $PBS_JOBID"
  12. echo "jobname : $PBS_JOBNAME"
  13. echo "job type : $PBS_ENVIRONMENT"
  14. echo "submit dir : $PBS_O_WORKDIR"
  15. echo "queue : $PBS_O_QUEUE"
  16. echo "user : $PBS_O_LOGNAME"
  17. echo "threads : $OMP_NUM_THREADS"
  18. set -ue
  19. # Directories
  20. start_dir=${PBS_O_WORKDIR-$PWD}
  21. # librunscript defines some helper functions
  22. . ${start_dir}/librunscript.sh
  23. # =============================================================================
  24. # *** BEGIN User configuration
  25. # =============================================================================
  26. # -----------------------------------------------------------------------------
  27. # *** General configuration
  28. # -----------------------------------------------------------------------------
  29. # Component configuration
  30. # (for syntax of the $config variable, see librunscript.sh)
  31. config="ifs nemo:elpin lim3 rnfmapper xios:detached oasis"
  32. # Experiment name (exactly 4 letters!)
  33. exp_name=[[[MOD:GENERAL:EXP_NAME]]]
  34. # Simulation start and end date. Use any (reasonable) syntax you want.
  35. run_start_date="[[[MOD:GENERAL:RUN_START_DATE]]]"
  36. run_end_date="[[[MOD:GENERAL:RUN_END_DATE]]]"
  37. # Set $force_run_from_scratch to 'true' if you want to force this run to start
  38. # from scratch, possibly ignoring any restart files present in the run
  39. # directory. Leave set to 'false' otherwise.
  40. # NOTE: If set to 'true' the run directory $run_dir is cleaned!
  41. force_run_from_scratch=[[[MOD:GENERAL:FORCE_RUN_FROM_SCRATCH]]]
  42. # Resolution
  43. ifs_grid=[[[MOD:IFS:GRID]]]
  44. nem_grid=[[[MOD:NEM:GRID]]]
  45. # Restart frequency. Use any (reasonable) number and time unit you want.
  46. # For runs without restart, leave this variable empty
  47. rst_freq="[[[MOD:GENERAL:RST_FREQ]]]"
  48. # Number of restart legs to be run in one go
  49. run_num_legs=[[[MOD:GENERAL:RUN_NUM_LEGS]]]
  50. # Coupling frequency between atmosphere and ocean (in seconds)
  51. cpl_freq_atm_oce_sec=[[[MOD:OASIS:CPL_FREQ_ATM_OCE_SEC]]]
  52. # Directories
  53. #start_dir=${PWD}
  54. ctrl_file_dir=${start_dir}/ctrl
  55. # Architecture
  56. build_arch=[[[PLT:ACTIVE:BUILD_ARCH]]]
  57. # This file is used to store information about restarts
  58. ece_info_file="ece.info"
  59. # -----------------------------------------------------------------------------
  60. # *** Read platform dependent configuration
  61. # -----------------------------------------------------------------------------
  62. . ${start_dir}/ecconf.cfg
  63. configure
  64. # -----------------------------------------------------------------------------
  65. # *** IFS configuration
  66. # -----------------------------------------------------------------------------
  67. ifs_version=36r4
  68. ifs_time_step_sec=[[[MOD:IFS:TIME_STEP_SEC]]]
  69. # list of desired IFS output frequencies in hours (e.g. "6 3")
  70. # make sure that the list is in DESCENDING order and that the values are multiples
  71. # of the IFS timestep
  72. ifs_output_freq_h="6 3"
  73. for hfreq in $ifs_output_freq_h
  74. do
  75. sfreq=$(( hfreq * 3600 ))
  76. if (( $(( sfreq % ifs_time_step_sec )) ))
  77. then
  78. error "Output frequency $hfreq [hrs] is not a multiple of IFS timestep $ifs_time_step_sec [s]"
  79. fi
  80. done
  81. ifs_output_freq=$(( ${ifs_output_freq_h##* } * 3600 / ifs_time_step_sec ))
  82. ifs_di_freq=$(( 24 * 3600 / ifs_time_step_sec ))
  83. ifs_ddh_freq=$(( 120 * 3600 / ifs_time_step_sec ))
  84. ifs_res_hor=$(echo ${ifs_grid} | sed 's:T\([0-9]\+\)L\([0-9]\+\):\1:')
  85. ifs_res_ver=$(echo ${ifs_grid} | sed 's:T\([0-9]\+\)L\([0-9]\+\):\2:')
  86. ifs_numproc=[[[MOD:IFS:NUMPROC]]]
  87. ifs_exe_file=${ecearth_src_dir}/ifs-${ifs_version}/bin/ifsmaster-${build_arch}
  88. ifs_lastout=false
  89. ifs_cmip5=[[[MOD:IFS:CMIP5]]]
  90. ifs_cmip5_rcp=[[[MOD:IFS:CMIP5_RCP]]]
  91. ifs_cmip_fixyear=[[[MOD:IFS:CMIP_FIXYEAR]]]
  92. ifs_mac2sp=[[[MOD:IFS:MAC2SP]]]
  93. ifs_cmip6piaer=[[[MOD:IFS:CMIP6PIAER]]]
  94. ifs_cmip6=[[[MOD:IFS:CMIP6]]]
  95. lcmip6_strat_simp=[[[MOD:IFS:CMIP6]]]
  96. ifs_A4xCO2=[[[MOD:IFS:CMIP6_A4xCO2]]]
  97. ifs_1PCTCO2=[[[MOD:IFS:CMIP6_1PCTCO2]]]
  98. # Time-varying orbital forcing (Qiong Zhang, SU-2013-09)
  99. # https://dev.ec-earth.org/projects/ecearth3/wiki/Orbital_forcing_in_EC-Earth_3
  100. #
  101. # ifs_orb_switch=FALSE, no orbital calculations applied
  102. # ifs_orb_switch=TRUE, use orbital calculations according to ifs_orb_mode
  103. # ifs_orb_mode="fixed_year", or "variable_year", or "fixed_parameters"
  104. # fixed_year: calculate the orbital parameters at ifs_orb_iyear, e.g.,1850
  105. # variable_year: calculate orbital parameters annually start from ifs_orb_iyear
  106. # fixed_parameters: prescribe orbital parameters for given year
  107. ifs_orb_switch=FALSE
  108. ifs_orb_mode="variable_year"
  109. ifs_orb_iyear=$(date -u -d "${run_start_date}" +%Y)
  110. # -----------------------------------------------------------------------------
  111. # *** IFS tuning (the commented values are EC-Earth 3.2beta and IFS cy36r4)
  112. # -----------------------------------------------------------------------------
  113. # ECE32b IFS cy36r4
  114. RPRCON=1.41E-3 # 1.2E-3 1.4E-3
  115. RVICE=0.125 # 0.13 0.15
  116. RLCRITSNOW=3.9E-5 # 3.0E-5 5.0E-5
  117. RSNOWLIN2=0.035 # 0.035 0.025
  118. ENTRORG=1.49E-4 # 1.5E-4 1.8E-4
  119. DETRPEN=0.7E-4 # 0.75E-4 0.75E-4
  120. ENTRDD=3.6E-4 # 3.0E-4 2.0E-4
  121. RMFDEPS=0.3 # 0.3 0.35
  122. RCLDIFF=3.E-6 # 3.E-6
  123. RCLDIFFC=5.0 # 5.0
  124. # -----------------------------------------------------------------------------
  125. # *** NEMO/LIM configuration
  126. # -----------------------------------------------------------------------------
  127. nem_time_step_sec=[[[MOD:NEM:NEM_TIME_STEP_SEC]]]
  128. lim_time_step_sec=[[[MOD:NEM:LIM_TIME_STEP_SEC]]]
  129. # This is only needed if the experiment is started from an existing set of NEMO
  130. # restart files
  131. nem_restart_file_path=${start_dir}/nemo-rst
  132. nem_restart_offset=0
  133. nem_res_hor=$(echo ${nem_grid} | sed 's:ORCA\([0-9]\+\)L[0-9]\+:\1:')
  134. nem_exe_file=${ecearth_src_dir}/nemo-3.6/CONFIG/${nem_grid}_LIM3/BLD/bin/nemo.exe
  135. nem_numproc=[[[MOD:NEM:NUMPROC]]]
  136. # Remove land grid-points
  137. if $(has_config nemo:elpin)
  138. then
  139. jpns=($(${ecearth_src_dir}/util/ELPiN/Launcher -e ORCA${nem_res_hor} ${nem_numproc}))
  140. info "nemo domain decompostion from ELpIN: ${jpns[@]}"
  141. nem_numproc=${jpns[0]}
  142. nem_jpni=${jpns[1]}
  143. nem_jpnj=${jpns[2]}
  144. else
  145. info "nemo original domain decomposition (not using ELPiN)"
  146. fi
  147. # -----------------------------------------------------------------------------
  148. # *** Runoff mapper configuration
  149. # -----------------------------------------------------------------------------
  150. rnf_exe_file=${ecearth_src_dir}/runoff-mapper/bin/runoff-mapper.exe
  151. rnf_numproc=1
  152. # -----------------------------------------------------------------------------
  153. # *** OASIS configuration
  154. # -----------------------------------------------------------------------------
  155. # Restart files for the coupling fields (note 8 character limit in OASIS)
  156. # rstas.nc : atmoshpere single-category fields
  157. # rstam.nc : atmoshpere multi-category fields
  158. # rstos.nc : ocean single-category fields
  159. # rstom.nc : ocean multi-category fields
  160. oas_rst_files="rstas.nc rstos.nc"
  161. # Decide whether the OASIS weight files for interpolation should be linked from
  162. # the setup directory (true) or not (false). In the latter case, the weights
  163. # are re-computed at the start of the run.
  164. oas_link_weights=true
  165. # Flux correction for runoff (not calving) sent from Oasis to ocean.
  166. # 1.07945 is computed to compensate for a P-E=-0.016 mm/day
  167. oas_mb_fluxcorr=1.07945
  168. # -----------------------------------------------------------------------------
  169. # *** XIOS configuration
  170. # -----------------------------------------------------------------------------
  171. xio_exe_file=${ecearth_src_dir}/xios-2/bin/xios_server.exe
  172. xio_numproc=[[[MOD:XIO:NUMPROC]]]
  173. # =============================================================================
  174. # *** END of User configuration
  175. # =============================================================================
  176. # =============================================================================
  177. # *** This is where the code begins ...
  178. # =============================================================================
  179. # -----------------------------------------------------------------------------
  180. # *** Create the run dir if necessary and go there
  181. # Everything is done from here.
  182. # -----------------------------------------------------------------------------
  183. if [ ! -d ${run_dir} ]
  184. then
  185. mkdir -p ${run_dir}
  186. fi
  187. cd ${run_dir}
  188. # -----------------------------------------------------------------------------
  189. # *** Determine the time span of this run and whether it's a restart leg
  190. # -----------------------------------------------------------------------------
  191. # Regularise the format of the start and end date of the simulation
  192. run_start_date=$(date -uR -d "${run_start_date}")
  193. run_end_date=$(date -uR -d "${run_end_date}")
  194. # Loop over the number of legs
  195. for (( ; run_num_legs>0 ; run_num_legs-- ))
  196. do
  197. # Check for restart information file and set the current leg start date
  198. # Ignore restart information file if force_run_from_scratch is true
  199. if ${force_run_from_scratch} || ! [ -r ${ece_info_file} ]
  200. then
  201. leg_is_restart=false
  202. leg_start_date=${run_start_date}
  203. leg_number=1
  204. else
  205. leg_is_restart=true
  206. . ./${ece_info_file}
  207. leg_start_date=${leg_end_date}
  208. leg_number=$((leg_number+1))
  209. fi
  210. # Compute the end date of the current leg
  211. if [ -n "${rst_freq}" ]
  212. then
  213. leg_end_date=$(date -uR -d "${leg_start_date} + ${rst_freq}")
  214. else
  215. leg_end_date=${run_end_date}
  216. fi
  217. if [ $(date -d "${leg_end_date}" +%s) -gt $(date -d "${run_end_date}" +%s) ]
  218. then
  219. leg_end_date=${run_end_date}
  220. fi
  221. # Some time variables needed later
  222. leg_length_sec=$(( $(date -d "${leg_end_date}" +%s) - $(date -d "${leg_start_date}" +%s) ))
  223. leg_start_sec=$(( $(date -d "${leg_start_date}" +%s) - $(date -d "${run_start_date}" +%s) ))
  224. leg_end_sec=$(( $(date -d "${leg_end_date}" +%s) - $(date -d "${run_start_date}" +%s) ))
  225. leg_start_date_yyyymmdd=$(date -u -d "${leg_start_date}" +%Y%m%d)
  226. leg_start_date_yyyy=$(date -u -d "${leg_start_date}" +%Y)
  227. leg_end_date_yyyy=$(date -u -d "${leg_end_date}" +%Y)
  228. # Check whether there's actually time left to simulate - exit otherwise
  229. if [ ${leg_length_sec} -le 0 ]
  230. then
  231. info "Leg start date equal to or after end of simulation."
  232. info "Nothing left to do. Exiting."
  233. exit 0
  234. fi
  235. # -------------------------------------------------------------------------
  236. # *** Prepare the run directory for a run from scratch
  237. # -------------------------------------------------------------------------
  238. if ! $leg_is_restart
  239. then
  240. # ---------------------------------------------------------------------
  241. # *** Check if run dir is empty. If not, and if we are allowed to do so
  242. # by ${force_run_from_scratch}, remove everything
  243. # ---------------------------------------------------------------------
  244. if $(ls * >& /dev/null)
  245. then
  246. if ${force_run_from_scratch}
  247. then
  248. rm -fr ${run_dir}/*
  249. else
  250. error "Run directory not empty and \$force_run_from_scratch not set."
  251. fi
  252. fi
  253. # ---------------------------------------------------------------------
  254. # *** Copy executables of model components
  255. # *** Additionally, create symlinks to the original place for reference
  256. # ---------------------------------------------------------------------
  257. cp ${ifs_exe_file} .
  258. ln -s ${ifs_exe_file} $(basename ${ifs_exe_file}).lnk
  259. cp ${nem_exe_file} .
  260. ln -s ${nem_exe_file} $(basename ${nem_exe_file}).lnk
  261. cp ${rnf_exe_file} .
  262. ln -s ${rnf_exe_file} $(basename ${rnf_exe_file}).lnk
  263. cp ${xio_exe_file} .
  264. ln -s ${xio_exe_file} $(basename ${xio_exe_file}).lnk
  265. # ---------------------------------------------------------------------
  266. # *** Files needed for IFS (linked)
  267. # ---------------------------------------------------------------------
  268. # Initial data
  269. ln -s \
  270. ${ini_data_dir}/ifs/${ifs_grid}/${leg_start_date_yyyymmdd}/ICMGGECE3INIUA \
  271. ICMGG${exp_name}INIUA
  272. ln -s \
  273. ${ini_data_dir}/ifs/${ifs_grid}/${leg_start_date_yyyymmdd}/ICMGGECE3INIT \
  274. ICMGG${exp_name}INIT
  275. ln -s \
  276. ${ini_data_dir}/ifs/${ifs_grid}/${leg_start_date_yyyymmdd}/ICMSHECE3INIT \
  277. ICMSH${exp_name}INIT
  278. # Other stuff
  279. ln -s ${ini_data_dir}/ifs/rtables/* .
  280. # Output control (ppt files)
  281. mkdir postins
  282. rm -f postins/*
  283. cp ${ctrl_file_dir}/ppt0000000000 postins/
  284. # create ppt files/links for each ifs_output_freq_h
  285. for freq in $ifs_output_freq_h ; do
  286. hhmm_freq=$(printf %02d00 $freq)
  287. if [ ! -f ${ctrl_file_dir}/pptdddddd${hhmm_freq} ] ; then
  288. echo "warning: output requested at $freq h. but ppt file postins/pptdddddd${hhmm_freq} does not exist."
  289. continue
  290. else
  291. cp ${ctrl_file_dir}/pptdddddd${hhmm_freq} postins/
  292. fi
  293. for t in `seq 0 $freq 23`
  294. do
  295. hhmm_t=$(printf %02d00 $t)
  296. if [ ! -f postins/pptdddddd${hhmm_t} ]
  297. then
  298. ln -s pptdddddd${hhmm_freq} postins/pptdddddd${hhmm_t}
  299. fi
  300. done
  301. done
  302. # create dirlist
  303. /bin/ls -1 postins/* > dirlist
  304. # ---------------------------------------------------------------------
  305. # *** Files needed for NEMO (linked)
  306. # ---------------------------------------------------------------------
  307. # Link initialisation files for matching ORCA grid
  308. # It would be simpler to link all files in ${ini_data_dir}/nemo/initial/${nem_grid}
  309. for f in \
  310. bathy_meter.nc coordinates.nc \
  311. ahmcoef.nc \
  312. K1rowdrg.nc M2rowdrg.nc mask_itf.nc \
  313. decay_scale_bot.nc decay_scale_cri.nc \
  314. mixing_power_bot.nc mixing_power_cri.nc mixing_power_pyc.nc \
  315. runoff_depth.nc subbasins.nc
  316. do
  317. [ -f ${ini_data_dir}/nemo/initial/${nem_grid}/$f ] && ln -s ${ini_data_dir}/nemo/initial/${nem_grid}/$f
  318. done
  319. # Link geothermal heating file (independent of grid) and matching weight file
  320. ln -s ${ini_data_dir}/nemo/initial/Goutorbe_ghflux.nc
  321. ln -s ${ini_data_dir}/nemo/initial/weights_Goutorbe1_2_orca${nem_res_hor}_bilinear.nc
  322. # Link either restart files or climatology files for the initial state
  323. if $(has_config nemo:start_from_restart)
  324. then
  325. # When linking restart files, we accept three options:
  326. # (1) Merged files for ocean and ice, i.e.
  327. # restart_oce.nc and restart_ice.nc
  328. # (2) One-file-per-MPI-rank, i.e.
  329. # restart_oce_????.nc and restart_ice_????.nc
  330. # No check is done whether the number of restart files agrees
  331. # with the number of MPI ranks for NEMO!
  332. # (3) One-file-per-MPI-rank with a prefix, i.e.
  333. # <exp_name>_<time_step>_restart_oce_????.nc (similar for the ice)
  334. # The prefix is ignored.
  335. # The code assumes that one of the options can be applied! If more
  336. # options are applicable, the first is chosen. If none of the
  337. # options apply, NEMO will crash with missing restart file.
  338. if ls -U ${nem_restart_file_path}/restart_[oi]ce.nc > /dev/null 2>&1
  339. then
  340. ln -s ${nem_restart_file_path}/restart_[oi]ce.nc ./
  341. elif ls -U ${nem_restart_file_path}/restart_[oi]ce_????.nc > /dev/null 2>&1
  342. then
  343. ln -s ${nem_restart_file_path}/restart_[oi]ce_????.nc ./
  344. else
  345. for f in ${nem_restart_file_path}/????_????????_restart_[oi]ce_????.nc
  346. do
  347. ln -s $f $(echo $f | sed 's/.*_\(restart_[oi]ce_....\.nc\)/\1/')
  348. done
  349. fi
  350. else
  351. # Temperature and salinity files for initialisation
  352. ln -s ${ini_data_dir}/nemo/climatology/absolute_salinity_WOA13_decav_Reg1L75_clim.nc
  353. ln -s ${ini_data_dir}/nemo/climatology/conservative_temperature_WOA13_decav_Reg1L75_clim.nc
  354. ln -s ${ini_data_dir}/nemo/climatology/weights_WOA13d1_2_orca${nem_res_hor}_bilinear.nc
  355. # Grid dependent runoff files
  356. case ${nem_grid} in
  357. ORCA1*) ln -s ${ini_data_dir}/nemo/climatology/runoff-icb_DaiTrenberth_Depoorter_ORCA1_JD.nc ;;
  358. ORCA025*) ln -s ${ini_data_dir}/nemo/climatology/ORCA_R025_runoff_v1.1.nc ;;
  359. esac
  360. fi
  361. # XIOS files
  362. . ${ctrl_file_dir}/iodef.xml.sh > iodef.xml
  363. ln -s ${ctrl_file_dir}/context_nemo.xml
  364. ln -s ${ctrl_file_dir}/domain_def_nemo.xml
  365. ln -s ${ctrl_file_dir}/field_def_nemo-lim.xml
  366. ln -s ${ctrl_file_dir}/field_def_nemo-opa.xml
  367. ln -s ${ctrl_file_dir}/field_def_nemo-pisces.xml
  368. ln -s ${ctrl_file_dir}/file_def_nemo-lim3.xml file_def_nemo-lim.xml
  369. ln -s ${ctrl_file_dir}/file_def_nemo-opa.xml
  370. ln -s ${ctrl_file_dir}/file_def_nemo-pisces.xml
  371. # ---------------------------------------------------------------------
  372. # *** Files needed for the Runoff mapper (linked)
  373. # ---------------------------------------------------------------------
  374. ln -s ${ini_data_dir}/runoff-mapper/runoff_maps.nc
  375. # ---------------------------------------------------------------------
  376. # *** Files needed for OASIS (linked)
  377. # ---------------------------------------------------------------------
  378. oas_grid_dir=${ini_data_dir}/oasis/T${ifs_res_hor}-ORCA${nem_res_hor}
  379. # Name table file
  380. ln -s ${ini_data_dir}/oasis/cf_name_table.txt
  381. # Grid definition files
  382. ln -s ${oas_grid_dir}/areas.nc
  383. ln -s ${oas_grid_dir}/grids.nc
  384. ln -s ${oas_grid_dir}/masks.nc
  385. # Weight files
  386. case ${ifs_res_hor} in
  387. 159) oas_agrd=080
  388. ;;
  389. 255) oas_agrd=128
  390. ;;
  391. 511) oas_agrd=256
  392. ;;
  393. 799) oas_agrd=400
  394. ;;
  395. *) error "Unsupported horizontal resolution (IFS): ${ifs_res_hor}"
  396. ;;
  397. esac
  398. case ${nem_res_hor} in
  399. 1) oas_ogrd=O1t0
  400. ;;
  401. 025) oas_ogrd=Ot25
  402. ;;
  403. *) error "Unsupported horizontal resolution (NEMO): ${nem_res_hor}"
  404. ;;
  405. esac
  406. if ${oas_link_weights}
  407. then
  408. for f in ${oas_grid_dir}/rmp_????_to_????_GAUSWGT.nc
  409. do
  410. ln -s $f
  411. done
  412. fi
  413. for f in ${oas_rst_files}
  414. do
  415. cp ${oas_grid_dir}/rst/$f .
  416. done
  417. else # i.e. $leg_is_restart == true
  418. # ---------------------------------------------------------------------
  419. # *** Remove all leftover output files from previous legs
  420. # ---------------------------------------------------------------------
  421. # IFS files
  422. rm -f ICM{SH,GG}${exp_name}+??????
  423. # NEMO files
  424. rm -f ${exp_name}_??_????????_????????_{grid_U,grid_V,grid_W,grid_T,icemod,SBC,scalar,SBC_scalar}.nc
  425. fi # ! $leg_is_restart
  426. # -------------------------------------------------------------------------
  427. # *** Create some control files
  428. # -------------------------------------------------------------------------
  429. # IFS, NEMO, LIM namelist and OASIS namcouple files
  430. . ${ctrl_file_dir}/namelist.ifs.sh > fort.4
  431. . ${ctrl_file_dir}/namelist.nemo.ref.sh > namelist_ref
  432. . ${ctrl_file_dir}/namelist.nemo-${nem_grid}-coupled.cfg.sh > namelist_cfg
  433. . ${ctrl_file_dir}/namelist.lim3.ref.sh > namelist_ice_ref
  434. . ${ctrl_file_dir}/namelist.lim3-${nem_grid}.cfg.sh > namelist_ice_cfg
  435. . ${ctrl_file_dir}/namelist.runoffmapper.sh > namelist.runoffmapper
  436. . ${ctrl_file_dir}/namcouple.sh > namcouple
  437. # -------------------------------------------------------------------------
  438. # *** IFS climatology data
  439. # -------------------------------------------------------------------------
  440. icmclfile=${ini_data_dir}/ifs/${ifs_grid}/climate/ICMCL
  441. tempfile=tmp.$$
  442. grib_set=${GRIB_BIN_PATH}${GRIB_BIN_PATH:+/}grib_set
  443. # Create data for december, the year before the leg starts
  444. ${grib_set} \
  445. -s dataDate=$(printf "%04d" $((leg_start_date_yyyy-1)))1215 ${icmclfile}-12 \
  446. ICMCL${exp_name}INIT
  447. # Create data for all month in the years of the leg
  448. for (( y=${leg_start_date_yyyy} ; y<=${leg_end_date_yyyy} ; y++ ))
  449. do
  450. yy=$(printf "%04d" $y)
  451. for m in {1..12}
  452. do
  453. mm=$(printf "%02d" $m)
  454. ${grib_set} -s dataDate=${yy}${mm}15 ${icmclfile}-${mm} ${tempfile}
  455. cat ${tempfile} >> ICMCL${exp_name}INIT
  456. done
  457. done
  458. # Create data for january, the year after the leg ends
  459. ${grib_set} -s dataDate=$(printf "%04d" $((leg_end_date_yyyy+1)))0115 ${icmclfile}-01 \
  460. ${tempfile}
  461. cat ${tempfile} >> ICMCL${exp_name}INIT
  462. # Clean up
  463. rm -f ${tempfile}
  464. # -------------------------------------------------------------------------
  465. # *** Link the appropriate NEMO restart files of the previous leg
  466. # -------------------------------------------------------------------------
  467. if $leg_is_restart
  468. then
  469. ns=$(printf %08d $(( leg_start_sec / nem_time_step_sec - nem_restart_offset )))
  470. for (( n=0 ; n<nem_numproc ; n++ ))
  471. do
  472. np=$(printf %04d ${n})
  473. ln -fs ${exp_name}_${ns}_restart_oce_${np}.nc restart_oce_${np}.nc
  474. ln -fs ${exp_name}_${ns}_restart_ice_${np}.nc restart_ice_${np}.nc
  475. done
  476. # Make sure there are no global restart files
  477. # If links are found, they will be removed. We are cautious and do
  478. # _not_ remove real files! However, if real global restart files are
  479. # present, NEMO/LIM will stop because time stamps will not match.
  480. [ -h restart_oce.nc ] && rm restart_oce.nc
  481. [ -h restart_ice.nc ] && rm restart_ice.nc
  482. fi
  483. # -------------------------------------------------------------------------
  484. # *** Remove some OASIS files of the previous leg
  485. # -------------------------------------------------------------------------
  486. if $leg_is_restart
  487. then
  488. rm -f anaisout_*
  489. fi
  490. # -------------------------------------------------------------------------
  491. # *** Start the run
  492. # -------------------------------------------------------------------------
  493. export DR_HOOK_IGNORE_SIGNALS='-1'
  494. export CPLNG='active'
  495. # Use the launch function from the platform configuration file
  496. t1=$(date +%s)
  497. launch \
  498. ${xio_numproc} ${xio_exe_file} -- \
  499. ${nem_numproc} ${nem_exe_file} -- \
  500. ${rnf_numproc} ${rnf_exe_file} -- \
  501. ${ifs_numproc} ${ifs_exe_file} -v ecmwf -e ${exp_name}
  502. t2=$(date +%s)
  503. tr=$(date -d "0 -$t1 sec + $t2 sec" +%T)
  504. # -------------------------------------------------------------------------
  505. # *** Check for signs of success
  506. # Note the tests provide no guarantee that things went fine! They are
  507. # just based on the IFS and NEMO log files. More tests (e.g. checking
  508. # restart files) could be implemented.
  509. # -------------------------------------------------------------------------
  510. # Checking for IFS success
  511. if [ -f ifs.stat ]
  512. then
  513. if [ "$(awk 'END{print $3}' ifs.stat)" == "CNT0" ]
  514. then
  515. info "Leg successfully completed according to IFS log file 'ifs.stat'."
  516. else
  517. error "Leg not completed according to IFS log file 'ifs.stat'."
  518. fi
  519. else
  520. error "IFS log file 'ifs.stat' not found after run."
  521. fi
  522. # Check for NEMO success
  523. if [ -f ocean.output ]
  524. then
  525. if [ "$(awk '/New day/{d=$10}END{print d}' ocean.output)" == "$(date -d "${leg_end_date} - 1 day" +%Y/%m/%d)" ]
  526. then
  527. info "Leg successfully completed according to NEMO log file 'ocean.output'."
  528. else
  529. error "Leg not completed according to NEMO log file 'ocean.output'."
  530. fi
  531. else
  532. error "NEMO log file 'ocean.output' not found after run."
  533. fi
  534. # -------------------------------------------------------------------------
  535. # *** Move IFS output files to archive directory
  536. # -------------------------------------------------------------------------
  537. outdir="output/ifs/$(printf %03d $((leg_number)))"
  538. mkdir -p ${outdir}
  539. prv_leg=$(printf %03d $((leg_number-1)))
  540. # This takes care of a special IFS feature: The output for the last time
  541. # step of each leg is written at the first time step of the new leg. The
  542. # following code makes sure that the output is appended to the appropriate
  543. # file. Since GRIB files are just streams, its done with a simple cat
  544. # command.
  545. for f in ICMSH${exp_name}+?????? ICMGG${exp_name}+??????
  546. do
  547. if [ -f output/ifs/${prv_leg}/${f} ]
  548. then
  549. cat ${f} >> output/ifs/${prv_leg}/${f}
  550. rm -f ${f}
  551. else
  552. mv ${f} ${outdir}
  553. fi
  554. done
  555. # -------------------------------------------------------------------------
  556. # *** Move NEMO output files to archive directory
  557. # -------------------------------------------------------------------------
  558. outdir="output/nemo/$(printf %03d $((leg_number)))"
  559. mkdir -p ${outdir}
  560. for f in ${exp_name}_??_????????_????????_*.nc ; do mv $f $outdir/ ; done
  561. #for v in grid_U grid_V grid_W grid_T icemod SBC scalar SBC_scalar
  562. #do
  563. # for f in ${exp_name}_??_????????_????????_${v}.nc
  564. # do
  565. # test -f $f && mv $f $outdir/
  566. # done
  567. #done
  568. # -------------------------------------------------------------------------
  569. # *** Move IFS restart files to archive directory
  570. # -------------------------------------------------------------------------
  571. if $leg_is_restart
  572. then
  573. outdir="restart/ifs/$(printf %03d $((leg_number)))"
  574. mkdir -p ${outdir}
  575. # Figure out the time part of the restart files (cf. CTIME on rcf files)
  576. # NOTE: Assuming that restarts are at full days (time=0000) only!
  577. nd="$(printf %06d $((leg_start_sec/(24*3600))))0000"
  578. mv srf${nd}.???? ${outdir}
  579. fi
  580. # -------------------------------------------------------------------------
  581. # *** Move NEMO restart files to archive directory
  582. # -------------------------------------------------------------------------
  583. if $leg_is_restart
  584. then
  585. outdir="restart/nemo/$(printf %03d $((leg_number)))"
  586. mkdir -p ${outdir}
  587. ns=$(printf %08d $(( leg_start_sec / nem_time_step_sec - nem_restart_offset )))
  588. for f in oce ice
  589. do
  590. mv ${exp_name}_${ns}_restart_${f}_????.nc ${outdir}
  591. done
  592. fi
  593. # -------------------------------------------------------------------------
  594. # *** Copy OASIS restart files to archive directory
  595. # NOTE: These files are copied and not moved as they are used in the
  596. # next leg!
  597. # Note also that the OASIS restart files present at the end of
  598. # the leg correspond to the start of the next leg!
  599. # -------------------------------------------------------------------------
  600. outdir="restart/oasis/$(printf %03d $((leg_number+1)))"
  601. mkdir -p ${outdir}
  602. for f in ${oas_rst_files}
  603. do
  604. test -f ${f} && cp ${f} ${outdir}
  605. done
  606. # -------------------------------------------------------------------------
  607. # *** Copy rcf files to the archive directory (of the next leg!)
  608. # -------------------------------------------------------------------------
  609. outdir="restart/ifs/$(printf %03d $((leg_number+1)))"
  610. mkdir -p ${outdir}
  611. for f in rcf
  612. do
  613. test -f ${f} && cp ${f} ${outdir}
  614. done
  615. # -------------------------------------------------------------------------
  616. # *** Move log files to archive directory
  617. # -------------------------------------------------------------------------
  618. outdir="log/$(printf %03d $((leg_number)))"
  619. mkdir -p ${outdir}
  620. for f in \
  621. ifs.log ifs.stat fort.4 ocean.output \
  622. time.step solver.stat \
  623. nout.000000 debug.??.?????? A*_??.nc O*_??.nc
  624. do
  625. test -f ${f} && mv ${f} ${outdir}
  626. done
  627. # -------------------------------------------------------------------------
  628. # *** Write the restart control file
  629. # -------------------------------------------------------------------------
  630. # Compute CPMIP performance
  631. sypd="$(cpmip_sypd $leg_length_sec $(($t2 - $t1)))"
  632. chpsy="$(cpmip_chpsy $leg_length_sec $(($t2 - $t1)) $(($ifs_numproc + $nem_numproc + $rnf_numproc + $xio_numproc)))"
  633. echo "#" | tee -a ${ece_info_file}
  634. echo "# Finished leg at `date '+%F %T'` after ${tr} (hh:mm:ss)" \
  635. | tee -a ${ece_info_file}
  636. echo "# CPMIP performance: $sypd SYPD $chpsy CHPSY"| tee -a ${ece_info_file}
  637. echo "leg_number=${leg_number}" | tee -a ${ece_info_file}
  638. echo "leg_start_date=\"${leg_start_date}\"" | tee -a ${ece_info_file}
  639. echo "leg_end_date=\"${leg_end_date}\"" | tee -a ${ece_info_file}
  640. # Need to reset force_run_from_scratch in order to avoid destroying the next leg
  641. force_run_from_scratch=false
  642. done # loop over legs
  643. # -----------------------------------------------------------------------------
  644. # *** Platform dependent finalising of the run
  645. # -----------------------------------------------------------------------------
  646. finalise
  647. exit 0