#!/usr/bin/env bash #PBS -q large #PBS -r y #PBS -W group_list=ecearth #PBS -l walltime=23:50:00 #PBS -l select=6:ncpus=24:mem=63000mb:mpiprocs=24:ompthreads=1 #PBS -M pierre-yves.barriat@uclouvain.be #PBS -m abe exec > ${PBS_O_WORKDIR-$PWD}/${PBS_JOBNAME-"local"}_${PBS_JOBID-"id"}.log echo "------------------ Job Info --------------------" echo "jobid : $PBS_JOBID" echo "jobname : $PBS_JOBNAME" echo "job type : $PBS_ENVIRONMENT" echo "submit dir : $PBS_O_WORKDIR" echo "queue : $PBS_O_QUEUE" echo "user : $PBS_O_LOGNAME" echo "threads : $OMP_NUM_THREADS" set -ue # Directories start_dir=${PBS_O_WORKDIR-$PWD} # librunscript defines some helper functions . ${start_dir}/librunscript.sh # ============================================================================= # *** BEGIN User configuration # ============================================================================= # ----------------------------------------------------------------------------- # *** General configuration # ----------------------------------------------------------------------------- # Component configuration # (for syntax of the $config variable, see librunscript.sh) config="ifs nemo lim3 rnfmapper xios:detached oasis" # Experiment name (exactly 4 letters!) exp_name=ECN0 # Simulation start and end date. Use any (reasonable) syntax you want. run_start_date="1990-01-01" run_end_date="${run_start_date} + 10 years" # Set $force_run_from_scratch to 'true' if you want to force this run to start # from scratch, possibly ignoring any restart files present in the run # directory. Leave set to 'false' otherwise. # NOTE: If set to 'true' the run directory $run_dir is cleaned! force_run_from_scratch=false special_restart=true special_restart_from=ECT0 special_restart_date="1995-01-01" # Resolution ifs_grid=T255L91 nem_grid=ORCA1L75 # Restart frequency. Use any (reasonable) number and time unit you want. # For runs without restart, leave this variable empty rst_freq="1 year" # Number of restart legs to be run in one go run_num_legs=4 # Coupling frequency between atmosphere and ocean (in seconds) cpl_freq_atm_oce_sec=2700 # Directories #start_dir=${PWD} ctrl_file_dir=${start_dir}/ctrl # Architecture build_arch=ecconf # This file is used to store information about restarts ece_info_file="ece.info" # ----------------------------------------------------------------------------- # *** Read platform dependent configuration # ----------------------------------------------------------------------------- . ${start_dir}/ecconf.cfg configure # ----------------------------------------------------------------------------- # *** IFS configuration # ----------------------------------------------------------------------------- ifs_version=36r4 ifs_time_step_sec=2700 ifs_output_freq=$(( 6 * 3600 / ifs_time_step_sec )) ifs_di_freq=$(( 24 * 3600 / ifs_time_step_sec )) ifs_ddh_freq=$(( 120 * 3600 / ifs_time_step_sec )) ifs_res_hor=$(echo ${ifs_grid} | sed 's:T\([0-9]\+\)L\([0-9]\+\):\1:') ifs_res_ver=$(echo ${ifs_grid} | sed 's:T\([0-9]\+\)L\([0-9]\+\):\2:') ifs_numproc=78 ifs_exe_file=${ecearth_src_dir}/ifs-${ifs_version}/bin/ifsmaster-${build_arch} ifs_lastout=false ifs_cmip5=TRUE ifs_cmip5_rcp=0 ifs_cmip_fixyear=2000 ifs_mac2sp=TRUE ifs_cmip6piaer=TRUE ifs_cmip6=TRUE ifs_A4xCO2=FALSE ifs_1PCTCO2=FALSE # Time-varying orbital forcing (Qiong Zhang, SU-2013-09) # https://dev.ec-earth.org/projects/ecearth3/wiki/Orbital_forcing_in_EC-Earth_3 # # ifs_orb_switch=FALSE, no orbital calculations applied # ifs_orb_switch=TRUE, use orbital calculations according to ifs_orb_mode # ifs_orb_mode="fixed_year", or "variable_year", or "fixed_parameters" # fixed_year: calculate the orbital parameters at ifs_orb_iyear, e.g.,1850 # variable_year: calculate orbital parameters annually start from ifs_orb_iyear # fixed_parameters: prescribe orbital parameters for given year ifs_orb_switch=FALSE ifs_orb_mode="variable_year" ifs_orb_iyear=$(date -u -d "${run_start_date}" +%Y) # ----------------------------------------------------------------------------- # *** IFS tuning (the commented values are EC-Earth 3.2beta and IFS cy36r4) # ----------------------------------------------------------------------------- # ECE32b IFS cy36r4 RPRCON=1.51E-3 # 1.2E-3 1.4E-3 RVICE=0.125 # 0.13 0.15 RLCRITSNOW=4.2E-5 # 3.0E-5 5.0E-5 RSNOWLIN2=0.035 # 0.035 0.025 ENTRORG=1.66E-4 # 1.5E-4 1.8E-4 DETRPEN=0.69E-4 # 0.75E-4 0.75E-4 ENTRDD=3.6E-4 # 3.0E-4 2.0E-4 RMFDEPS=0.27 # 0.3 0.35 RCLDIFF=3.E-6 # 3.E-6 RCLDIFFC=5.0 # 5.0 # ----------------------------------------------------------------------------- # *** NEMO/LIM configuration # ----------------------------------------------------------------------------- nem_time_step_sec=2700 lim_time_step_sec=2700 # This is only needed if the experiment is started from an existing set of NEMO # restart files nem_restart_file_path=${start_dir}/nemo-rst nem_restart_offset=0 nem_res_hor=$(echo ${nem_grid} | sed 's:ORCA\([0-9]\+\)L[0-9]\+:\1:') nem_exe_file=${ecearth_src_dir}/nemo-3.6/CONFIG/${nem_grid}_LIM3/BLD/bin/nemo.exe nem_numproc=64 # ----------------------------------------------------------------------------- # *** Runoff mapper configuration # ----------------------------------------------------------------------------- rnf_exe_file=${ecearth_src_dir}/runoff-mapper/bin/runoff-mapper.exe rnf_numproc=1 # ----------------------------------------------------------------------------- # *** OASIS configuration # ----------------------------------------------------------------------------- # Restart files for the coupling fields (note 8 character limit in OASIS) # rstas.nc : atmoshpere single-category fields # rstam.nc : atmoshpere multi-category fields # rstos.nc : ocean single-category fields # rstom.nc : ocean multi-category fields oas_rst_files="rstas.nc rstos.nc" # Decide whether the OASIS weight files for interpolation should be linked from # the setup directory (true) or not (false). In the latter case, the weights # are re-computed at the start of the run. oas_link_weights=true # Flux correction for runoff (not calving) sent from Oasis to ocean. # 1.07945 is computed to compensate for a P-E=-0.016 mm/day oas_mb_fluxcorr=1.07945 # ----------------------------------------------------------------------------- # *** XIOS configuration # ----------------------------------------------------------------------------- xio_exe_file=${ecearth_src_dir}/xios-2/bin/xios_server.exe xio_numproc=1 # ============================================================================= # *** END of User configuration # ============================================================================= # ============================================================================= # *** This is where the code begins ... # ============================================================================= # ----------------------------------------------------------------------------- # *** Create the run dir if necessary and go there # Everything is done from here. # ----------------------------------------------------------------------------- if [ ! -d ${run_dir} ] then mkdir -p ${run_dir} if $special_restart then force_run_from_scratch=false rsync -av --delete ${run_dir}/../${special_restart_from}/ --exclude log --exclude output --exclude restart --exclude="${special_restart_from}_*" --exclude="srf*" --exclude="restart_*" --exclude="debug.*" --exclude="output.*" ${run_dir} cp -f ${nem_exe_file} ${run_dir} cp -f ${ifs_exe_file} ${run_dir} cp -f ${rnf_exe_file} ${run_dir} cp -f ${xio_exe_file} ${run_dir} special_year=${special_restart_date:0:4} sed -i "/$special_year/q" ${run_dir}/ece.info . ${run_dir}/ece.info special_restart_leg=$(printf %03d $((leg_number+1))) # PUT HERE THE INSTRUCTIONS TO COPY THE restart files rsync -av ${run_dir}/../${special_restart_from}/restart/ifs/${special_restart_leg}/ ${run_dir} rsync -av ${run_dir}/../${special_restart_from}/restart/oasis/${special_restart_leg}/ ${run_dir} cd ${run_dir}/../${special_restart_from}/restart/nemo/${special_restart_leg} for f in *.nc; do nf=${exp_name}${f:4} cp $f ${run_dir}/$nf done cd - cd ${run_dir} for f in ${exp_name}_????????_restart_???_????.nc; do nf=${f:14} ln -s $f $nf done cd - rm -f ${run_dir}/ICMCL${special_restart_from}INIT mv ${run_dir}/ICMGG${special_restart_from}INIUA ${run_dir}/ICMGG${exp_name}INIUA mv ${run_dir}/ICMGG${special_restart_from}INIT ${run_dir}/ICMGG${exp_name}INIT mv ${run_dir}/ICMSH${special_restart_from}INIT ${run_dir}/ICMSH${exp_name}INIT fi else force_run_from_scratch=false special_restart=false fi cd ${run_dir} # ----------------------------------------------------------------------------- # *** Determine the time span of this run and whether it's a restart leg # ----------------------------------------------------------------------------- # Regularise the format of the start and end date of the simulation run_start_date=$(date -uR -d "${run_start_date}") run_end_date=$(date -uR -d "${run_end_date}") # Loop over the number of legs for (( ; run_num_legs>0 ; run_num_legs-- )) do # Check for restart information file and set the current leg start date # Ignore restart information file if force_run_from_scratch is true if ${force_run_from_scratch} || ! [ -r ${ece_info_file} ] then leg_is_restart=false leg_start_date=${run_start_date} leg_number=1 else leg_is_restart=true . ./${ece_info_file} leg_start_date=${leg_end_date} leg_number=$((leg_number+1)) fi # Compute the end date of the current leg if [ -n "${rst_freq}" ] then leg_end_date=$(date -uR -d "${leg_start_date} + ${rst_freq}") else leg_end_date=${run_end_date} fi if [ $(date -d "${leg_end_date}" +%s) -gt $(date -d "${run_end_date}" +%s) ] then leg_end_date=${run_end_date} fi # Some time variables needed later leg_length_sec=$(( $(date -d "${leg_end_date}" +%s) - $(date -d "${leg_start_date}" +%s) )) leg_start_sec=$(( $(date -d "${leg_start_date}" +%s) - $(date -d "${run_start_date}" +%s) )) leg_end_sec=$(( $(date -d "${leg_end_date}" +%s) - $(date -d "${run_start_date}" +%s) )) leg_start_date_yyyymmdd=$(date -u -d "${leg_start_date}" +%Y%m%d) leg_start_date_yyyy=$(date -u -d "${leg_start_date}" +%Y) leg_end_date_yyyy=$(date -u -d "${leg_end_date}" +%Y) # Check whether there's actually time left to simulate - exit otherwise if [ ${leg_length_sec} -le 0 ] then info "Leg start date equal to or after end of simulation." info "Nothing left to do. Exiting." exit 0 fi # ------------------------------------------------------------------------- # *** Prepare the run directory for a run from scratch # ------------------------------------------------------------------------- if ! $leg_is_restart then # --------------------------------------------------------------------- # *** Check if run dir is empty. If not, and if we are allowed to do so # by ${force_run_from_scratch}, remove everything # --------------------------------------------------------------------- if $(ls * >& /dev/null) then if ${force_run_from_scratch} then rm -fr ${run_dir}/* else error "Run directory not empty and \$force_run_from_scratch not set." fi fi # --------------------------------------------------------------------- # *** Copy executables of model components # *** Additionally, create symlinks to the original place for reference # --------------------------------------------------------------------- cp ${ifs_exe_file} . ln -s ${ifs_exe_file} $(basename ${ifs_exe_file}).lnk cp ${nem_exe_file} . ln -s ${nem_exe_file} $(basename ${nem_exe_file}).lnk cp ${rnf_exe_file} . ln -s ${rnf_exe_file} $(basename ${rnf_exe_file}).lnk cp ${xio_exe_file} . ln -s ${xio_exe_file} $(basename ${xio_exe_file}).lnk # --------------------------------------------------------------------- # *** Files needed for IFS (linked) # --------------------------------------------------------------------- # Initial data ln -s \ ${ini_data_dir}/ifs/${ifs_grid}/${leg_start_date_yyyymmdd}/ICMGGECE3INIUA \ ICMGG${exp_name}INIUA ln -s \ ${ini_data_dir}/ifs/${ifs_grid}/${leg_start_date_yyyymmdd}/ICMGGECE3INIT \ ICMGG${exp_name}INIT ln -s \ ${ini_data_dir}/ifs/${ifs_grid}/${leg_start_date_yyyymmdd}/ICMSHECE3INIT \ ICMSH${exp_name}INIT # Other stuff ln -s ${ini_data_dir}/ifs/rtables/* . # Output control (ppt files) mkdir postins cp ${ctrl_file_dir}/ppt* postins/ f_min=$(( ifs_output_freq * ifs_time_step_sec / 60 )) for t in $(seq $f_min $f_min 1439) do hh=$(printf %02d $((t/60)) ) mm=$(printf %02d $((t%60)) ) ln -s pptdddddd0000 postins/pptdddddd$hh$mm done /bin/ls -1 postins/* > dirlist # --------------------------------------------------------------------- # *** Files needed for NEMO (linked) # --------------------------------------------------------------------- # Link initialisation files for matching ORCA grid for f in \ bathy_meter.nc coordinates.nc \ ahmcoef.nc \ K1rowdrg.nc M2rowdrg.nc mask_itf.nc \ decay_scale_bot.nc decay_scale_cri.nc \ mixing_power_bot.nc mixing_power_cri.nc mixing_power_pyc.nc \ runoff_depth.nc do [ -f ${ini_data_dir}/nemo/initial/${nem_grid}/$f ] && ln -s ${ini_data_dir}/nemo/initial/${nem_grid}/$f done # Link geothermal heating file (independent of grid) and matching weight file ln -s ${ini_data_dir}/nemo/initial/Goutorbe_ghflux.nc ln -s ${ini_data_dir}/nemo/initial/weights_Goutorbe1_2_orca${nem_res_hor}_bilinear.nc # Link either restart files or climatology files for the initial state if $(has_config nemo:start_from_restart) then # When linking restart files, we accept three options: # (1) Merged files for ocean and ice, i.e. # restart_oce.nc and restart_ice.nc # (2) One-file-per-MPI-rank, i.e. # restart_oce_????.nc and restart_ice_????.nc # No check is done whether the number of restart files agrees # with the number of MPI ranks for NEMO! # (3) One-file-per-MPI-rank with a prefix, i.e. # __restart_oce_????.nc (similar for the ice) # The prefix is ignored. # The code assumes that one of the options can be applied! If more # options are applicable, the first is chosen. If none of the # options apply, NEMO will crash with missing restart file. if ls -U ${nem_restart_file_path}/restart_[oi]ce.nc > /dev/null 2>&1 then ln -s ${nem_restart_file_path}/restart_[oi]ce.nc ./ elif ls -U ${nem_restart_file_path}/restart_[oi]ce_????.nc > /dev/null 2>&1 then ln -s ${nem_restart_file_path}/restart_[oi]ce_????.nc ./ else for f in ${nem_restart_file_path}/????_????????_restart_[oi]ce_????.nc do ln -s $f $(echo $f | sed 's/.*_\(restart_[oi]ce_....\.nc\)/\1/') done fi else # Temperature and salinity files for initialisation ln -s ${ini_data_dir}/nemo/climatology/absolute_salinity_WOA13_decav_Reg1L75_clim.nc ln -s ${ini_data_dir}/nemo/climatology/conservative_temperature_WOA13_decav_Reg1L75_clim.nc ln -s ${ini_data_dir}/nemo/climatology/weights_WOA13d1_2_orca${nem_res_hor}_bilinear.nc # Grid dependent runoff files case ${nem_grid} in ORCA1*) ln -s ${ini_data_dir}/nemo/climatology/runoff-icb_DaiTrenberth_Depoorter_ORCA1_JD.nc ;; ORCA025*) ln -s ${ini_data_dir}/nemo/climatology/ORCA_R025_runoff_v1.1.nc ;; esac fi # XIOS files . ${ctrl_file_dir}/iodef.xml.sh > iodef.xml ln -s ${ctrl_file_dir}/context_nemo.xml ln -s ${ctrl_file_dir}/domain_def_nemo.xml ln -s ${ctrl_file_dir}/field_def_nemo-lim.xml ln -s ${ctrl_file_dir}/field_def_nemo-opa.xml ln -s ${ctrl_file_dir}/field_def_nemo-pisces.xml ln -s ${ctrl_file_dir}/file_def_nemo-lim3.xml file_def_nemo-lim.xml ln -s ${ctrl_file_dir}/file_def_nemo-opa.xml ln -s ${ctrl_file_dir}/file_def_nemo-pisces.xml # --------------------------------------------------------------------- # *** Files needed for the Runoff mapper (linked) # --------------------------------------------------------------------- ln -s ${ini_data_dir}/runoff-mapper/runoff_maps.nc # --------------------------------------------------------------------- # *** Files needed for OASIS (linked) # --------------------------------------------------------------------- oas_grid_dir=${ini_data_dir}/oasis/T${ifs_res_hor}-ORCA${nem_res_hor} # Name table file ln -s ${ini_data_dir}/oasis/cf_name_table.txt # Grid definition files ln -s ${oas_grid_dir}/areas.nc ln -s ${oas_grid_dir}/grids.nc ln -s ${oas_grid_dir}/masks.nc # Weight files case ${ifs_res_hor} in 159) oas_agrd=080 ;; 255) oas_agrd=128 ;; 511) oas_agrd=256 ;; 799) oas_agrd=400 ;; *) error "Unsupported horizontal resolution (IFS): ${ifs_res_hor}" ;; esac case ${nem_res_hor} in 1) oas_ogrd=O1t0 ;; 025) oas_ogrd=Ot25 ;; *) error "Unsupported horizontal resolution (NEMO): ${nem_res_hor}" ;; esac if ${oas_link_weights} then for f in ${oas_grid_dir}/rmp_????_to_????_GAUSWGT.nc do ln -s $f done fi for f in ${oas_rst_files} do cp ${oas_grid_dir}/rst/$f . done else # i.e. $leg_is_restart == true # --------------------------------------------------------------------- # *** Remove all leftover output files from previous legs # --------------------------------------------------------------------- # IFS files rm -f ICM{SH,GG}${exp_name}+?????? # NEMO files rm -f ${exp_name}_??_????????_????????_{grid_U,grid_V,grid_W,grid_T,icemod,SBC,scalar,SBC_scalar}.nc fi # ! $leg_is_restart # ------------------------------------------------------------------------- # *** Create some control files # ------------------------------------------------------------------------- # IFS, NEMO, LIM namelist and OASIS namcouple files . ${ctrl_file_dir}/namelist.ifs.sh > fort.4 . ${ctrl_file_dir}/namelist.nemo.ref.sh > namelist_ref . ${ctrl_file_dir}/namelist.nemo-${nem_grid}-coupled.cfg.sh > namelist_cfg . ${ctrl_file_dir}/namelist.lim3.ref.sh > namelist_ice_ref . ${ctrl_file_dir}/namelist.lim3-${nem_grid}.cfg.sh > namelist_ice_cfg . ${ctrl_file_dir}/namelist.runoffmapper.sh > namelist.runoffmapper . ${ctrl_file_dir}/namcouple.sh > namcouple # ------------------------------------------------------------------------- # *** IFS climatology data # ------------------------------------------------------------------------- icmclfile=${ini_data_dir}/ifs/${ifs_grid}/climate/ICMCL tempfile=tmp.$$ grib_set=${GRIB_BIN_PATH}${GRIB_BIN_PATH:+/}grib_set # Create data for december, the year before the leg starts ${grib_set} \ -s dataDate=$(printf "%04d" $((leg_start_date_yyyy-1)))1215 ${icmclfile}-12 \ ICMCL${exp_name}INIT # Create data for all month in the years of the leg for (( y=${leg_start_date_yyyy} ; y<=${leg_end_date_yyyy} ; y++ )) do yy=$(printf "%04d" $y) for m in {1..12} do mm=$(printf "%02d" $m) ${grib_set} -s dataDate=${yy}${mm}15 ${icmclfile}-${mm} ${tempfile} cat ${tempfile} >> ICMCL${exp_name}INIT done done # Create data for january, the year after the leg ends ${grib_set} -s dataDate=$(printf "%04d" $((leg_end_date_yyyy+1)))0115 ${icmclfile}-01 \ ${tempfile} cat ${tempfile} >> ICMCL${exp_name}INIT # Clean up rm -f ${tempfile} # ------------------------------------------------------------------------- # *** Link the appropriate NEMO restart files of the previous leg # ------------------------------------------------------------------------- if $leg_is_restart then ns=$(printf %08d $(( leg_start_sec / nem_time_step_sec - nem_restart_offset ))) for (( n=0 ; n> output/ifs/${prv_leg}/${f} rm -f ${f} else mv ${f} ${outdir} fi done # ------------------------------------------------------------------------- # *** Move NEMO output files to archive directory # ------------------------------------------------------------------------- outdir="output/nemo/$(printf %03d $((leg_number)))" mkdir -p ${outdir} for v in grid_U grid_V grid_W grid_T icemod SBC scalar SBC_scalar do for f in ${exp_name}_??_????????_????????_${v}.nc do test -f $f && mv $f $outdir/ done done # ------------------------------------------------------------------------- # *** Move IFS restart files to archive directory # ------------------------------------------------------------------------- if $leg_is_restart then outdir="restart/ifs/$(printf %03d $((leg_number)))" mkdir -p ${outdir} # Figure out the time part of the restart files (cf. CTIME on rcf files) # NOTE: Assuming that restarts are at full days (time=0000) only! nd="$(printf %06d $((leg_start_sec/(24*3600))))0000" mv srf${nd}.???? ${outdir} fi # ------------------------------------------------------------------------- # *** Move NEMO restart files to archive directory # ------------------------------------------------------------------------- if $leg_is_restart then outdir="restart/nemo/$(printf %03d $((leg_number)))" mkdir -p ${outdir} ns=$(printf %08d $(( leg_start_sec / nem_time_step_sec - nem_restart_offset ))) for f in oce ice do mv ${exp_name}_${ns}_restart_${f}_????.nc ${outdir} done fi # ------------------------------------------------------------------------- # *** Copy OASIS restart files to archive directory # NOTE: These files are copied and not moved as they are used in the # next leg! # Note also that the OASIS restart files present at the end of # the leg correspond to the start of the next leg! # ------------------------------------------------------------------------- outdir="restart/oasis/$(printf %03d $((leg_number+1)))" mkdir -p ${outdir} for f in ${oas_rst_files} do test -f ${f} && cp ${f} ${outdir} done # ------------------------------------------------------------------------- # *** Copy rcf files to the archive directory (of the next leg!) # ------------------------------------------------------------------------- outdir="restart/ifs/$(printf %03d $((leg_number+1)))" mkdir -p ${outdir} for f in rcf do test -f ${f} && cp ${f} ${outdir} done # ------------------------------------------------------------------------- # *** Move log files to archive directory # ------------------------------------------------------------------------- outdir="log/$(printf %03d $((leg_number)))" mkdir -p ${outdir} for f in \ ifs.log ifs.stat fort.4 ocean.output \ time.step solver.stat \ nout.000000 debug.??.?????? A*_??.nc O*_??.nc do test -f ${f} && mv ${f} ${outdir} done # ------------------------------------------------------------------------- # *** Write the restart control file # ------------------------------------------------------------------------- # Compute CPMIP performance sypd="$(cpmip_sypd $leg_length_sec $(($t2 - $t1)))" chpsy="$(cpmip_chpsy $leg_length_sec $(($t2 - $t1)) $(($ifs_numproc + $nem_numproc + $rnf_numproc + $xio_numproc)))" echo "#" | tee -a ${ece_info_file} echo "# Finished leg at `date '+%F %T'` after ${tr} (hh:mm:ss)" \ | tee -a ${ece_info_file} echo "# CPMIP performance: $sypd SYPD $chpsy CHPSY"| tee -a ${ece_info_file} echo "leg_number=${leg_number}" | tee -a ${ece_info_file} echo "leg_start_date=\"${leg_start_date}\"" | tee -a ${ece_info_file} echo "leg_end_date=\"${leg_end_date}\"" | tee -a ${ece_info_file} # Need to reset force_run_from_scratch in order to avoid destroying the next leg force_run_from_scratch=false special_restart=false done # loop over legs # ----------------------------------------------------------------------------- # *** Platform dependent finalising of the run # ----------------------------------------------------------------------------- finalise exit 0