#!/usr/bin/env bash set -xuve cd %CURRENT_ROOTDIR% # librunscript defines some helper functions source ./librunscript.sh # ============================================================================= # *** BEGIN User configuration # ============================================================================= # ----------------------------------------------------------------------------- # *** General configuration # ----------------------------------------------------------------------------- # Component configuration # (for syntax of the $config variable, see librunscript.sh) # get config from autosubmit variables [[ "%OCEAN_ini%" = *[!\ ]* ]] && start_nemo_from_restart=":start_from_restart" || start_nemo_from_restart="" [[ "%PISCES_ini%" = *[!\ ]* ]] && start_pisces_from_restart=":start_from_restart" || start_pisces_from_restart="" [ "%OCE_NUDG%" = TRUE ] && ocenudg=":ocenudg" || ocenudg="" [ "%SURF_RESTO%" = TRUE ] && surfresto=":surfresto" || surfresto="" [ "%PISCES%" = TRUE ] && [ "%PISCES_OFF%" = FALSE ] && pisces="pisces" || pisces="" [ "%PISCES_OFF%" = TRUE ] && pisces="pisces:offline" [ "%NEMO_remove_land%" = TRUE ] && elpin=":elpin" || elpin="" if [[ "%SAVE_IC%" = FALSE ]] || [[ "%SAVE_IC%" = "" ]]; then save_ic="" ; else save_ic="save_ic:%SAVE_IC%" ; fi config="nemo${start_nemo_from_restart}${ocenudg}${surfresto}${elpin} ${pisces}${start_pisces_from_restart} lim3 xios:detached ${save_ic}" nem_forcing_set=%NEM_FORCING_SET% nem_fixed_forcing_year=%NEM_FIXED_FORCING_YEAR% if [ -z ${nem_forcing_set-} ] ; then nem_forcing_set=CoreII ; fi # Experiment name (exactly 4 letters!) export exp_name=[[[MOD:GENERAL:EXP_NAME]]] # Simulation start and end date. Use any (reasonable) syntax you want. run_start_date="[[[MOD:GENERAL:RUN_START_DATE]]]" # define run_start_date_as with proper format e.g. 19900101 export run_start_date_as=$(date -u -d "${run_start_date}" +%Y%m%d) run_end_date="[[[MOD:GENERAL:RUN_END_DATE]]]" # Simulation member. Use any (reasonable) syntax you want. export member="%MEMBER%" # Set $force_run_from_scratch to 'true' if you want to force this run to start # from scratch, possibly ignoring any restart files present in the run # directory. Leave set to 'false' otherwise. # NOTE: If set to 'true' the run directory $run_dir is cleaned! CHUNK=%CHUNK% force_run_from_scratch=[[[MOD:GENERAL:FORCE_RUN_FROM_SCRATCH]]] force_run_from_scratch=${force_run_from_scratch:-false} force_run_from_scratch=$(echo ${force_run_from_scratch} | tr '[:upper:]' '[:lower:]') # we only apply this for the first chunk if ${force_run_from_scratch} && [[ "${CHUNK}" != "1" ]] ; then force_run_from_scratch=false fi # Resolution nem_grid=[[[MOD:NEM:GRID]]] # Restart frequency. Use any (reasonable) number and time unit you want. # For runs without restart, leave this variable empty rst_freq="[[[MOD:GENERAL:RST_FREQ]]]" # Number of restart legs to be run in one go run_num_legs=[[[MOD:GENERAL:RUN_NUM_LEGS]]] # Directories start_dir=${PWD} ctrl_file_dir=${start_dir}/ctrl output_control_files_dir=${start_dir}/[[[MOD:GENERAL:OUTPUT_CONTROL_FILES_DIR]]] # Architecture build_arch=[[[PLT:ACTIVE:BUILD_ARCH]]] use_machinefile=[[[PLT:ACTIVE:USE_MACHINEFILE]]] # This file is used to store information about restarts ece_info_file="ece.info" # ----------------------------------------------------------------------------- # *** Read platform dependent configuration # ----------------------------------------------------------------------------- source ./ecconf.cfg configure # ----------------------------------------------------------------------------- # *** Time step settings # ----------------------------------------------------------------------------- case "${nem_grid}" in ORCA1L*) nem_time_step_sec=2700; lim_time_step_sec=2700 ;; ORCA025L*) nem_time_step_sec=900 ; lim_time_step_sec=900 ;; *) error "Can't set time steps for unknown horizontal grid: ${nem_grid}" ;; esac # ----------------------------------------------------------------------------- # *** NEMO/LIM configuration # ----------------------------------------------------------------------------- # This is only needed if the experiment is started from an existing set of NEMO # restart files nem_restart_file_path=${ini_data_dir} nem_restart_offset=0 nem_res_hor=$(echo ${nem_grid} | sed 's:ORCA\([0-9]\+\)L[0-9]\+:\1:') # Pick correct NEMO configuration, which is one of: # NEMO standalone, NEMO+PISCES-standaone, PISCES-offline nem_config_name=${nem_grid}_LIM3_standalone has_config pisces && nem_config_name=${nem_grid}_LIM3_PISCES_standalone has_config pisces:offline && nem_config_name=${nem_grid}_OFF_PISCES_standalone nem_exe_file=${ecearth_src_dir}/nemo-3.6/CONFIG/${nem_config_name}/BLD/bin/nemo.exe nem_numproc=[[[MOD:NEM:NUMPROC]]] # Thermal conductivity of snow, see comment in ctrl/namelist.lim3.ref.sh nem_rn_cdsn=0.27 # ----------------------------------------------------------------------------- # *** XIOS configuration # ----------------------------------------------------------------------------- xio_exe_file=${ecearth_src_dir}/xios-2.5/bin/xios_server.exe xio_numproc=[[[MOD:XIO:NUMPROC]]] # ----------------------------------------------------------------------------- # *** Extra initial conditions saved during the run # ----------------------------------------------------------------------------- if has_config save_ic then source ./libsave_ic.sh declare -a save_ic_date save_ic_date1 save_ic_sec save_ic_day save_ic_ppt_file save_ic_nemo_ts fi # ============================================================================= # *** END of User configuration # ============================================================================= # ============================================================================= # *** This is where the code begins ... # ============================================================================= # ----------------------------------------------------------------------------- # *** Create the run dir if necessary and go there # Everything is done from here. # ----------------------------------------------------------------------------- if [ ! -d ${run_dir} ] then mkdir -p ${run_dir} fi cd ${run_dir} # ----------------------------------------------------------------------------- # *** Determine the time span of this run and whether it's a restart leg # ----------------------------------------------------------------------------- # Regularise the format of the start and end date of the simulation run_start_date=$(absolute_date_noleap "${run_start_date}") run_end_date=$(absolute_date_noleap "${run_end_date}") # Loop over the number of legs for (( ; run_num_legs>0 ; run_num_legs-- )) do # Check for restart information file and set the current leg start date # Ignore restart information file if force_run_from_scratch is true if ${force_run_from_scratch} || ! [ -r ${ece_info_file} ] then leg_is_restart=false leg_start_date=${run_start_date} leg_number=1 else leg_is_restart=true . ./${ece_info_file} leg_start_date=${leg_end_date} leg_number=$((leg_number+1)) fi # Compute the end date of the current leg if [ -n "${rst_freq}" ] then leg_end_date=$(absolute_date_noleap "${leg_start_date} + ${rst_freq}") else leg_end_date=${run_end_date} fi if [ $(date -d "${leg_end_date}" +%s) -gt $(date -d "${run_end_date}" +%s) ] then leg_end_date=${run_end_date} fi # Some time variables needed later leg_length_sec=$(( $(date -d "${leg_end_date}" +%s) - $(date -d "${leg_start_date}" +%s) )) leg_start_sec=$(( $(date -d "${leg_start_date}" +%s) - $(date -d "${run_start_date}" +%s) )) leg_end_sec=$(( $(date -d "${leg_end_date}" +%s) - $(date -d "${run_start_date}" +%s) )) leg_start_date_yyyymmdd=$(date -u -d "${leg_start_date}" +%Y%m%d) leg_start_date_yyyy=$(date -u -d "${leg_start_date}" +%Y) leg_end_date_yyyy=$(date -u -d "${leg_end_date}" +%Y) # Correct for leap days because NEMO standalone uses no-leap calendar leg_length_sec=$(( leg_length_sec - $(leap_days "${leg_start_date}" "${leg_end_date}")*24*3600 )) leg_start_sec=$(( leg_start_sec - $(leap_days "${run_start_date}" "${leg_start_date}")*24*3600 )) leg_end_sec=$(( leg_end_sec - $(leap_days "${run_start_date}" "${leg_end_date}")*24*3600 )) # Check whether there's actually time left to simulate - exit otherwise if [ ${leg_length_sec} -le 0 ] then info "Leg start date equal to or after end of simulation." info "Nothing left to do. Exiting." exit 0 fi # Initial conditions saved during the run do_save_ic=false save_ic_custom=false has_config save_ic && save_ic_get_config # if you do not use an option with save_ic, you must define 'do_save_ic' and # 'save_ic_date_offset' here or in ../libsave_ic.sh/save_ic_get_config() # with AS runtime, no need to edit the script, set SAVE_IC_OFFSET (and optionally SAVE_IC_COND) if $save_ic_custom then [[ "%SAVE_IC_COND%" = "" ]] && save_ic_cond=true || save_ic_cond='%SAVE_IC_COND%' if eval $save_ic_cond ; then do_save_ic=true ; else do_save_ic=false ; fi save_ic_date_offset=( %SAVE_IC_OFFSET% ) fi ${do_save_ic} && save_ic_define_vars # ------------------------------------------------------------------------- # *** Prepare the run directory for a run from scratch # ------------------------------------------------------------------------- if ! $leg_is_restart then # --------------------------------------------------------------------- # *** Check if run dir is empty. If not, and if we are allowed to do so # by ${force_run_from_scratch}, remove everything # --------------------------------------------------------------------- if $(ls * >& /dev/null) then if ${force_run_from_scratch} then rm -fr ${run_dir}/* else error "Run directory not empty and \$force_run_from_scratch not set." fi fi # --------------------------------------------------------------------- # *** Copy executables of model components # *** Additionally, create symlinks to the original place for reference # --------------------------------------------------------------------- cp ${nem_exe_file} . ln -s ${nem_exe_file} $(basename ${nem_exe_file}).lnk cp ${xio_exe_file} . ln -s ${xio_exe_file} $(basename ${xio_exe_file}).lnk # --------------------------------------------------------------------- # *** Files needed for NEMO (linked) # --------------------------------------------------------------------- # Link initialisation files for matching ORCA grid for f in \ bathy_meter.nc coordinates.nc \ ahmcoef.nc \ K1rowdrg.nc M2rowdrg.nc mask_itf.nc \ decay_scale_bot.nc decay_scale_cri.nc \ mixing_power_bot.nc mixing_power_cri.nc mixing_power_pyc.nc \ runoff_depth.nc subbasins.nc do [ -f ${ini_data_dir}/nemo/initial/${nem_grid}/$f ] && ln -s ${ini_data_dir}/nemo/initial/${nem_grid}/$f done # Copying the time independent NEMO files for the matching ORCA grid in order to facilitate cmorisation: for f in \ bathy_meter.nc subbasins.nc do mkdir -p output/nemo/ofx-data [ -f ${ini_data_dir}/nemo/initial/${nem_grid}/$f ] && cp -f ${ini_data_dir}/nemo/initial/${nem_grid}/$f output/nemo/ofx-data/ done # Link geothermal heating file (independent of grid) and matching weight file ln -s ${ini_data_dir}/nemo/initial/Goutorbe_ghflux.nc ln -s ${ini_data_dir}/nemo/initial/weights_Goutorbe1_2_orca${nem_res_hor}_bilinear.nc # Link either restart files or climatology files for the initial state if $(has_config nemo:start_from_restart) then # When linking restart files, we accept three options: # (1) Merged files for ocean and ice, i.e. # restart_oce.nc and restart_ice.nc # (2) One-file-per-MPI-rank, i.e. # restart_oce_????.nc and restart_ice_????.nc # No check is done whether the number of restart files agrees # with the number of MPI ranks for NEMO! # (3) One-file-per-MPI-rank with a prefix, i.e. # __restart_oce_????.nc (similar for the ice) # The prefix is ignored. # The code assumes that one of the options can be applied! If more # options are applicable, the first is chosen. If none of the # options apply, NEMO will crash with missing restart file. if ls -U ${nem_restart_file_path}/restart_[oi]ce.nc > /dev/null 2>&1 then ln -s ${nem_restart_file_path}/restart_[oi]ce.nc ./ elif ls -U ${nem_restart_file_path}/restart_[oi]ce_????.nc > /dev/null 2>&1 then ln -s ${nem_restart_file_path}/restart_[oi]ce_????.nc ./ else for f in ${nem_restart_file_path}/????_????????_restart_[oi]ce_????.nc do ln -s $f $(echo $f | sed 's/.*_\(restart_[oi]ce_....\.nc\)/\1/') done fi else # Temperature and salinity files for initialisation ln -s ${ini_data_dir}/nemo/climatology/absolute_salinity_WOA13_decav_Reg1L75_clim.nc ln -s ${ini_data_dir}/nemo/climatology/conservative_temperature_WOA13_decav_Reg1L75_clim.nc ln -s ${ini_data_dir}/nemo/climatology/weights_WOA13d1_2_orca${nem_res_hor}_bilinear.nc fi # Grid dependent runoff files case ${nem_grid} in ORCA1*) ln -s ${ini_data_dir}/nemo/climatology/runoff-icb_DaiTrenberth_Depoorter_ORCA1_JD.nc if [[ "${nem_forcing_set}" == "JRA55_1.4" ]] ; then ln -s ${ini_data_dir}/nemo/forcing/JRA55_1.4/runoff_depth_JRA.nc fi ;; ORCA025*) ln -s ${ini_data_dir}/nemo/climatology/ORCA_R025_runoff_v1.1.nc ;; esac #for ocean_nudging if $(has_config nemo:ocenudg) ; then ln -fs ${ini_data_dir}/nemo/oce_nudg/resto.nc ./ fi # Salinity file (needed for diagnostics) [ -f ${ini_data_dir}/nemo/climatology/${nem_grid}/sali_ref_clim_monthly.nc ] && ln -s ${ini_data_dir}/nemo/climatology/${nem_grid}/sali_ref_clim_monthly.nc # Write fake file for previous fresh water budget adjustment (nn_fwb==2 in namelist) echo " 0 0.0000000000000000E+00 0.0000000000000000E+00" > EMPave_old.dat # XIOS files . ${ctrl_file_dir}/iodef.xml.sh > iodef.xml ln -s ${ctrl_file_dir}/context_nemo.xml . ${ctrl_file_dir}/domain_def_nemo.xml.sh > domain_def_nemo.xml ln -s ${ctrl_file_dir}/axis_def_nemo.xml ln -s ${ctrl_file_dir}/grids_def_nemo.xml ln -s ${ctrl_file_dir}/field_def_nemo-lim.xml ln -s ${ctrl_file_dir}/field_def_nemo-opa.xml ln -s ${ctrl_file_dir}/field_def_nemo-pisces.xml ln -s ${ctrl_file_dir}/field_def_nemo-inerttrc.xml ln -s ${output_control_files_dir}/file_def_nemo-lim3.xml file_def_nemo-lim.xml ln -s ${output_control_files_dir}/file_def_nemo-opa.xml ln -s ${output_control_files_dir}/file_def_nemo-pisces.xml if [ -f ${ini_data_dir}/xios/ORCA${nem_res_hor}/coordinates_xios.nc ] then cp ${ini_data_dir}/xios/ORCA${nem_res_hor}/coordinates_xios.nc ./ else info "File 'coordinates_xios.nc' not found. NEMO can not be run with land domain removal!" fi # --------------------------------------------------------------------- # *** Files needed for TOP/PISCES (linked) # --------------------------------------------------------------------- if $(has_config pisces) then ln -fs ${ini_data_dir}/nemo/pisces/dust_INCA_ORCA_R1.nc ln -fs ${ini_data_dir}/nemo/pisces/ndeposition_Duce_ORCA_R1.nc ln -fs ${ini_data_dir}/nemo/pisces/pmarge_etopo_ORCA_R1.nc ln -fs ${ini_data_dir}/nemo/pisces/river_global_news_ORCA_R1.nc ln -fs ${ini_data_dir}/nemo/pisces/Solubility_T62_Mahowald_ORCA_R1.nc ln -fs ${ini_data_dir}/nemo/pisces/par_fraction_gewex_clim90s00s_ORCA_R1.nc ln -fs ${ini_data_dir}/nemo/pisces/DIC_GLODAP_annual_ORCA_R1.nc ln -fs ${ini_data_dir}/nemo/pisces/Alkalini_GLODAP_annual_ORCA_R1.nc ln -fs ${ini_data_dir}/nemo/pisces/O2_WOA2009_monthly_ORCA_R1.nc ln -fs ${ini_data_dir}/nemo/pisces/PO4_WOA2009_monthly_ORCA_R1.nc ln -fs ${ini_data_dir}/nemo/pisces/Si_WOA2009_monthly_ORCA_R1.nc ln -fs ${ini_data_dir}/nemo/pisces/DOC_PISCES_monthly_ORCA_R1.nc ln -fs ${ini_data_dir}/nemo/pisces/Fer_PISCES_monthly_ORCA_R1.nc ln -fs ${ini_data_dir}/nemo/pisces/NO3_WOA2009_monthly_ORCA_R1.nc ln -fs ${ini_data_dir}/nemo/pisces/mole-fraction-of-carbon-dioxide-in-air_input4MIPs_GHGConcentrations_CMIP_UoM-CMIP-1-2-0_gr1-GMNHSH_1637-2019.txt atcco2.txt fi #linking surface boundary conditions for CFCs (even if CFCs are not included) ln -fs ${ini_data_dir}/nemo/cfc/CFCs_CDIAC_extension_1637_2019.dat CFCs_CDIAC.dat if $(has_config pisces:start_from_restart) then # Same three options as for nemo:start_from_restart if ls -U ${nem_restart_file_path}/restart_trc.nc > /dev/null 2>&1 then ln -s ${nem_restart_file_path}/restart_trc.nc ./ elif ls -U ${nem_restart_file_path}/restart_trc_????.nc > /dev/null 2>&1 then ln -s ${nem_restart_file_path}/restart_trc_????.nc ./ else for f in ${nem_restart_file_path}/????_????????_restart_trc_????.nc do ln -s $f $(echo $f | sed 's/.*_\(restart_trc_....\.nc\)/\1/') done fi fi # --------------------------------------------------------------------- # *** Files needed for TOP/PISCES (linked) ADDED FOR PISCES OFFLINE # --------------------------------------------------------------------- if $(has_config pisces:offline) then ln -fs ${ini_data_dir}/nemo/pisces/pisces_dyn/OfflineForcing_grid_U.nc OfflineForcing_grid_U.nc ln -fs ${ini_data_dir}/nemo/pisces/pisces_dyn/OfflineForcing_grid_V.nc OfflineForcing_grid_V.nc ln -fs ${ini_data_dir}/nemo/pisces/pisces_dyn/OfflineForcing_grid_W.nc OfflineForcing_grid_W.nc ln -fs ${ini_data_dir}/nemo/pisces/pisces_dyn/OfflineForcing_grid_T.nc OfflineForcing_grid_T.nc ln -fs ${ctrl_file_dir}/context_nemo_offline.xml context_nemo.xml fi else # i.e. $leg_is_restart == true # --------------------------------------------------------------------- # *** Remove all leftover output files from previous legs # --------------------------------------------------------------------- # NEMO output files rm -f ${exp_name}_??_????????_????????_{grid_U,grid_V,grid_W,grid_T,icemod,SBC,scalar,SBC_scalar,diad_T}.nc restart_dir="restart/nemo/$(printf %03d $((leg_number)))" if ! $(has_config pisces:offline) ; then [ -f ${restart_dir}/EMPave_old.dat ] && cp ${restart_dir}/EMPave_old.dat . fi fi # ! $leg_is_restart #-------------------------------------------------------------------------- # *** Surface restoring and ocean nudging options #-------------------------------------------------------------------------- #for ocean_nudging if $(has_config nemo:ocenudg) ; then ln -fs ${ini_data_dir}/nemo/oce_nudg/temp_sal_*.nc ./ fi #for surface restoring if $(has_config nemo:surfresto) ; then ln -fs ${ini_data_dir}/nemo/surface_restoring/sss_restore_data*.nc ./ ln -fs ${ini_data_dir}/nemo/surface_restoring/sst_restore_data*.nc ./ ln -fs ${ini_data_dir}/nemo/surface_restoring/mask_restore*.nc ./ fi # ------------------------------------------------------------------------- # *** Remove land grid-points # ------------------------------------------------------------------------- if $(has_config nemo:elpin) then if [ ! -f coordinates_xios.nc ] then error "ELpIN requested, but file 'coordinates_xios.nc' was not found" fi jpns=($(${ecearth_src_dir}/util/ELPiN/ELPiNv2.cmd ${nem_numproc})) info "nemo domain decompostion from ELpIN: ${jpns[@]}" nem_numproc=${jpns[0]} nem_jpni=${jpns[1]} nem_jpnj=${jpns[2]} else info "nemo original domain decomposition (not using ELPiN)" fi # ------------------------------------------------------------------------- # *** Initial conditions saved during the run # ------------------------------------------------------------------------- ${do_save_ic} && save_ic_prepare_output # ------------------------------------------------------------------------- # *** Create some control files # ------------------------------------------------------------------------- # used in namelist.nemo is_climatology=".true." if [[ "${nem_forcing_set}" == "JRA55_1.4" || "${nem_forcing_set}" == "era5" || "${nem_forcing_set}" == "ERA5_HRES" || "${nem_forcing_set}" == "ERA5_ENDA" ]] ; then ln_dm2dc=".false." else ln_dm2dc=".true." fi if [[ ${nem_fixed_forcing_year-} == -1 ]] && ! [[ "${nem_forcing_set}" == "CoreII" ]] ; then is_climatology=".false." ; fi if [[ "${nem_forcing_set}" == "JRA55_1.4" ]] ; then forcing_runoff="namelist.nemo-runoff-JRA55_1.4" else forcing_runoff="namelist.nemo-runoff-default" fi # NEMO and LIM namelists . ${ctrl_file_dir}/namelist.nemo.ref.sh > namelist_ref . ${ctrl_file_dir}/namelist.nemo-${nem_grid}-standalone.cfg.sh > namelist_cfg . ${ctrl_file_dir}/namelist.lim3.ref.sh > namelist_ice_ref . ${ctrl_file_dir}/namelist.lim3-${nem_grid}.cfg.sh > namelist_ice_cfg # NEMO/TOP+PISCES namelists has_config pisces && . ${ctrl_file_dir}/namelist.nemo.top.ref.sh > namelist_top_ref has_config pisces && . ${ctrl_file_dir}/namelist.nemo.top.cfg.sh > namelist_top_cfg has_config pisces && . ${ctrl_file_dir}/namelist.nemo.pisces.ref.sh > namelist_pisces_ref has_config pisces && . ${ctrl_file_dir}/namelist.nemo.pisces.cfg.sh > namelist_pisces_cfg has_config nemo && . ${ctrl_file_dir}/namelist.nemo.age.ref.sh > namelist_age_ref has_config nemo && . ${ctrl_file_dir}/namelist.nemo.age.cfg.sh > namelist_age_cfg #include CFCs namelist even if CFCs are not included has_config nemo && . ${ctrl_file_dir}/namelist.nemo.cfc.ref.sh > namelist_cfc_ref has_config nemo && . ${ctrl_file_dir}/namelist.nemo.cfc.cfg.sh > namelist_cfc_cfg # ------------------------------------------------------------------------- # *** Link the appropriate NEMO restart files of the previous leg # ------------------------------------------------------------------------- if $leg_is_restart then ns=$(printf %08d $(( leg_start_sec / nem_time_step_sec - nem_restart_offset ))) for (( n=0 ; n