ecearth3/runtime/autosubmit/ece-nemo.sh.tmpl

#!/usr/bin/env bash

set -xuve

cd %CURRENT_ROOTDIR%

# librunscript defines some helper functions
source ./librunscript.sh

# =============================================================================
# *** BEGIN User configuration
# =============================================================================

# -----------------------------------------------------------------------------
# *** General configuration
# -----------------------------------------------------------------------------

# Component configuration
# (for syntax of the $config variable, see librunscript.sh)
# get config from autosubmit variables
[[ "%OCEAN_ini%" = *[!\ ]* ]] && start_nemo_from_restart=":start_from_restart" || start_nemo_from_restart=""
[[ "%PISCES_ini%" = *[!\ ]* ]] && start_pisces_from_restart=":start_from_restart" || start_pisces_from_restart=""
[ "%OCE_NUDG%" = TRUE ] && ocenudg=":ocenudg" || ocenudg=""
[ "%SURF_RESTO%" = TRUE ] && surfresto=":surfresto" || surfresto=""
[ "%PISCES%" = TRUE ] && [ "%PISCES_OFF%" = FALSE ] && pisces="pisces" || pisces=""
[ "%PISCES_OFF%" = TRUE ] && pisces="pisces:offline"
[ "%NEMO_remove_land%" = TRUE ] && elpin=":elpin" || elpin=""
if [[ "%SAVE_IC%" = FALSE ]] || [[ "%SAVE_IC%" = "" ]]; then save_ic="" ; else save_ic="save_ic:%SAVE_IC%" ; fi

config="nemo${start_nemo_from_restart}${ocenudg}${surfresto}${elpin} ${pisces}${start_pisces_from_restart} lim3 xios:detached ${save_ic}"

nem_forcing_set=%NEM_FORCING_SET%
nem_fixed_forcing_year=%NEM_FIXED_FORCING_YEAR%

if [ -z ${nem_forcing_set-} ] ; then nem_forcing_set=CoreII ; fi

# Experiment name (exactly 4 letters!)
export exp_name=[[[MOD:GENERAL:EXP_NAME]]]

# Simulation start and end date. Use any (reasonable) syntax you want.
run_start_date="[[[MOD:GENERAL:RUN_START_DATE]]]"
# define run_start_date_as with proper format e.g. 19900101
export run_start_date_as=$(date -u -d "${run_start_date}" +%Y%m%d)
run_end_date="[[[MOD:GENERAL:RUN_END_DATE]]]"

# Simulation member. Use any (reasonable) syntax you want.
export member="%MEMBER%"

# Set $force_run_from_scratch to 'true' if you want to force this run to start
# from scratch, possibly ignoring any restart files present in the run
# directory. Leave set to 'false' otherwise.
# NOTE: If set to 'true' the run directory $run_dir is cleaned!
CHUNK=%CHUNK%
force_run_from_scratch=[[[MOD:GENERAL:FORCE_RUN_FROM_SCRATCH]]]
force_run_from_scratch=${force_run_from_scratch:-false}
force_run_from_scratch=$(echo ${force_run_from_scratch} | tr '[:upper:]' '[:lower:]')
# we only apply this for the first chunk
if ${force_run_from_scratch} && [[ "${CHUNK}" != "1" ]] ; then
  force_run_from_scratch=false
fi

# Resolution
nem_grid=[[[MOD:NEM:GRID]]]

# Restart frequency. Use any (reasonable) number and time unit you want.
# For runs without restart, leave this variable empty
rst_freq="[[[MOD:GENERAL:RST_FREQ]]]"

# Number of restart legs to be run in one go
run_num_legs=[[[MOD:GENERAL:RUN_NUM_LEGS]]]

# Directories
start_dir=${PWD}
ctrl_file_dir=${start_dir}/ctrl
output_control_files_dir=${start_dir}/[[[MOD:GENERAL:OUTPUT_CONTROL_FILES_DIR]]]

# Architecture
build_arch=[[[PLT:ACTIVE:BUILD_ARCH]]]
use_machinefile=[[[PLT:ACTIVE:USE_MACHINEFILE]]]

# This file is used to store information about restarts
ece_info_file="ece.info"

# -----------------------------------------------------------------------------
# *** Read platform dependent configuration
# -----------------------------------------------------------------------------
source ./ecconf.cfg

configure

# -----------------------------------------------------------------------------
# *** Time step settings
# -----------------------------------------------------------------------------
case "${nem_grid}" in

    ORCA1L*)   nem_time_step_sec=2700; lim_time_step_sec=2700 ;;
    ORCA025L*) nem_time_step_sec=900 ; lim_time_step_sec=900  ;;

    *)  error "Can't set time steps for unknown horizontal grid: ${nem_grid}"
        ;;
esac

# -----------------------------------------------------------------------------
# *** NEMO/LIM configuration
# -----------------------------------------------------------------------------

# This is only needed if the experiment is started from an existing set of NEMO
# restart files
nem_restart_file_path=${ini_data_dir}

nem_restart_offset=0

nem_res_hor=$(echo ${nem_grid} | sed 's:ORCA\([0-9]\+\)L[0-9]\+:\1:')

# Pick correct NEMO configuration, which is one of:
#   NEMO standalone, NEMO+PISCES-standaone, PISCES-offline
                             nem_config_name=${nem_grid}_LIM3_standalone
has_config pisces         && nem_config_name=${nem_grid}_LIM3_PISCES_standalone
has_config pisces:offline && nem_config_name=${nem_grid}_OFF_PISCES_standalone

nem_exe_file=${ecearth_src_dir}/nemo-3.6/CONFIG/${nem_config_name}/BLD/bin/nemo.exe

nem_numproc=[[[MOD:NEM:NUMPROC]]]

# Thermal conductivity of snow, see comment in ctrl/namelist.lim3.ref.sh
nem_rn_cdsn=0.27

# -----------------------------------------------------------------------------
# *** XIOS configuration
# -----------------------------------------------------------------------------

xio_exe_file=${ecearth_src_dir}/xios-2.5/bin/xios_server.exe

xio_numproc=[[[MOD:XIO:NUMPROC]]]

# -----------------------------------------------------------------------------
# *** Extra initial conditions saved during the run
# -----------------------------------------------------------------------------
if has_config save_ic
then
    source ./libsave_ic.sh
    declare -a save_ic_date save_ic_date1 save_ic_sec save_ic_day save_ic_ppt_file save_ic_nemo_ts
fi

# =============================================================================
# *** END of User configuration
# =============================================================================

# =============================================================================
# *** This is where the code begins ...
# =============================================================================

# -----------------------------------------------------------------------------
# *** Create the run dir if necessary and go there
#     Everything is done from here.
# -----------------------------------------------------------------------------
if [ ! -d ${run_dir} ]
then
    mkdir -p ${run_dir}
fi
cd ${run_dir}

# -----------------------------------------------------------------------------
# *** Determine the time span of this run and whether it's a restart leg
# -----------------------------------------------------------------------------

# Regularise the format of the start and end date of the simulation
run_start_date=$(absolute_date_noleap "${run_start_date}")
run_end_date=$(absolute_date_noleap "${run_end_date}")

# Loop over the number of legs
for (( ; run_num_legs>0 ; run_num_legs-- ))
do

    # Check for restart information file and set the current leg start date
    #   Ignore restart information file if force_run_from_scratch is true
    if ${force_run_from_scratch} || ! [ -r ${ece_info_file} ]
    then
        leg_is_restart=false
        leg_start_date=${run_start_date}
        leg_number=1
    else
        leg_is_restart=true
        . ./${ece_info_file}
        leg_start_date=${leg_end_date}
        leg_number=$((leg_number+1))
    fi

    # Compute the end date of the current leg
    if [ -n "${rst_freq}" ]
    then
        leg_end_date=$(absolute_date_noleap "${leg_start_date} + ${rst_freq}")
    else
        leg_end_date=${run_end_date}
    fi

    if [ $(date -d "${leg_end_date}" +%s) -gt $(date -d "${run_end_date}" +%s) ]
    then
        leg_end_date=${run_end_date}
    fi

    # Some time variables needed later
    leg_length_sec=$(( $(date -d "${leg_end_date}" +%s) - $(date -d "${leg_start_date}" +%s) ))
    leg_start_sec=$(( $(date -d "${leg_start_date}" +%s) - $(date -d "${run_start_date}" +%s) ))
    leg_end_sec=$(( $(date -d "${leg_end_date}" +%s) - $(date -d "${run_start_date}" +%s) ))
    leg_start_date_yyyymmdd=$(date -u -d "${leg_start_date}" +%Y%m%d)
    leg_start_date_yyyy=$(date -u -d "${leg_start_date}" +%Y)
    leg_end_date_yyyy=$(date -u -d "${leg_end_date}" +%Y)

    # Correct for leap days because NEMO standalone uses no-leap calendar
    leg_length_sec=$(( leg_length_sec - $(leap_days "${leg_start_date}" "${leg_end_date}")*24*3600 ))
    leg_start_sec=$(( leg_start_sec - $(leap_days "${run_start_date}" "${leg_start_date}")*24*3600 ))
    leg_end_sec=$(( leg_end_sec - $(leap_days "${run_start_date}" "${leg_end_date}")*24*3600 ))

    # Check whether there's actually time left to simulate - exit otherwise
    if [ ${leg_length_sec} -le 0 ]
    then
        info "Leg start date equal to or after end of simulation."
        info "Nothing left to do. Exiting."
        exit 0
    fi

    # Initial conditions saved during the run
    do_save_ic=false
    save_ic_custom=false
    has_config save_ic && save_ic_get_config
    # if you do not use an option with save_ic, you must define 'do_save_ic' and
    # 'save_ic_date_offset' here or in ../libsave_ic.sh/save_ic_get_config()
    # with AS runtime, no need to edit the script, set SAVE_IC_OFFSET (and optionally SAVE_IC_COND)
    if $save_ic_custom
    then
        [[ "%SAVE_IC_COND%" = "" ]] && save_ic_cond=true || save_ic_cond='%SAVE_IC_COND%'
        if eval $save_ic_cond ; then do_save_ic=true ; else do_save_ic=false ; fi
        save_ic_date_offset=( %SAVE_IC_OFFSET% )
    fi
    ${do_save_ic} && save_ic_define_vars

    # -------------------------------------------------------------------------
    # *** Prepare the run directory for a run from scratch
    # -------------------------------------------------------------------------
    if ! $leg_is_restart
    then
        # ---------------------------------------------------------------------
        # *** Check if run dir is empty. If not, and if we are allowed to do so
        #     by ${force_run_from_scratch}, remove everything
        # ---------------------------------------------------------------------
        if $(ls * >& /dev/null)
        then
            if ${force_run_from_scratch}
            then
                rm -fr ${run_dir}/*
            else
                error "Run directory not empty and \$force_run_from_scratch not set."
            fi
        fi

        # ---------------------------------------------------------------------
        # *** Copy executables of model components
        # *** Additionally, create symlinks to the original place for reference
        # ---------------------------------------------------------------------
        cp    ${nem_exe_file} .
        ln -s ${nem_exe_file} $(basename ${nem_exe_file}).lnk

        cp    ${xio_exe_file} .
        ln -s ${xio_exe_file} $(basename ${xio_exe_file}).lnk

        # ---------------------------------------------------------------------
        # *** Files needed for NEMO (linked)
        # ---------------------------------------------------------------------

        # Link initialisation files for matching ORCA grid
        for f in \
            bathy_meter.nc coordinates.nc \
            ahmcoef.nc \
            K1rowdrg.nc M2rowdrg.nc mask_itf.nc \
            decay_scale_bot.nc decay_scale_cri.nc \
            mixing_power_bot.nc mixing_power_cri.nc mixing_power_pyc.nc \
            runoff_depth.nc subbasins.nc
        do
            [ -f ${ini_data_dir}/nemo/initial/${nem_grid}/$f ] && ln -s ${ini_data_dir}/nemo/initial/${nem_grid}/$f
        done

        # Copying the time independent NEMO files for the matching ORCA grid in order to facilitate cmorisation:
        for f in \
            bathy_meter.nc subbasins.nc
        do
            mkdir -p output/nemo/ofx-data
            [ -f ${ini_data_dir}/nemo/initial/${nem_grid}/$f ] && cp -f ${ini_data_dir}/nemo/initial/${nem_grid}/$f output/nemo/ofx-data/
        done

        # Link geothermal heating file (independent of grid) and matching weight file
        ln -s ${ini_data_dir}/nemo/initial/Goutorbe_ghflux.nc
        ln -s ${ini_data_dir}/nemo/initial/weights_Goutorbe1_2_orca${nem_res_hor}_bilinear.nc

        # Link either restart files or climatology files for the initial state
        if $(has_config nemo:start_from_restart)
        then
            # When linking restart files, we accept three options:
            # (1) Merged files for ocean and ice, i.e.
            #     restart_oce.nc and restart_ice.nc
            # (2) One-file-per-MPI-rank, i.e.
            #     restart_oce_????.nc and restart_ice_????.nc
            #     No check is done whether the number of restart files agrees
            #     with the number of MPI ranks for NEMO!
            # (3) One-file-per-MPI-rank with a prefix, i.e.
            #     <exp_name>_<time_step>_restart_oce_????.nc (similar for the ice)
            #     The prefix is ignored.
            # The code assumes that one of the options can be applied! If more
            # options are applicable, the first is chosen. If none of the
            # options apply, NEMO will crash with missing restart file.
            if   ls -U ${nem_restart_file_path}/restart_[oi]ce.nc > /dev/null 2>&1
            then
                 ln -s ${nem_restart_file_path}/restart_[oi]ce.nc ./

            elif ls -U ${nem_restart_file_path}/restart_[oi]ce_????.nc > /dev/null 2>&1
            then
                 ln -s ${nem_restart_file_path}/restart_[oi]ce_????.nc ./

            else
                for f in ${nem_restart_file_path}/????_????????_restart_[oi]ce_????.nc
                do
                    ln -s $f $(echo $f | sed 's/.*_\(restart_[oi]ce_....\.nc\)/\1/')
                done
            fi
        else

            # Temperature and salinity files for initialisation
            ln -s ${ini_data_dir}/nemo/climatology/absolute_salinity_WOA13_decav_Reg1L75_clim.nc
            ln -s ${ini_data_dir}/nemo/climatology/conservative_temperature_WOA13_decav_Reg1L75_clim.nc
            ln -s ${ini_data_dir}/nemo/climatology/weights_WOA13d1_2_orca${nem_res_hor}_bilinear.nc
        fi

        # Grid dependent runoff files
        case ${nem_grid} in
          ORCA1*)
            ln -s ${ini_data_dir}/nemo/climatology/runoff-icb_DaiTrenberth_Depoorter_ORCA1_JD.nc
            if [[ "${nem_forcing_set}" == "JRA55_1.4" ]] ; then
              ln -s ${ini_data_dir}/nemo/forcing/JRA55_1.4/runoff_depth_JRA.nc
            fi
            ;;
          ORCA025*)
            ln -s ${ini_data_dir}/nemo/climatology/ORCA_R025_runoff_v1.1.nc
            ;;
        esac

        #for ocean_nudging
        if $(has_config nemo:ocenudg) ; then
          ln -fs ${ini_data_dir}/nemo/oce_nudg/resto.nc ./
        fi
 
        # Salinity file (needed for diagnostics)
        [ -f ${ini_data_dir}/nemo/climatology/${nem_grid}/sali_ref_clim_monthly.nc ] && ln -s ${ini_data_dir}/nemo/climatology/${nem_grid}/sali_ref_clim_monthly.nc

        # Write fake file for previous fresh water budget adjustment (nn_fwb==2 in namelist)
        echo "                               0  0.0000000000000000E+00  0.0000000000000000E+00" > EMPave_old.dat

        # XIOS files
        . ${ctrl_file_dir}/iodef.xml.sh > iodef.xml
        ln -s ${ctrl_file_dir}/context_nemo.xml
	. ${ctrl_file_dir}/domain_def_nemo.xml.sh > domain_def_nemo.xml
        ln -s ${ctrl_file_dir}/axis_def_nemo.xml
        ln -s ${ctrl_file_dir}/grids_def_nemo.xml
        ln -s ${ctrl_file_dir}/field_def_nemo-lim.xml
        ln -s ${ctrl_file_dir}/field_def_nemo-opa.xml
        ln -s ${ctrl_file_dir}/field_def_nemo-pisces.xml
        ln -s ${ctrl_file_dir}/field_def_nemo-inerttrc.xml
        ln -s ${output_control_files_dir}/file_def_nemo-lim3.xml file_def_nemo-lim.xml
        ln -s ${output_control_files_dir}/file_def_nemo-opa.xml
        ln -s ${output_control_files_dir}/file_def_nemo-pisces.xml

        if [ -f ${ini_data_dir}/xios/ORCA${nem_res_hor}/coordinates_xios.nc ]
        then
            cp ${ini_data_dir}/xios/ORCA${nem_res_hor}/coordinates_xios.nc ./
        else
            info "File 'coordinates_xios.nc' not found. NEMO can not be run with land domain removal!"
        fi

        # ---------------------------------------------------------------------
        # *** Files needed for TOP/PISCES (linked)
        # ---------------------------------------------------------------------
        if $(has_config pisces)
        then
            ln -fs ${ini_data_dir}/nemo/pisces/dust_INCA_ORCA_R1.nc
            ln -fs ${ini_data_dir}/nemo/pisces/ndeposition_Duce_ORCA_R1.nc
            ln -fs ${ini_data_dir}/nemo/pisces/pmarge_etopo_ORCA_R1.nc
            ln -fs ${ini_data_dir}/nemo/pisces/river_global_news_ORCA_R1.nc
            ln -fs ${ini_data_dir}/nemo/pisces/Solubility_T62_Mahowald_ORCA_R1.nc

            ln -fs ${ini_data_dir}/nemo/pisces/par_fraction_gewex_clim90s00s_ORCA_R1.nc
            ln -fs ${ini_data_dir}/nemo/pisces/DIC_GLODAP_annual_ORCA_R1.nc
            ln -fs ${ini_data_dir}/nemo/pisces/Alkalini_GLODAP_annual_ORCA_R1.nc
            ln -fs ${ini_data_dir}/nemo/pisces/O2_WOA2009_monthly_ORCA_R1.nc
            ln -fs ${ini_data_dir}/nemo/pisces/PO4_WOA2009_monthly_ORCA_R1.nc
            ln -fs ${ini_data_dir}/nemo/pisces/Si_WOA2009_monthly_ORCA_R1.nc
            ln -fs ${ini_data_dir}/nemo/pisces/DOC_PISCES_monthly_ORCA_R1.nc
            ln -fs ${ini_data_dir}/nemo/pisces/Fer_PISCES_monthly_ORCA_R1.nc
            ln -fs ${ini_data_dir}/nemo/pisces/NO3_WOA2009_monthly_ORCA_R1.nc
            ln -fs ${ini_data_dir}/nemo/pisces/mole-fraction-of-carbon-dioxide-in-air_input4MIPs_GHGConcentrations_CMIP_UoM-CMIP-1-2-0_gr1-GMNHSH_1637-2019.txt atcco2.txt
        fi
            #linking surface boundary conditions for CFCs (even if CFCs are not included)
            ln -fs ${ini_data_dir}/nemo/cfc/CFCs_CDIAC_extension_1637_2019.dat CFCs_CDIAC.dat

        if $(has_config pisces:start_from_restart)
        then
            # Same three options as for nemo:start_from_restart
            if   ls -U ${nem_restart_file_path}/restart_trc.nc > /dev/null 2>&1
            then
                ln -s ${nem_restart_file_path}/restart_trc.nc ./

            elif ls -U ${nem_restart_file_path}/restart_trc_????.nc > /dev/null 2>&1
            then
                ln -s ${nem_restart_file_path}/restart_trc_????.nc ./

            else
                for f in ${nem_restart_file_path}/????_????????_restart_trc_????.nc
                do
                    ln -s $f $(echo $f | sed 's/.*_\(restart_trc_....\.nc\)/\1/')
                done
            fi
        fi

        # ---------------------------------------------------------------------
        # *** Files needed for TOP/PISCES (linked) ADDED FOR PISCES OFFLINE
        # ---------------------------------------------------------------------
        if $(has_config pisces:offline)
        then
            ln -fs ${ini_data_dir}/nemo/pisces/pisces_dyn/OfflineForcing_grid_U.nc OfflineForcing_grid_U.nc
            ln -fs ${ini_data_dir}/nemo/pisces/pisces_dyn/OfflineForcing_grid_V.nc OfflineForcing_grid_V.nc
            ln -fs ${ini_data_dir}/nemo/pisces/pisces_dyn/OfflineForcing_grid_W.nc OfflineForcing_grid_W.nc
            ln -fs ${ini_data_dir}/nemo/pisces/pisces_dyn/OfflineForcing_grid_T.nc OfflineForcing_grid_T.nc
            ln -fs ${ctrl_file_dir}/context_nemo_offline.xml context_nemo.xml
        fi

    else # i.e. $leg_is_restart == true

        # ---------------------------------------------------------------------
        # *** Remove all leftover output files from previous legs
        # ---------------------------------------------------------------------

        # NEMO output files
        rm -f ${exp_name}_??_????????_????????_{grid_U,grid_V,grid_W,grid_T,icemod,SBC,scalar,SBC_scalar,diad_T}.nc

        restart_dir="restart/nemo/$(printf %03d $((leg_number)))"
        if ! $(has_config pisces:offline) ; then
            [ -f ${restart_dir}/EMPave_old.dat ] && cp ${restart_dir}/EMPave_old.dat .
        fi
    fi # ! $leg_is_restart

    #--------------------------------------------------------------------------
    # *** Surface restoring and ocean nudging options
    #--------------------------------------------------------------------------
        #for ocean_nudging
        if $(has_config nemo:ocenudg) ; then
          ln -fs ${ini_data_dir}/nemo/oce_nudg/temp_sal_*.nc ./
        fi


        #for surface restoring
        if $(has_config nemo:surfresto) ; then
          ln -fs ${ini_data_dir}/nemo/surface_restoring/sss_restore_data*.nc  ./
          ln -fs ${ini_data_dir}/nemo/surface_restoring/sst_restore_data*.nc  ./
          ln -fs ${ini_data_dir}/nemo/surface_restoring/mask_restore*.nc ./
        fi

    # -------------------------------------------------------------------------
    # *** Remove land grid-points
    # -------------------------------------------------------------------------
    if $(has_config nemo:elpin)
    then
        if [ ! -f coordinates_xios.nc ]
        then
            error "ELpIN requested, but file 'coordinates_xios.nc' was not found"
        fi
        jpns=($(${ecearth_src_dir}/util/ELPiN/ELPiNv2.cmd ${nem_numproc}))
        info "nemo domain decompostion from ELpIN: ${jpns[@]}"
        nem_numproc=${jpns[0]}
        nem_jpni=${jpns[1]}
        nem_jpnj=${jpns[2]}
    else
        info "nemo original domain decomposition (not using ELPiN)"
    fi

    # -------------------------------------------------------------------------
    # *** Initial conditions saved during the run
    # -------------------------------------------------------------------------
    ${do_save_ic} && save_ic_prepare_output

    # -------------------------------------------------------------------------
    # *** Create some control files
    # -------------------------------------------------------------------------

    # used in namelist.nemo
    is_climatology=".true."
    if [[ "${nem_forcing_set}" == "JRA55_1.4" || "${nem_forcing_set}" == "era5" || "${nem_forcing_set}" == "ERA5_HRES" || "${nem_forcing_set}" == "ERA5_ENDA" ]] ; then
      ln_dm2dc=".false."
    else
      ln_dm2dc=".true."
    fi

    if [[ ${nem_fixed_forcing_year-} == -1 ]]  && ! [[ "${nem_forcing_set}" == "CoreII" ]] ; then  is_climatology=".false." ; fi

    if [[ "${nem_forcing_set}" == "JRA55_1.4" ]] ; then
      forcing_runoff="namelist.nemo-runoff-JRA55_1.4"
    else
      forcing_runoff="namelist.nemo-runoff-default"
    fi

    # NEMO and LIM namelists
    . ${ctrl_file_dir}/namelist.nemo.ref.sh                        > namelist_ref
    . ${ctrl_file_dir}/namelist.nemo-${nem_grid}-standalone.cfg.sh > namelist_cfg
    . ${ctrl_file_dir}/namelist.lim3.ref.sh                        > namelist_ice_ref
    . ${ctrl_file_dir}/namelist.lim3-${nem_grid}.cfg.sh            > namelist_ice_cfg

    # NEMO/TOP+PISCES namelists
    has_config pisces && . ${ctrl_file_dir}/namelist.nemo.top.ref.sh    > namelist_top_ref
    has_config pisces && . ${ctrl_file_dir}/namelist.nemo.top.cfg.sh    > namelist_top_cfg
    has_config pisces && . ${ctrl_file_dir}/namelist.nemo.pisces.ref.sh > namelist_pisces_ref
    has_config pisces && . ${ctrl_file_dir}/namelist.nemo.pisces.cfg.sh > namelist_pisces_cfg
    has_config nemo      && . ${ctrl_file_dir}/namelist.nemo.age.ref.sh                 > namelist_age_ref
    has_config nemo      && . ${ctrl_file_dir}/namelist.nemo.age.cfg.sh                 > namelist_age_cfg
    #include CFCs namelist even if CFCs are not included
    has_config nemo      && . ${ctrl_file_dir}/namelist.nemo.cfc.ref.sh                 > namelist_cfc_ref
    has_config nemo      && . ${ctrl_file_dir}/namelist.nemo.cfc.cfg.sh                 > namelist_cfc_cfg
    # -------------------------------------------------------------------------
    # *** Link the appropriate NEMO restart files of the previous leg
    # -------------------------------------------------------------------------
    if $leg_is_restart
    then
        ns=$(printf %08d $(( leg_start_sec / nem_time_step_sec - nem_restart_offset )))
        for (( n=0 ; n<nem_numproc ; n++ ))
        do
            np=$(printf %04d ${n})
            ln -fs ${exp_name}_${ns}_restart_oce_${np}.nc restart_oce_${np}.nc
            ln -fs ${exp_name}_${ns}_restart_ice_${np}.nc restart_ice_${np}.nc
            has_config pisces && \
            ln -fs ${exp_name}_${ns}_restart_trc_${np}.nc restart_trc_${np}.nc
        done

        # Make sure there are no global restart files
        # If links are found, they will be removed. We are cautious and do
        # _not_ remove real files! However, if real global restart files are
        # present, NEMO/LIM will stop because time stamps will not match.
        [ -h restart_oce.nc ] && rm restart_oce.nc
        [ -h restart_ice.nc ] && rm restart_ice.nc
        [ -h restart_trc.nc ] && rm restart_trc.nc
    fi

    # -------------------------------------------------------------------------
    # *** Start the run
    # -------------------------------------------------------------------------

    # Link NEMO forcing files


    if ! [[ "${nem_forcing_set}" == "CoreII" ]] && \
       ! [[ "${nem_forcing_set}" == "DFS5.2" ]] && \
       ! [[ "${nem_forcing_set}" == "eraint" ]] && \
       ! [[ "${nem_forcing_set}" == "CoreII_interannual" ]]&& \
       ! [[ "${nem_forcing_set}" == "era5" ]] && \
       ! [[ "${nem_forcing_set}" == "ERA5_HRES" ]] && \
       ! [[ "${nem_forcing_set}" == "ERA5_ENDA" ]] && \
       ! [[ "${nem_forcing_set}" == "JRA55_1.4" ]]
    then
        echo "${nem_forcing_set}" not supported;
        exit 1
    fi

    case "${nem_forcing_set}" in
      CoreII)
          for variable in u_10 v_10 t_10 q_10 ncar_precip ncar_rad ; do
              [ -f "${ini_data_dir}/nemo/forcing/CoreII/${variable}.15JUNE2009_fill.nc" ] && ln -sf "${ini_data_dir}/nemo/forcing/CoreII/${variable}.15JUNE2009_fill.nc"
          done
          # Weight files for forcing
          ln -fs ${ini_data_dir}/nemo/forcing/CoreII/weights_coreII_2_orca${nem_res_hor}_bilinear.nc
          ln -fs ${ini_data_dir}/nemo/forcing/CoreII/weights_coreII_2_orca${nem_res_hor}_bicubic.nc
          ;;
      * )
          for forcing_file in ${ini_data_dir}/nemo/forcing/${nem_forcing_set}/* ; do
            [ -f ${forcing_file} ] && ln -fs ${forcing_file} ./$(basename ${forcing_file})
          done
          #Weight files for forcing
          ln -sf ${ini_data_dir}/nemo/forcing/weights/* .
      ;;
    esac

    # Use the launch function from the platform configuration file
    t1=$(date +%s)
    launch \
        ${xio_numproc} ${xio_exe_file} -- \
        ${nem_numproc} ${nem_exe_file}
    t2=$(date +%s)

    tr=$(date -d "0 -$t1 sec + $t2 sec" +%T)

    # -------------------------------------------------------------------------
    # *** Check for signs of success
    #     Note the tests provide no guarantee that things went fine! They are
    #     just based on the IFS and NEMO log files. More tests (e.g. checking
    #     restart files) could be implemented.
    # -------------------------------------------------------------------------
    # Check for NEMO success
    if [ -f ocean.output ]
    then
        if [ "$(sed -n '/New day/h; ${g;s:.*\([0-9/]\{10\}\).*:\1:;p;}' ocean.output)" == "$(date -u -d "$(absolute_date_noleap "${leg_end_date} - 1 day")" +%Y/%m/%d)" ]
        then
            info "Leg successfully completed according to NEMO log file 'ocean.output'."
        else
            error "Leg not completed according to NEMO log file 'ocean.output'."
        fi

    else
        error "NEMO log file 'ocean.output' not found after run."
    fi

    # -------------------------------------------------------------------------
    # *** Post-process initial conditions saved during the run if requested
    # -------------------------------------------------------------------------
    ${do_save_ic} && save_ic_postproc

    # -------------------------------------------------------------------------
    # *** Move NEMO output files to archive directory
    # -------------------------------------------------------------------------
    outdir="output/nemo/$(printf %03d $((leg_number)))"
    mkdir -p ${outdir}

    for v in grid_U grid_V grid_W grid_T icemod SBC scalar SBC_scalar diad_T ptrc_T bioscalar \
             grid_T_2D grid_U_2D grid_V_2D grid_W_2D grid_T_3D grid_U_3D grid_V_3D grid_W_3D \
             grid_T_SFC grid_1point grid_T_3D_ncatice vert_sum \
             grid_ptr_W_3basin_3D grid_ptr_T_3basin_2D grid_ptr_T_2D \
             zoom_700_sum zoom_300_sum zoom_2000_sum
    do
        for f in ${exp_name}_*_????????_????????_*${v}.nc
        do
            test -f $f && mv $f $outdir/
        done
    done

    # -------------------------------------------------------------------------
    # *** Move NEMO restart files to archive directory
    # -------------------------------------------------------------------------
    if $leg_is_restart
    then
        outdir="restart/nemo/$(printf %03d $((leg_number)))"
        mkdir -p ${outdir}

        ns=$(printf %08d $(( leg_start_sec / nem_time_step_sec - nem_restart_offset )))
        for f in oce ice trc
        do
            test -f ${exp_name}_${ns}_restart_${f}_0000.nc && \
                 mv ${exp_name}_${ns}_restart_${f}_????.nc ${outdir}
        done
        if ! $(has_config pisces:offline)
        then
            cp EMPave_old.dat ${outdir}/EMPave_old.dat
        fi
    fi
    if ! $(has_config pisces:offline)
    then
        cp EMPave.dat EMPave_old.dat
    fi
    # -------------------------------------------------------------------------
    # *** Move log files to archive directory
    # -------------------------------------------------------------------------
    outdir="log/$(printf %03d $((leg_number)))"
    mkdir -p ${outdir}

    for f in \
        ocean.output time.step solver.stat
    do
        test -f ${f} && mv ${f} ${outdir}
    done

    # -------------------------------------------------------------------------
    # *** Write the restart control file
    # -------------------------------------------------------------------------


    # Compute CPMIP performance
    sypd="$(cpmip_sypd $leg_length_sec $(($t2 - $t1)))"
    chpsy="$(cpmip_chpsy  $leg_length_sec $(($t2 - $t1)) $(($nem_numproc + $xio_numproc)))"

    echo "#"                                             | tee -a ${ece_info_file}
    echo "# Finished leg at `date '+%F %T'` after ${tr} (hh:mm:ss)" \
                                                         | tee -a ${ece_info_file}
    echo "# CPMIP performance: $sypd SYPD   $chpsy CHPSY"| tee -a ${ece_info_file}
    echo "leg_number=${leg_number}"                      | tee -a ${ece_info_file}
    echo "leg_start_date=\"${leg_start_date}\""          | tee -a ${ece_info_file}
    echo "leg_end_date=\"${leg_end_date}\""              | tee -a ${ece_info_file}

    # Need to reset force_run_from_scratch in order to avoid destroying the next leg
    force_run_from_scratch=false

done # loop over legs

# -----------------------------------------------------------------------------
# *** Platform dependent finalising of the run
# -----------------------------------------------------------------------------
finalise