ecearth3/runtime/autosubmit/ece-esm.sh.tmpl

#!/usr/bin/env bash

set -xuve

cd %CURRENT_ROOTDIR%

# librunscript defines some helper functions
source ./librunscript.sh

# =============================================================================
# *** BEGIN User configuration
# =============================================================================

# -----------------------------------------------------------------------------
# *** General configuration
# -----------------------------------------------------------------------------
# Component configuration (for syntax of the $config variable, see librunscript.sh)
#
# Currently maintained:
#     config="ifs amip"                      # "GCM forced-SST" : IFS + AMIP
#     config="ifs amip lpjg:fdbck tm5:co2"   # "C-cycle"        : forced-GCM + LPJ-Guess + TM5
#     config="ifs amip lpjg:fdbck"           # "Veg"            : forced-GCM + LPJ-Guess
#     config="ifs amip tm5:chem,o3,ch4,aero" # "AerChem"        : forced-GCM + TM5
#
#     config="ifs nemo lim3 rnfmapper xios:detached oasis"                      # "GCM"     : IFS+NEMO
#     config="ifs nemo lim3 rnfmapper xios:detached oasis lpjg:fdbck tm5:co2"   # "C-cycle" : GCM+LPJG+TM5
#     config="ifs nemo lim3 rnfmapper xios:detached oasis lpjg:fdbck"           # "Veg"     : GCM+LPJ-Guess
#     config="ifs nemo lim3 rnfmapper xios:detached oasis tm5:chem,o3,ch4,aero" # "AerChem" : GCM+TM5

#config="ifs nemo lim3 rnfmapper xios:detached oasis lpjg:fdbck tm5:co2"

# get config from autosubmit variables

#ifs
ifs="ifs"
TEMPLATE_NAME=%TEMPLATE_NAME%
[[ "$TEMPLATE_NAME" = ifs3* ]] && amip="amip" || amip=""
[ "%ATM_NUDGING%" = TRUE ] && atmnudg="atmnudg" || atmnudg=""
[ "%LSPPT%" = TRUE ] && sppt="sppt" || sppt=""

#nemo
nemo=""; pisces=""; lim3=""; rnfmapper=""; xios=""
[[ "$TEMPLATE_NAME" = nemo3* ]] && error "ece-esm.sh runscript does not support nemo only!"
if [[ "$TEMPLATE_NAME" = ecearth3* ]];
then
  [[ "%OCEAN_ini%" = *[!\ ]* ]] && start_from_restart=":start_from_restart" || start_from_restart=""
  [ "%OCE_NUDG" = TRUE ] && ocenudg=":ocenudg" || ocenudg=""
  [ "%SURF_RESTO%" = TRUE ] && surfresto=":surfresto" || surfresto=""
  [ "%NEMO_remove_land%" = TRUE ] && elpin=":elpin" || elpin=""
  nemo="nemo"${start_from_restart}${ocenudg}${surfresto}${elpin}
  [ "%PISCES%" = TRUE ] && pisces="pisces" || pisces=""
  lim3="lim3"; xios="xios:detached"; rnfmapper="rnfmapper"
fi

#others
[[ "%LPJG%" = TRUE ]] && lpjg=%LPJG_CONFIG% || lpjg=""
[ "%NEMO_remove_land%" = TRUE ] && elpin=":elpin" || elpin=""
[ "%IFS_VEG_SOURCE%" == "" ]  && ifs_veg_source="era20c" || ifs_veg_source="%IFS_VEG_SOURCE%"
if [[ "%SAVE_IC%" = FALSE ]] || [[ "%SAVE_IC%" = "" ]] || [[ "%SAVE_IC%" = "%%SAVE_IC%%" ]]; then save_ic="" ; else save_ic="save_ic:%SAVE_IC%" ; fi

config="${ifs} ${amip} ${atmnudg} ${sppt} ${nemo} ${pisces} ${lim3} ${rnfmapper} ${xios} oasis ${lpjg}"

# minimum sanity
has_config amip nemo && error "Cannot have both nemo and amip in config!!"

# Experiment name (exactly 4 letters!)
export exp_name=[[[MOD:GENERAL:EXP_NAME]]]

# Simulation start and end date. Use any (reasonable) syntax you want.
run_start_date="[[[MOD:GENERAL:RUN_START_DATE]]]"
run_end_date="[[[MOD:GENERAL:RUN_END_DATE]]]"

# Simulation member. Use any (reasonable) syntax you want.
member="%MEMBER%"

# Set $force_run_from_scratch to 'true' if you want to force this run to start
# from scratch, possibly ignoring any restart files present in the run
# directory. Leave set to 'false' otherwise.
# NOTE: If set to 'true' the run directory $run_dir is cleaned!
force_run_from_scratch=[[[MOD:GENERAL:FORCE_RUN_FROM_SCRATCH]]]

# Resolution (TM5 resolution is set at compilation)
ifs_grid=[[[MOD:IFS:GRID]]]
nem_grid=[[[MOD:NEM:GRID]]]

# Restart frequency. Use any (reasonable) number and time unit you want.
# For runs without restart, leave this variable empty
rst_freq="[[[MOD:GENERAL:RST_FREQ]]]"

# Number of restart legs to be run in one go
run_num_legs=[[[MOD:GENERAL:RUN_NUM_LEGS]]]

# Coupling frequencies
has_config ifs tm5  && cpl_freq_atm_ctm_hrs=[[[MOD:OASIS:CPL_FREQ_ATM_CHE_HRS]]]
has_config ifs lpjg && cpl_freq_atm_lpjg_hrs=[[[MOD:OASIS:CPL_FREQ_ATM_LPJG_HRS]]]

# Don't change the coupling frequency because UPDCLIE (where SST and SIC
# are updated) is called every 24 hours (hardcoded in ifs-36r4/src/ifs/utility/updtim.F90)
has_config amip && cpl_freq_amip_sec=86400

# Directories
start_dir=${PWD}
ctrl_file_dir=${start_dir}/ctrl
output_control_files_dir=${start_dir}/[[[MOD:GENERAL:OUTPUT_CONTROL_FILES_DIR]]]

# Architecture
build_arch=[[[PLT:ACTIVE:BUILD_ARCH]]]

# This file is used to store information about restarts
ece_info_file="ece.info"

# -----------------------------------------------------------------------------
# *** Read platform dependent configuration
# -----------------------------------------------------------------------------
source ./ecconf.cfg

configure

# -----------------------------------------------------------------------------
# *** Time step settings
# -----------------------------------------------------------------------------
if has_config ifs
then
    case "${ifs_grid}" in

        T159L*) ifs_time_step_sec=3600 ;;
        T255L*) ifs_time_step_sec=2700 ;;
        T511L*) ifs_time_step_sec=900  ;;

        *)  error "Can't set time steps for unknown horizontal grid: ${ifs_grid}"
            ;;
    esac
fi

if has_config nemo
then
    case "${nem_grid}" in

        ORCA1L*)   nem_time_step_sec=2700; lim_time_step_sec=2700 ;;
        ORCA025L*) nem_time_step_sec=900 ; lim_time_step_sec=900  ;;

        *)  error "Can't set time steps for unknown horizontal grid: ${nem_grid}"
            ;;
    esac
fi

if has_config ifs nemo
then
    case "${ifs_grid}--${nem_grid}" in

        T159L*--ORCA1L*)
            ifs_time_step_sec=3600; nem_time_step_sec=2700; lim_time_step_sec=2700; cpl_freq_atm_oce_sec=10800
            ;;
        T255L*--ORCA1L*)
            ifs_time_step_sec=2700; nem_time_step_sec=2700; lim_time_step_sec=2700; cpl_freq_atm_oce_sec=2700
            ;;
        T511L*--ORCA025L*)
            ifs_time_step_sec=900 ; nem_time_step_sec=900 ; lim_time_step_sec=900 ; cpl_freq_atm_oce_sec=2700
            ;;

        *)  error "Can't set time steps for unknown combination of horizontal grids: ${ifs_grid}-${nem_grid}"
            ;;
    esac
fi

# -----------------------------------------------------------------------------
# *** IFS configuration
# -----------------------------------------------------------------------------

ifs_version=36r4

ifs_di_freq=$(( 24 * 3600 / ifs_time_step_sec ))
ifs_ddh_freq=$(( 120 * 3600 / ifs_time_step_sec ))

export ifs_res_hor=$(echo ${ifs_grid} | sed 's:T\([0-9]\+\)L\([0-9]\+\):\1:')
ifs_res_ver=$(echo ${ifs_grid} | sed 's:T\([0-9]\+\)L\([0-9]\+\):\2:')

ifs_numproc=[[[MOD:IFS:NUMPROC]]]

ifs_exe_file=${ecearth_src_dir}/ifs-${ifs_version}/bin/ifsmaster-${build_arch}

ifs_lastout=false

ifs_cmip5=[[[MOD:IFS:CMIP5]]]
ifs_cmip5_rcp=[[[MOD:IFS:CMIP5_RCP]]]
ifs_cmip_fixyear=[[[MOD:IFS:CMIP_FIXYEAR]]]
[ -z "${ifs_cmip_fixyear}" ] && ifs_cmip_fixyear=0

ifs_cmip6=[[[MOD:IFS:CMIP6]]]
ifs_mac2sp=[[[MOD:IFS:CMIP6]]]
ifs_cmip6piaer=[[[MOD:IFS:CMIP6]]]
ifs_cmip6_scenario=[[[MOD:IFS:CMIP6_SCENARIO]]]

lcmip6_strataer_simp=FALSE
lcmip6_strataer_full=[[[MOD:IFS:CMIP6]]]
lcmip6_strataer_bckgd=FALSE

ifs_A4xCO2=[[[MOD:IFS:CMIP6_A4xCO2]]]
ifs_1PCTCO2=[[[MOD:IFS:CMIP6_1PCTCO2]]]

# Time-varying orbital forcing (Qiong Zhang, SU-2013-09)
# https://dev.ec-earth.org/projects/ecearth3/wiki/Orbital_forcing_in_EC-Earth_3
#
#   ifs_orb_switch=FALSE, no orbital calculations applied
#   ifs_orb_switch=TRUE, use orbital calculations according to ifs_orb_mode
#   ifs_orb_mode="fixed_year", or "variable_year", or "fixed_parameters"
#     fixed_year: calculate the orbital parameters at ifs_orb_iyear, e.g.,1850
#     variable_year: calculate orbital parameters annually start from ifs_orb_iyear
#     fixed_parameters: prescribe orbital parameters for given year
ifs_orb_switch=FALSE
ifs_orb_mode="variable_year"
ifs_orb_iyear=$(date -u -d "${run_start_date}" +%Y)

# IFS tuning parameters
ifs_tuning_parameter_file=${ctrl_file_dir}/ifs-tuning-parameters-${ifs_grid}.sh
if [ -f ${ifs_tuning_parameter_file} ]
then
    source ${ifs_tuning_parameter_file}
else
    error "Sorry, ${ifs_tuning_parameter_file} not found, exiting."
fi


# Select source of vegetation data:
#  ifs       climatology from IFS
#  era20c    vegetation from an off-line LPJ-Guess run forced with ERA20C
#            (currently available only for T255 and T159)
#  cmip6     vegetation from an EC-Earth3-Veg (interactive LPJ-Guess) run 
#            (currently available only for T255)
#  none      don't create an ICMCL file with vegetation data (this is set
#            automatically if LPJG is used with feedback)
#
# set above in AS runtime

has_config lpjg:fdbck && ifs_veg_source="none"

case ${ifs_veg_source} in
"ifs" )
    # Use Lambert-Beer to compute effective vegetation cover
    n_compute_eff_veg_fraction=2
    ;;
"era20c" )
    # LPJG vegetation is provided as effective cover
    # Don't use Lambert-Beer
    n_compute_eff_veg_fraction=0

    case "${ifs_grid}" in
        T159L*) veg_version=v29 ;;
        T255L*) veg_version=v16 ;;
        *)  error "Vegetation from off-line LPJ-Guess not available for ${ifs_grid}" ;;
    esac    
    ;;
"cmip6" )
    # LPJG vegetation is provided as effective cover
    # Don't use Lambert-Beer
    n_compute_eff_veg_fraction=0

    case "${ifs_grid}" in
        T255L*) veg_version=v32 ;;
        *)  error "Vegetation from CMIP6 EC-Earth3-Veg not available for ${ifs_grid}" ;;
    esac    
    ;;
"none" )
    # LPJG with feedback
    n_compute_eff_veg_fraction=0
    ! has_config lpjg:fdbck && error "IFS requires an offline source of vegetation"
    ;;
* )
    error "Vegetation from ${ifs_veg_source} not implemented"
    ;;
esac

# use DMI land ice physics and varying snow albedo
ifs_landice=false

# -----------------------------------------------------------------------------
# *** NEMO/LIM configuration
# -----------------------------------------------------------------------------

# This is only needed if the experiment is started from an existing set of NEMO
# restart files
nem_restart_file_path=${ini_data_dir}

nem_restart_offset=0

nem_res_hor=$(echo ${nem_grid} | sed 's:ORCA\([0-9]\+\)L[0-9]\+:\1:')

#TODO: fix nemo config
{ has_config pisces && nem_config_name=${nem_grid}_LIM3_PISCES; } \
|| { has_config ocenudg && nem_config_name=${nem_grid}_LIM3_NUDG; } \
|| nem_config_name=${nem_grid}_LIM3

nem_exe_file=${ecearth_src_dir}/nemo-3.6/CONFIG/${nem_config_name}/BLD/bin/nemo.exe

nem_numproc=[[[MOD:NEM:NUMPROC]]]

# -----------------------------------------------------------------------------
# *** Runoff mapper configuration
# -----------------------------------------------------------------------------

rnf_exe_file=${ecearth_src_dir}/runoff-mapper/bin/runoff-mapper.exe
rnf_numproc=1

# -----------------------------------------------------------------------------
# *** LPJ-GUESS configuration
# -----------------------------------------------------------------------------

lpjg_time_step_sec=[[[MOD:LPJG:TIME_STEP_SEC]]]
lpjg_numproc=[[[MOD:LPJG:NUMPROC]]]

has_config lpjg       && lpjg_on=1
has_config lpjg:fdbck && lpjg_fdbck=1
has_config tm5        && lpjg_fdbck_tm5=1 || lpjg_fdbck_tm5=0

export lpjg_fixNdepafter=[[[MOD:LPJG:LPJG_FIXNDEPAFTER]]]
export lpjg_fixLUafter=[[[MOD:LPJG:LPJG_FIXLUAFTER]]]
if [ $ifs_cmip_fixyear -gt 0 ]
then 
    info '!!!! CMIP FIX YEAR SETTINGS:'
    info "ifs_cmip_fixyear:  $ifs_cmip_fixyear"
    info "lpjg_fixNDepAfter: $lpjg_fixNdepafter"
    info "lpjg_fixLUAfter:   $lpjg_fixLUafter"
    info '!!!!'
fi    

lpjg_res=T${ifs_res_hor}
lpjg_exe_file=${ecearth_src_dir}/lpjg/build/guess_${lpjg_res}

# -----------------------------------------------------------------------------
# *** AMIP-reader configuration
# -----------------------------------------------------------------------------

amip_exe_file=${ecearth_src_dir}/amip-forcing/bin/amip-forcing.exe
amip_numproc=1

# -----------------------------------------------------------------------------
# *** TM5 configuration
# -----------------------------------------------------------------------------
if $(has_config tm5)
then
    # With TM5, NPRTRV is set to 1 in the namelist. To avoid some out-of-bound
    # arrays in IFS, we must limit the number of cores for IFS
    if (( ifs_numproc > (ifs_res_hor+1) ))
    then
        error "too much cores requested for IFS, max is $((ifs_res_hor+1))"
    fi
fi

has_config tm5:co2 && export tm5_co2=1 || export tm5_co2=0
tm5_time_step_sec=[[[MOD:TM5:TIME_STEP_SEC]]]
tm5_numproc=$(( [[[MOD:TM5:NUMPROC_X]]] * [[[MOD:TM5:NUMPROC_Y]]] ))

# nb of coupled levels: nb of levels in either TM5 or IFS (must fit what's used at compilation)
export tm5_exch_nlevs=34
# limited number of levels for aerosols,
# currently set to lmax_conv
export tm5_exch_nlevs_cutoff=23

# executable
has_config tm5:co2  && tm5_exe_file=${ecearth_src_dir}/tm5mp/build-co2/appl-tm5-co2.x
has_config tm5:chem && tm5_exe_file=${ecearth_src_dir}/tm5mp/build/appl-tm5.x

# fields sent back to IFS
has_config tm5:o3   && tm5_to_ifs=O3 || tm5_to_ifs=
has_config tm5:ch4  && tm5_to_ifs=${tm5_to_ifs},CH4
has_config tm5:aero && tm5_to_ifs=${tm5_to_ifs},"\
N2,SU2,BC2,OC2,N3,SU3,BC3,OC3,SS3,DU3,\
N4,SU4,BC4,OC4,SS4,DU4,N5,BC5,OC5,N6,DU6,N7,DU7,\
NO3,MSA,\
AOD_01,AOD_02,AOD_03,AOD_04,AOD_05,AOD_06,AOD_07,AOD_08,AOD_09,AOD_10,AOD_11,AOD_12,AOD_13,AOD_14,\
SSA_01,SSA_02,SSA_03,SSA_04,SSA_05,SSA_06,SSA_07,SSA_08,SSA_09,SSA_10,SSA_11,SSA_12,SSA_13,SSA_14,\
ASF_01,ASF_02,ASF_03,ASF_04,ASF_05,ASF_06,ASF_07,ASF_08,ASF_09,ASF_10,ASF_11,ASF_12,ASF_13,ASF_14"

export tm5_to_ifs=$(echo ${tm5_to_ifs} | sed "s/^,//")

# coupled to LPJ-Guess?
has_config tm5:co2 lpjg && export cpl_tm_guess=T || export cpl_tm_guess=F

# -----------------------------------------------------------------------------
# *** OASIS configuration
# -----------------------------------------------------------------------------

# Restart files for the coupling fields (note 8 character limit in OASIS)
#   rstas.nc : atmosphere single-category fields
#   rstam.nc : atmosphere multi-category fields
#   rstos.nc : ocean single-category fields
#   rstom.nc : ocean multi-category fields
oas_rst_ifs_nemo="rstas.nc rstos.nc"

oas_rst_ifs_tm5="r_hum.nc r_g2d.nc r_udr.nc r_cc_.nc r_clw.nc r_div.nc \
                 r_cco.nc r_vor.nc r_ddr.nc r_ciw.nc r_tmp.nc r_dmf.nc \
                 r_s2d.nc r_ccu.nc r_umf.nc"

has_config tm5:o3 || has_config tm5:ch4 && oas_rst_ifs_tm5=$oas_rst_ifs_tm5' o3ch4.nc'
has_config tm5:aero                     && oas_rst_ifs_tm5=$oas_rst_ifs_tm5' C???????'

# Decide whether the OASIS weight files for interpolation should be linked from
# the setup directory (true) or not (false). In the latter case, the weights
# are re-computed at the start of the run.
oas_link_weights=true

# Flux correction for runoff (not calving) sent from Oasis to ocean.
# 1.07945 is computed to compensate for a P-E=-0.016 mm/day
has_config nemo && oas_mb_fluxcorr=1.07945

# -----------------------------------------------------------------------------
# *** XIOS configuration
# -----------------------------------------------------------------------------

xio_exe_file=${ecearth_src_dir}/xios-2.5/bin/xios_server.exe

xio_numproc=[[[MOD:XIO:NUMPROC]]]

# -----------------------------------------------------------------------------
# *** Extra initial conditions saved during the run
# -----------------------------------------------------------------------------
if has_config save_ic
then
    source ./libsave_ic.sh
    declare -a save_ic_date save_ic_date1 save_ic_sec save_ic_day save_ic_ppt_file save_ic_nemo_ts
    oas_rst_files=${oas_rst_ifs_nemo}
fi

# =============================================================================
# *** END of User configuration
# =============================================================================

# =============================================================================
# *** This is where the code begins ...
# =============================================================================

# -----------------------------------------------------------------------------
# *** Create the run dir if necessary and go there
#     Everything is done from here.
# -----------------------------------------------------------------------------
if [ ! -d ${run_dir} ]
then
    mkdir -p ${run_dir}
fi
cd ${run_dir}

# -----------------------------------------------------------------------------
# *** Determine the time span of this run and whether it's a restart leg
# -----------------------------------------------------------------------------

# Regularise the format of the start and end date of the simulation
run_start_date=$(date -uR -d "${run_start_date}")
run_end_date=$(date -uR -d "${run_end_date}")


# -----------------------------------------------------------------------------
# *** Set path to grib_set
# -----------------------------------------------------------------------------

grib_set=${GRIB_BIN_PATH}${GRIB_BIN_PATH:+/}grib_set

# Loop over the number of legs
for (( ; run_num_legs>0 ; run_num_legs-- ))
do

    # Check for restart information file and set the current leg start date
    #   Ignore restart information file if force_run_from_scratch is true
    if ${force_run_from_scratch} || ! [ -r ${ece_info_file} ]
    then
        leg_is_restart=false
        leg_start_date=${run_start_date}
        leg_number=1
    else
        leg_is_restart=true
        . ./${ece_info_file}
        leg_start_date=${leg_end_date}
        leg_number=$((leg_number+1))
    fi

    # Compute the end date of the current leg
    if [ -n "${rst_freq}" ]
    then
        leg_end_date=$(date -uR -d "${leg_start_date} + ${rst_freq}")
    else
        leg_end_date=${run_end_date}
    fi

    if [[ "%Chunk_LAST%" == "TRUE" ]]
    then
        leg_end_date=${run_end_date}
        ifs_lastout=true
    fi

    # Some time variables needed later
    leg_length_sec=$(( $(date -u -d "${leg_end_date}" +%s) - $(date -u -d "${leg_start_date}" +%s) ))
    leg_start_sec=$(( $(date -u -d "${leg_start_date}" +%s) - $(date -u -d "${run_start_date}" +%s) ))
    leg_end_sec=$(( $(date -u -d "${leg_end_date}" +%s) - $(date -u -d "${run_start_date}" +%s) ))
    leg_start_date_yyyymmdd=$(date -u -d "${leg_start_date}" +%Y%m%d)
    leg_start_date_yyyy=$(date -u -d "${leg_start_date}" +%Y)
    leg_end_date_yyyy=$(date -u -d "${leg_end_date}" +%Y)

    # Check whether there's actually time left to simulate - exit otherwise
    if [ ${leg_length_sec} -le 0 ]
    then
        info "Leg start date equal to or after end of simulation."
        info "Nothing left to do. Exiting."
        exit 0
    fi

    # Initial conditions saved during the run
    do_save_ic=false
    save_ic_custom=false
    has_config save_ic && save_ic_get_config
    # if you do not use an option with save_ic, you must define 'do_save_ic' and
    # 'save_ic_date_offset' here or in ../libsave_ic.sh/save_ic_get_config()
    # with AS runtime, no need to edit the script, set SAVE_IC_OFFSET (and optionally SAVE_IC_COND)
    if $save_ic_custom
    then
        [[ "%SAVE_IC_COND%" = "" ]] || [[ "%SAVE_IC_COND%" = "%%SAVE_IC_COND%%" ]] \
            && save_ic_cond=true || save_ic_cond='%SAVE_IC_COND%'
        if eval $save_ic_cond ; then do_save_ic=true ; else do_save_ic=false ; fi
        save_ic_date_offset=( %SAVE_IC_OFFSET% )
    fi
    ${do_save_ic} && save_ic_define_vars

    # -------------------------------------------------------------------------
    # *** Prepare the run directory for a run from scratch
    # -------------------------------------------------------------------------
    if ! $leg_is_restart
    then
        # ---------------------------------------------------------------------
        # *** Check if run dir is empty. If not, and if we are allowed to do so
        #     by ${force_run_from_scratch}, remove everything
        # ---------------------------------------------------------------------
        if $(ls * >& /dev/null)
        then
            if ${force_run_from_scratch}
            then
                rm -fr ${run_dir}/*
            else
                error "Run directory not empty and \$force_run_from_scratch not set."
            fi
        fi

        # ---------------------------------------------------------------------
        # *** Copy executables of model components
        # *** Additionally, create symlinks to the original place for reference
        # ---------------------------------------------------------------------
        cp    ${ifs_exe_file} .
        ln -s ${ifs_exe_file} $(basename ${ifs_exe_file}).lnk

        if $(has_config amip)
        then
            cp    ${amip_exe_file} .
            ln -s ${amip_exe_file} $(basename ${amip_exe_file}).lnk
        fi

        if $(has_config nemo)
        then
            cp    ${nem_exe_file} .
            ln -s ${nem_exe_file} $(basename ${nem_exe_file}).lnk

            cp    ${rnf_exe_file} .
            ln -s ${rnf_exe_file} $(basename ${rnf_exe_file}).lnk

            cp    ${xio_exe_file} .
            ln -s ${xio_exe_file} $(basename ${xio_exe_file}).lnk
        fi

        if $(has_config lpjg)
        then
            cp    ${lpjg_exe_file} .
            ln -s ${lpjg_exe_file} $(basename ${lpjg_exe_file}).lnk
        fi

        if $(has_config tm5)
        then
            cp    ${tm5_exe_file} .
            ln -s ${tm5_exe_file} $(basename ${tm5_exe_file}).lnk
        fi

        # ---------------------------------------------------------------------
        # *** Files needed for IFS (linked)
        # ---------------------------------------------------------------------

        # Initial data
        ln -s \
        ${ini_data_dir}/ifs/${ifs_grid}/${leg_start_date_yyyymmdd}/ICMGGECE3INIUA \
                                                            ICMGG${exp_name}INIUA
        ln -s \
        ${ini_data_dir}/ifs/${ifs_grid}/${leg_start_date_yyyymmdd}/ICMSHECE3INIT \
                                                            ICMSH${exp_name}INIT
        rm -f ICMGG${exp_name}INIT
        cp ${ini_data_dir}/ifs/${ifs_grid}/${leg_start_date_yyyymmdd}/ICMGGECE3INIT \
                                                            ICMGG${exp_name}INIT

        # add bare_soil_albedo to ICMGG*INIT
        tempfile=tmp.$$
        ${grib_set} -s dataDate=$(date -u -d "$run_start_date" +%Y%m%d) \
            ${ini_data_dir}/ifs/${ifs_grid}/climate/bare_soil_albedos.grb \
            ${tempfile}

        cat ${tempfile} >> ICMGG${exp_name}INIT
        rm -f ${tempfile}

        # add land ice mask if needed
        if ${ifs_landice}
        then
            tempfile=tmp.$$
            cdo divc,10 -setcode,82 -selcode,141 ICMGG${exp_name}INIT ${tempfile}
            ${grib_set} -s gridType=reduced_gg ${tempfile} ${tempfile}
            cat ${tempfile} >> ICMGG${exp_name}INIT
            rm -f ${tempfile}
        fi

        # Other stuff
        ln -s ${ini_data_dir}/ifs/rtables/* .

        # Output control (ppt files)
        if [ ! -f ${output_control_files_dir}/pptdddddd0600 ] &&  [ ! -f ${output_control_files_dir}/pptdddddd0300 ];then
           echo "Error from ece-esm.sh: Neither the file pptdddddd0600 or pptdddddd0300 exists in the directory:"
           echo " " ${output_control_files_dir}
           exit -1
        fi
        mkdir postins
        cp ${output_control_files_dir}/ppt* postins/
        if [ -f postins/pptdddddd0600 ];then
           ln -s pptdddddd0600 postins/pptdddddd0000
           ln -s pptdddddd0600 postins/pptdddddd1200
           ln -s pptdddddd0600 postins/pptdddddd1800
        fi
        if [ -f postins/pptdddddd0300 ];then
           ln -s pptdddddd0300 postins/pptdddddd0900
           ln -s pptdddddd0300 postins/pptdddddd1500
           ln -s pptdddddd0300 postins/pptdddddd2100
           if [ ! -f postins/pptdddddd0600 ];then
               ln -s pptdddddd0300 postins/pptdddddd0000
               ln -s pptdddddd0300 postins/pptdddddd0600
               ln -s pptdddddd0300 postins/pptdddddd1200
               ln -s pptdddddd0300 postins/pptdddddd1800
           fi
        fi
        /bin/ls -1 postins/* > dirlist

        # ---------------------------------------------------------------------
        # *** Files needed for LPJ-GUESS
        # ---------------------------------------------------------------------
        if $(has_config lpjg)
        then
            # Check for valid grid
            if [ $lpjg_res != "T255" -a $lpjg_res != "T159" ]
            then
                error "LPJG-gridlist doesn't exist for ifs-grid: ${ifs_grid}" 
            fi
            # Initial data - saved state for LPJ-GUESS (.bin format)
            lpjgstartdir=$(printf "lpjg_state_%04d" $leg_start_date_yyyy)
            ln -sf ${ini_data_dir}/lpjg/ini_state/${lpjg_res}/${lpjgstartdir} ${run_dir}/${lpjgstartdir}

            # Control files (i.e. .ins, landuse, N deposition, soil type files etc.)
            cp  -f ${ecearth_src_dir}/lpjg/data/ins/*.ins .
            mkdir -p ${run_dir}/landuse

        fi

        # ---------------------------------------------------------------------
        # *** Files needed for NEMO (linked)
        # ---------------------------------------------------------------------
        if $(has_config nemo)
        then
            # Link initialisation files for matching ORCA grid
            for f in \
                bathy_meter.nc coordinates.nc \
                ahmcoef.nc \
                K1rowdrg.nc M2rowdrg.nc mask_itf.nc \
                decay_scale_bot.nc decay_scale_cri.nc \
                mixing_power_bot.nc mixing_power_cri.nc mixing_power_pyc.nc \
                runoff_depth.nc subbasins.nc
            do
                [ -f ${ini_data_dir}/nemo/initial/${nem_grid}/$f ] && ln -s ${ini_data_dir}/nemo/initial/${nem_grid}/$f
            done

            # Link geothermal heating file (independent of grid) and matching weight file
            ln -s ${ini_data_dir}/nemo/initial/Goutorbe_ghflux.nc
            ln -s ${ini_data_dir}/nemo/initial/weights_Goutorbe1_2_orca${nem_res_hor}_bilinear.nc

            # Link the salinity climatology file (needed for diagnostics)
            ln -s ${ini_data_dir}/nemo/climatology/${nem_grid}/sali_ref_clim_monthly.nc

            # Link either restart files or climatology files for the initial state
            if $(has_config nemo:start_from_restart)
            then
                # When linking restart files, we accept three options:
                # (1) Merged files for ocean and ice, i.e.
                #     restart_oce.nc and restart_ice.nc
                # (2) One-file-per-MPI-rank, i.e.
                #     restart_oce_????.nc and restart_ice_????.nc
                #     No check is done whether the number of restart files agrees
                #     with the number of MPI ranks for NEMO!
                # (3) One-file-per-MPI-rank with a prefix, i.e.
                #     <exp_name>_<time_step>_restart_oce_????.nc (similar for the ice)
                #     The prefix is ignored.
                # The code assumes that one of the options can be applied! If more
                # options are applicable, the first is chosen. If none of the
                # options apply, NEMO will crash with missing restart file.
                if   ls -U ${nem_restart_file_path}/restart_[oi]ce.nc > /dev/null 2>&1
                then
                    ln -s ${nem_restart_file_path}/restart_[oi]ce.nc ./

                elif ls -U ${nem_restart_file_path}/restart_[oi]ce_????.nc > /dev/null 2>&1
                then
                    ln -s ${nem_restart_file_path}/restart_[oi]ce_????.nc ./

                else
                    for f in ${nem_restart_file_path}/????_????????_restart_[oi]ce_????.nc
                    do
                        ln -s $f $(echo $f | sed 's/.*_\(restart_[oi]ce_....\.nc\)/\1/')
                    done
                fi
            else

                # Temperature and salinity files for initialisation
                ln -s ${ini_data_dir}/nemo/climatology/absolute_salinity_WOA13_decav_Reg1L75_clim.nc
                ln -s ${ini_data_dir}/nemo/climatology/conservative_temperature_WOA13_decav_Reg1L75_clim.nc
                ln -s ${ini_data_dir}/nemo/climatology/weights_WOA13d1_2_orca${nem_res_hor}_bilinear.nc

                # Grid dependent runoff files
                case ${nem_grid} in
                    ORCA1*)   ln -s ${ini_data_dir}/nemo/climatology/runoff-icb_DaiTrenberth_Depoorter_ORCA1_JD.nc ;;
                    ORCA025*) ln -s ${ini_data_dir}/nemo/climatology/ORCA_R025_runoff_v1.1.nc ;;
                esac
            fi

            # for ocean_nudging
            if $(has_config nemo:ocenudg) ; then
                ln -s ${ini_data_dir}/nemo/oce_nudg/resto.nc
            fi

            # XIOS files
            . ${ctrl_file_dir}/iodef.xml.sh > iodef.xml
            ln -s ${ctrl_file_dir}/context_nemo.xml
            ln -s ${ctrl_file_dir}/domain_def_nemo.xml
            ln -s ${ctrl_file_dir}/axis_def_nemo.xml
            ln -s ${ctrl_file_dir}/grids_def_nemo.xml
            ln -s ${ctrl_file_dir}/field_def_nemo-lim.xml
            ln -s ${ctrl_file_dir}/field_def_nemo-opa.xml
            ln -s ${ctrl_file_dir}/field_def_nemo-pisces.xml
            ln -s ${ctrl_file_dir}/field_def_nemo-inerttrc.xml
            ln -s ${output_control_files_dir}/file_def_nemo-lim3.xml file_def_nemo-lim.xml
            ln -s ${output_control_files_dir}/file_def_nemo-opa.xml
            ln -s ${output_control_files_dir}/file_def_nemo-pisces.xml

            if [ -f ${ini_data_dir}/xios/ORCA${nem_res_hor}/coordinates_xios.nc ]
            then
                cp ${ini_data_dir}/xios/ORCA${nem_res_hor}/coordinates_xios.nc ./
            else
                info "File 'coordinates_xios.nc' not found. NEMO can not be run with land domain removal!"
            fi
        fi

        # ---------------------------------------------------------------------
        # *** Files needed for the Runoff mapper (linked)
        # ---------------------------------------------------------------------

        has_config rnfmapper && \
            ln -s ${ini_data_dir}/runoff-mapper/runoff_maps.nc

        # -------------------------------------------------------------------------
        # *** File and dir needed for TM5
        # -------------------------------------------------------------------------
        if $(has_config tm5)
        then
            tm5_istart=[[[MOD:TM5:ISTART]]]

            case ${tm5_istart} in
                33|32) ln -s \
                    ${ini_data_dir}/tm5/restart/TM5_restart_${leg_start_date_yyyymmdd}_0000_glb300x200.nc
                    ;;
                31) ln -s ${ini_data_dir}/tm5/restart/tm5_save.hdf
                    ;;
                5)  ln -s ${ini_data_dir}/tm5/restart/tm5_mmix.hdf
                    ;;
                2|9)
                    ;;
                *)  error "Unsupported initial fields option (TM5): ${tm5_istart}"
                    ;;
            esac

            # Profiling dir for TM5
            mkdir -p ${run_dir}/tm5_profile
        fi

        # ---------------------------------------------------------------------
        # *** Files needed for OASIS (linked)
        # ---------------------------------------------------------------------

        # Name table file
        ln -s ${ini_data_dir}/oasis/cf_name_table.txt

        # -- Get grid definition and weight files for IFS/NEMO or IFS/AMIP coupling
        has_config nemo && \
            oas_grid_dir=${ini_data_dir}/oasis/T${ifs_res_hor}-ORCA${nem_res_hor} && \
            mycp='cp'

        has_config amip && \
            oas_grid_dir=${ini_data_dir}/oasis/AMIP && \
            mycp='cdo -f nc copy' # to enforce nc format, needed for 'cdo merge' to work (have nc4c with Primavera files)

        # Grid definition files
        if $(has_config tm5)
        then
            ${mycp} ${oas_grid_dir}/areas.nc gcm_areas.nc
            ${mycp} ${oas_grid_dir}/grids.nc gcm_grids.nc
            ${mycp} ${oas_grid_dir}/masks.nc gcm_masks.nc
        else
            ln -s ${oas_grid_dir}/areas.nc
            ln -s ${oas_grid_dir}/grids.nc
            ln -s ${oas_grid_dir}/masks.nc
        fi

        # Weight files
        case ${ifs_res_hor} in
            159)  oas_agrd=080
                  ;;
            255)  oas_agrd=128
                  ;;
            511)  oas_agrd=256
                  ;;
            799)  oas_agrd=400
                  ;;
            *)    error "Unsupported horizontal resolution (IFS): ${ifs_res_hor}"
                  ;;
        esac

        if $(has_config nemo)
        then
            case ${nem_res_hor} in
                1)  oas_ogrd=O1t0
                    ;;
                025)  oas_ogrd=Ot25
                    ;;
                *)  error "Unsupported horizontal resolution (NEMO): ${nem_res_hor}"
                    ;;
            esac
        fi

        if ${oas_link_weights}
        then
            for f in ${oas_grid_dir}/rmp_????_to_????_GAUSWGT.nc
            do
                ln -s $f
            done
        fi

        if $(has_config ifs nemo)
        then
            for f in ${oas_rst_ifs_nemo}
            do
                cp ${oas_grid_dir}/rst/$f .
            done
        fi

        # -- Get grid definition, weight and restart files for IFS/TM5 coupling
        if $(has_config tm5)
        then
            oas_grid_dir=${ini_data_dir}/oasis/T${ifs_res_hor}-TM3x2

            cp ${oas_grid_dir}/tm5_areas.nc .
            cp ${oas_grid_dir}/tm5_grids.nc .
            cp ${oas_grid_dir}/tm5_masks.nc .

            if ${oas_link_weights}
            then
                for f in ${oas_grid_dir}/rmp_????_to_????_*.nc
                do
                    ln -s $f
                done
            fi

            for f in ${oas_rst_ifs_tm5}
            do
                cp ${oas_grid_dir}/rst/$f .
            done

            # -- Merge grid definition files
            cdo merge gcm_areas.nc tm5_areas.nc areas.nc
            cdo merge gcm_grids.nc tm5_grids.nc grids.nc
            cdo merge gcm_masks.nc tm5_masks.nc masks.nc
        fi

    else # i.e. $leg_is_restart == true

        # ---------------------------------------------------------------------
        # *** Remove all leftover output files from previous legs
        # ---------------------------------------------------------------------

        # IFS files
        rm -f ICM{SH,GG}${exp_name}+??????

        # NEMO files
        rm -f ${exp_name}_??_????????_????????_{grid_U,grid_V,grid_W,grid_T,icemod,SBC,scalar,SBC_scalar,diad_T}.nc

        # TM5 start file type (33 if you have netcdf4 compiled w/ parallel IO, else to 31)
        tm5_istart=33

        if [ $tm5_istart -eq 31 ] && $(has_config tm5)
        then
            ln -sf save_${leg_start_date_yyyymmdd}00_glb300x200.hdf tm5_save.hdf
        fi

    fi # ! $leg_is_restart

    #--------------------------------------------------------------------------
    # *** Surface restoring and ocean nudging options
    #--------------------------------------------------------------------------
    #for ocean_nudging
    if $(has_config nemo:ocenudg) ; then
        ln -fs ${ini_data_dir}/nemo/oce_nudg/temp_sal_*.nc ./
    fi

    #for surface restoring
    if $(has_config nemo:surfresto) ; then
        ln -fs ${ini_data_dir}/nemo/surface_restoring/sss_restore_data*.nc  ./
        ln -fs ${ini_data_dir}/nemo/surface_restoring/sst_restore_data*.nc  ./
        ln -fs ${ini_data_dir}/nemo/surface_restoring/mask_restore*.nc ./
    fi

    # -------------------------------------------------------------------------
    # *** Remove land grid-points
    # -------------------------------------------------------------------------
    if $(has_config nemo:elpin)
    then
        if [ ! -f coordinates_xios.nc ]
        then
            error "ELpIN requested, but file 'coordinates_xios.nc' was not found"
        fi
        jpns=($(${ecearth_src_dir}/util/ELPiN/ELPiNv2.cmd ${nem_numproc}))
        info "nemo domain decompostion from ELpIN: ${jpns[@]}"
        nem_numproc=${jpns[0]}
        nem_jpni=${jpns[1]}
        nem_jpnj=${jpns[2]}
    elif has_config nemo
    then
        info "nemo original domain decomposition (not using ELPiN)"
    fi

    # -------------------------------------------------------------------------
    # *** Initial conditions saved during the run
    # -------------------------------------------------------------------------
    ${do_save_ic} && save_ic_prepare_output

    # -------------------------------------------------------------------------
    # *** Create some control files
    # -------------------------------------------------------------------------

    # Create TM5 runtime rcfile
    tm5_start_date=$(date -u -d "${leg_start_date}" +%F\ %T)
    tm5_end_date=$(date -u -d "${leg_end_date}" +%F\ %T)

    has_config tm5 && ${ecearth_src_dir}/tm5mp/setup_tm5 --no-compile \
        --time-start="${tm5_start_date}" --time-final="${tm5_end_date}" \
        --istart=${tm5_istart} ${ctrl_file_dir}/tm5-config-run.rc

    # IFS frequency output for namelist
    if [ -f postins/pptdddddd0300 ]
    then
        ifs_output_freq=$(( 3 * 3600 / ifs_time_step_sec ))
    elif [ -f postins/pptdddddd0600 ]
    then
        ifs_output_freq=$(( 6 * 3600 / ifs_time_step_sec ))
    else
        error "IFS output frequency undefined."
    fi

    # IFS, NEMO, LIM, AMIP namelist and OASIS namcouple files
    has_config ifs       && . ${ctrl_file_dir}/namelist.ifs.sh                          > fort.4
    has_config nemo      && . ${ctrl_file_dir}/namelist.nemo.ref.sh                     > namelist_ref
    has_config nemo      && . ${ctrl_file_dir}/namelist.nemo-${nem_grid}-coupled.cfg.sh > namelist_cfg
    has_config lim3      && . ${ctrl_file_dir}/namelist.lim3.ref.sh                     > namelist_ice_ref
    has_config lim3      && . ${ctrl_file_dir}/namelist.lim3-${nem_grid}.cfg.sh         > namelist_ice_cfg
    has_config rnfmapper && . ${ctrl_file_dir}/namelist.runoffmapper.sh                 > namelist.runoffmapper
    has_config amip      && . ${ctrl_file_dir}/namelist.amip.sh                         > namelist.amip

    . ${ctrl_file_dir}/namcouple.sh > namcouple

    # -------------------------------------------------------------------------
    # *** LPJ-GUESS initial data
    # -------------------------------------------------------------------------
    if $(has_config lpjg)
    then
        # LPJG runtime rcfile - update with leg dates
        . ${ctrl_file_dir}/namelist.lpjg.sh > lpjg_steps.rc

        # determine lpjg scenario-name and co2-file middle-fix
        case $(echo ${ifs_cmip6_scenario} | tr '[:upper:]' '[:lower:]') in
            hist*)     lpjg_scen="ssp370"; lu_src="AIM"    ; lpjg_sco2_mfix="AIM-ssp370-1-2-1"            ; lu_file_posfix="2018_10_08.txt";;
            ssp1-1.9*) lpjg_scen="ssp119"; lu_src="IMAGE"  ; lpjg_sco2_mfix="IMAGE-ssp119-1-2-1"          ; lu_file_posfix="2019_03_13.txt";;
            ssp1-2.6*) lpjg_scen="ssp126"; lu_src="IMAGE"  ; lpjg_sco2_mfix="IMAGE-ssp126-1-2-1"          ; lu_file_posfix="2018_10_08.txt";;
            ssp2-4.5*) lpjg_scen="ssp245"; lu_src="MESSAGE"; lpjg_sco2_mfix="MESSAGE-GLOBIOM-ssp245-1-2-1"; lu_file_posfix="2018_10_08.txt";;
            ssp3-7.0*) lpjg_scen="ssp370"; lu_src="AIM"    ; lpjg_sco2_mfix="AIM-ssp370-1-2-1"            ; lu_file_posfix="2018_10_08.txt";;
            ssp5-8.5*) lpjg_scen="ssp585"; lu_src="MAGPIE" ; lpjg_sco2_mfix="REMIND-MAGPIE-ssp585-1-2-1"  ; lu_file_posfix="2018_10_08.txt";;
            *)  error "Scenario ${ifs_cmip6_scenario} not defined for LPJ-GUESS" ;;
        esac

        lpjg_scenario_new="historical + $lpjg_sco2_mfix"
        lpjg_scenario_info=${run_dir}/lpjg_scenario.info

        if [ -f $lpjg_scenario_info ]
        then
            source $lpjg_scenario_info
            if [ "$lpjg_scenario_new" = "$lpjg_scenario" ]
            then
                lpjg_copy_rte=false
            else
                lpjg_copy_rte=true
            fi
        else
            lpjg_copy_rte=true
        fi

        # copy RTE only if necessary (at beginning of a run or when scenario changes)
        if $lpjg_copy_rte
        then
            # write info about installed scenarios to file
            echo "lpjg_scenario=\"historical + $lpjg_sco2_mfix\"" > $lpjg_scenario_info
            # set file prefixes depending on scenario
            lu_file_prefix="1850_2100_luh2_Hist_ScenarioMIP_UofMD"
            lu_file_midfix="2_1_f"

            lu_path="${ini_data_dir}/lpjg/landuse/${lpjg_res}/${lpjg_scen}"

            # copy and reference-link landuse,  gross transitions, crops, n-fertilisation
            for inp in lu gross crop nfert
            do
                if [ $inp == "crop" -o $inp == "nfert" ]
                then
                    lu_src_file="${lu_path}/${inp}_rfirr_${lu_file_prefix}_${lu_src}_${lpjg_scen}_${lu_file_midfix}_${lpjg_res}_${lu_file_posfix}"
                else
                    lu_src_file="${lu_path}/${inp}_${lu_file_prefix}_${lu_src}_${lpjg_scen}_${lu_file_midfix}_${lpjg_res}_${lu_file_posfix}"
                fi
                cp -f $lu_src_file ${run_dir}/landuse/${inp}_luh2.txt
                ln -fs $lu_src_file ${run_dir}/landuse/${inp}_luh2.txt.lnk
            done
            
            # nitrogen deposition files
            mkdir -p ${run_dir}/ndep
            for inp in drynhx2 drynoy2 wetnhx2 wetnoy2
            do
                ndep_src_file="${ini_data_dir}/lpjg/ndep/${lpjg_res}/${lpjg_scen}/${lpjg_scen}_${lpjg_res}_${inp}.nc"
                cp -f  $ndep_src_file ${run_dir}/ndep/${inp}.nc
                ln -fs  $ndep_src_file ${run_dir}/ndep/${inp}.nc.lnk
            done

            # concatenate historic and scenario (2015+) co2 concentration file
            # hist co2 file 
            lpjg_hco2_file="${ini_data_dir}/ifs/cmip6-data/mole-fraction-of-carbon-dioxide-in-air_input4MIPs_GHGConcentrations_CMIP_UoM-CMIP-1-2-0_gr1-GMNHSH_0000-2014.nc"
            # scenario co2 file 
            lpjg_sco2_file="${ini_data_dir}/ifs/cmip6-data/mole-fraction-of-carbon-dioxide-in-air_input4MIPs_GHGConcentrations_ScenarioMIP_UoM-${lpjg_sco2_mfix}_gr1-GMNHSH_2015-2500.nc"
            # combined file
            lpjg_co2_file="${run_dir}/mole_fraction_of_carbon_dioxide_in_air_input4MIPs_lpjg.nc"
            rm -f $lpjg_co2_file
            cdo mergetime $lpjg_hco2_file $lpjg_sco2_file $lpjg_co2_file
        fi

        # Populate or update LPJG run directories
        for (( n=1; n<=${lpjg_numproc}; n++ ))
        do
            # if run from scratch or number of procs has been extended
            if ! $leg_is_restart || [ ! -d ${run_dir}/run${n} ]
            then
                # Make output directories
                mkdir -p ${run_dir}/run${n}/output

                # Copy *.ins, lpjg_steps.rc and OASIS-MCT restart files
                cp ${run_dir}/*.ins ${run_dir}/run${n}

                # Copy output control files
                cp ${output_control_files_dir}/lpjg_cmip6_output.ins ${run_dir}/run${n}
                ln -s ${output_control_files_dir}/lpjg_cmip6_output.ins ${run_dir}/run${n}/lpjg_cmip6_output.ins.lnk

                cp ${ini_data_dir}/lpjg/oasismct/ghg*.txt ${run_dir}/run${n}
                cp ${ini_data_dir}/lpjg/oasismct/${lpjg_res}/ece_gridlist_${lpjg_res}.txt ${run_dir}/run${n}/ece_gridlist.txt
                ln -s ${ini_data_dir}/lpjg/oasismct/${lpjg_res}/ece_gridlist_${lpjg_res}.txt ${run_dir}/run${n}/ece_gridlist.txt.lnk

                # Data only needed by master
                if [ $n == 1 ]
                then
                    cp ${ini_data_dir}/lpjg/oasismct/${lpjg_res}/*.nc ${run_dir}
                    cp ${ini_data_dir}/lpjg/oasismct/lpjgv.txt ${run_dir}/run${n}
                fi
            fi
            # Refresh output-dirs after they hav been removed at end of the last leg
            mkdir -p ${run_dir}/run${n}/output/CMIP6
            mkdir -p ${run_dir}/run${n}/output/CRESCENDO
        done

        if $leg_is_restart
        then
            lpjg_restart_dir="restart/lpjg/$(printf %03d $((leg_number-1)))"
            lpjg_rst_state="${lpjg_restart_dir}/lpjg_state_${leg_start_date_yyyy}"
            if [ -d "$lpjg_rst_state" ]
            then
                ln -sf $lpjg_rst_state
            else
                echo "lpjg restart dir $lpjg_rst_state not available"
                exit -1
            fi
            # get oasis restart files
            oasis_restart_dir="restart/oasis/$(printf %03d $((leg_number)))"
            cp -f ${oasis_restart_dir}/*.nc ${run_dir}/

        fi
    fi


    # -------------------------------------------------------------------------
    # *** Create ICMCL file with vegetation fields
    #     not needed if LPJG is used with feedback
    # -------------------------------------------------------------------------
    tempfile=tmp.$$

    case ${ifs_veg_source} in
    "ifs" )
        # Vegetation from IFS (climatology)

        icmclfile=${ini_data_dir}/ifs/${ifs_grid}/climate/ICMCL_ONLY_VEG_PD

        # Create data for december, the year before the leg starts
        ${grib_set} \
            -s dataDate=$(printf "%04d" $((leg_start_date_yyyy-1)))1215 \
            ${icmclfile}-12 ICMCL${exp_name}INIT

        # Create data for all month in the years of the leg
        for (( y=${leg_start_date_yyyy} ; y<=${leg_end_date_yyyy} ; y++ ))
        do
            yy=$(printf "%04d" $y)
            for m in {1..12}
            do
                mm=$(printf "%02d" $m)
                ${grib_set} -s dataDate=${yy}${mm}15 ${icmclfile}-${mm} ${tempfile}
                cat ${tempfile} >> ICMCL${exp_name}INIT
            done
        done

        # Create data for january, the year after the leg ends
        ${grib_set} \
            -s dataDate=$(printf "%04d" $((leg_end_date_yyyy+1)))0115 \
            ${icmclfile}-01 ${tempfile}
        cat ${tempfile} >> ICMCL${exp_name}INIT
        ;;
    "era20c"|"cmip6" )
        # Vegetation from a LPJG run (off-line or EC-Earth3-Veg)

        rm -f ICMCL${exp_name}INIT

        # Create data for all years of the leg, including one year
        # before and one year after
        for (( yr=leg_start_date_yyyy-1 ; yr<=leg_end_date_yyyy+1 ; yr+=1 ))
        do

            case ${ifs_veg_source} in
            'era20c' )
                # no scenario needed with era20c
                icmcl_scenario="" ;;
            'cmip6' )
                # select scenario, use SSP3-7.0 as default
                # if not otherwise specified
                icmcl_scenario="historical"
                if ( [ $ifs_cmip_fixyear -le 0 ] && [ $yr -ge 2015 ] ) || \
                     [ $ifs_cmip_fixyear -ge 2015 ]
                then
                    [[ ${ifs_cmip6_scenario} =~ ^SSP ]] \
                        && icmcl_scenario=${ifs_cmip6_scenario} \
                        || icmcl_scenario='SSP3-7.0'
                fi ;;
            esac
            
            if [ $ifs_cmip_fixyear -le 0 ]
            then
                cat ${ini_data_dir}/ifs/${ifs_grid}/icmcl_${veg_version}/${icmcl_scenario}/icmcl_$yr.grb >> ICMCL${exp_name}INIT
            else
                # Fixed year forcing, requires cdo!
                # If cdo is not available at runtime you need to fix proper
                # icmcl files beforehand and use them here
                cdo setyear,$yr ${ini_data_dir}/ifs/${ifs_grid}/icmcl_${veg_version}/${icmcl_scenario}/icmcl_${ifs_cmip_fixyear}.grb ${tempfile}
                cat ${tempfile} >> ICMCL${exp_name}INIT
            fi
        done
        ;;
    "none" )
        info "no ICMCL file is created"
        ;;
    * )
        error "Vegetation from ${ifs_veg_source} not implemented"
        ;;
    esac

    # Clean up
    rm -f ${tempfile}

    # -------------------------------------------------------------------------
    # *** Link the appropriate NEMO restart files of the previous leg
    # -------------------------------------------------------------------------
    if $leg_is_restart && $(has_config nemo)
    then
        ns=$(printf %08d $(( leg_start_sec / nem_time_step_sec - nem_restart_offset )))
        for (( n=0 ; n<nem_numproc ; n++ ))
        do
            np=$(printf %04d ${n})
            ln -fs ${exp_name}_${ns}_restart_oce_${np}.nc restart_oce_${np}.nc
            ln -fs ${exp_name}_${ns}_restart_ice_${np}.nc restart_ice_${np}.nc
        done

        # Make sure there are no global restart files
        # If links are found, they will be removed. We are cautious and do
        # _not_ remove real files! However, if real global restart files are
        # present, NEMO/LIM will stop because time stamps will not match.
        [ -h restart_oce.nc ] && rm restart_oce.nc
        [ -h restart_ice.nc ] && rm restart_ice.nc
    fi

    # -------------------------------------------------------------------------
    # *** Remove some OASIS files of the previous leg
    # -------------------------------------------------------------------------
    if $leg_is_restart
    then
        rm -f anaisout_*
    fi

    # -------------------------------------------------------------------------
    # *** Start the run
    # -------------------------------------------------------------------------
    export DR_HOOK_IGNORE_SIGNALS='-1'
    export CPLNG='active'

    # Use the launch function from the platform configuration file
    has_config nemo ifs && \
        cmd="${xio_numproc} ${xio_exe_file} -- \
             ${rnf_numproc} ${rnf_exe_file} -- \
             ${ifs_numproc} ${ifs_exe_file} -v ecmwf -e ${exp_name} -- \
             ${nem_numproc} ${nem_exe_file}" || \
        cmd="${ifs_numproc} ${ifs_exe_file} -v ecmwf -e ${exp_name}"

    has_config lpjg && cmd=${cmd}" -- ${lpjg_numproc} ${lpjg_exe_file} guess.ins -parallel"
    has_config tm5  && cmd=${cmd}" -- ${tm5_numproc}  ${tm5_exe_file} tm5-run.rc"
    has_config amip && cmd=${cmd}" -- ${amip_numproc} ${amip_exe_file}"

    t1=$(date +%s)
    launch $cmd
    t2=$(date +%s)

    tr=$(date -d "0 -$t1 sec + $t2 sec" +%T)

    # -------------------------------------------------------------------------
    # *** Check for signs of success
    #     Note the tests provide no guarantee that things went fine! They are
    #     just based on the IFS, NEMO and TM5 log files. More tests (e.g. checking
    #     restart files) could be implemented.
    # -------------------------------------------------------------------------

    # Checking for IFS success
    if [ -f ifs.stat ]
    then
        if [ "$(awk 'END{print $3}' ifs.stat)" == "CNT0" ]
        then
            info "Leg successfully completed according to IFS log file 'ifs.stat'."
        else
            error "Leg not completed according to IFS log file 'ifs.stat'."
        fi
    else
        error "IFS log file 'ifs.stat' not found after run."
    fi

    # Check for NEMO success
    if $(has_config nemo)
    then
        if [ -f ocean.output ]
        then
            if [ "$(sed -n '/New day/h; ${g;s:.*\([0-9/]\{10\}\).*:\1:;p;}' ocean.output)" == "$(date -u -d "${leg_end_date} - 1 day" +%Y/%m/%d)" ]
            then
                info "Leg successfully completed according to NEMO log file 'ocean.output'."
            else
                error "Leg not completed according to NEMO log file 'ocean.output'."
            fi
        else
            error "NEMO log file 'ocean.output' not found after run."
        fi
    fi

    # Check for TM5 success
    if $(has_config tm5)
    then
        if [ -f tm5.ok ]
        then
            info "Leg successfully completed according to existing TM5 file 'tm5.ok'."
        else
            error "Leg not completed according to non-existing TM5 file 'tm5.ok'."
        fi
    fi

    # -------------------------------------------------------------------------
    # *** Post-process initial conditions saved during the run if requested
    # -------------------------------------------------------------------------
    ${do_save_ic} && save_ic_postproc

    # -------------------------------------------------------------------------
    # *** Move IFS output files to archive directory
    # -------------------------------------------------------------------------
    outdir="output/ifs/$(printf %03d $((leg_number)))"
    mkdir -p ${outdir}

    prv_leg=$(printf %03d $((leg_number-1)))

    # This takes care of a special IFS feature: The output for the last time
    # step of each leg is written at the first time step of the new leg. The
    # following code makes sure that the output is appended to the appropriate
    # file. Since GRIB files are just streams, its done with a simple cat
    # command.
    for f in ICMSH${exp_name}+?????? ICMGG${exp_name}+??????
    do
        if [ -f output/ifs/${prv_leg}/${f} ]
        then
            cat ${f} >> output/ifs/${prv_leg}/${f}
            rm -f ${f}
        else
            mv ${f} ${outdir}
        fi
    done

    # -------------------------------------------------------------------------
    # *** Move NEMO output files to archive directory
    # -------------------------------------------------------------------------
    if $(has_config nemo)
    then
        outdir="output/nemo/$(printf %03d $((leg_number)))"
        mkdir -p ${outdir}

        for v in grid_U grid_V grid_W grid_T icemod SBC scalar SBC_scalar diad_T \
                 grid_T_2D grid_U_2D grid_V_2D grid_W_2D grid_T_3D grid_U_3D grid_V_3D grid_W_3D \
                 grid_1point grid_T_3D_ncatice vert_sum \
                 grid_ptr_W_3basin_3D grid_ptr_T_3basin_2D grid_ptr_T_2D \
                 zoom_700_sum zoom_300_sum zoom_2000_sum
        do
            for f in ${exp_name}_*_????????_????????_*${v}.nc
            do
                test -f $f && mv $f $outdir/
            done
        done
    fi

    # -------------------------------------------------------------------------
    # *** Move LPJ-GUESS output files to archive directory
    # -------------------------------------------------------------------------
    if $(has_config lpjg)
    then
        outdir="output/lpjg/$(printf %03d $((leg_number)))"
    if [ -d ${outdir} ]
    then
        rm -rf  ${outdir}
    fi
        mkdir -p ${outdir}

        # LPJG run directories
        # concatenate *.out files from each run* into output dir
	# DMG: adding escaped % for autosubmit, undo when issue #42 is resolved       
	# flist=$(find ${run_dir}/run1/output -type f | sed "s%${run_dir}/run1/output\/%%")
	flist=$(find ${run_dir}/run1/output -type f | sed "s%${run_dir}/run1/output\/%%%%")
        mkdir ${outdir}/CMIP6
        for (( n=1; n<=${lpjg_numproc}; n++ ))
    do
        for ofile in $flist
            do
        if (( n == 1 ))
        then
            cat ${run_dir}/run${n}/output/${ofile} > ${outdir}/$ofile
        else
            awk '(FNR!=1){print $0}' ${run_dir}/run${n}/output/${ofile} >> ${outdir}/$ofile
        fi
        done
        rm -rf ${run_dir}/run${n}/output
        done
    # move monthly file if available
    if [ -f ${run_dir}/LPJ-GUESS_monthlyoutput.txt ]
    then
        mv ${run_dir}/LPJ-GUESS_monthlyoutput.txt ${outdir}
    fi
    fi

    # -------------------------------------------------------------------------
    # *** Move TM5 output files to archive directory
    # -------------------------------------------------------------------------
    if $(has_config tm5)
    then
        outdir="output/tm5/$(printf %03d $((leg_number)))"
        mkdir -p ${outdir}

        set +e
        mv budget_??????????_??????????_global.hdf      ${outdir}
        mv j_statistics_??????????_??????????.hdf       ${outdir}
        mv mmix_??????????_??????????_glb???x???.hdf    ${outdir}
        mv aerocom?_TM5_*_????????_daily.nc             ${outdir}
        mv aerocom?_TM5_*_??????_monthly.nc             ${outdir}
        mv AOD_????_??_??.nc                            ${outdir}
        mv -f TM5MP_${exp_name}_griddef.nc              ${outdir}
        mv TM5MP_${exp_name}_TP_????_??_??.nc           ${outdir}
        mv TM5MP_${exp_name}_vmr3_????_??_??.nc         ${outdir}
        mv general_TM5_${exp_name}_??????????_hourly.nc ${outdir}
        mv general_TM5_${exp_name}_??????_monthly.nc    ${outdir}
        set -e

        # move profiling files if any
        if [ "$(ls -A ${run_dir}/tm5_profile)" ]
        then
            outdir="output/tm5/profile_$(printf %03d $((leg_number)))"
            mkdir -p ${outdir}

            for f in ${run_dir}/tm5_profile/*
            do
                test -f ${f} && mv $f ${outdir}
            done
        fi
    fi

    # -------------------------------------------------------------------------
    # *** Move IFS restart files to archive directory
    # -------------------------------------------------------------------------
    if $leg_is_restart
    then
        outdir="restart/ifs/$(printf %03d $((leg_number)))"
        mkdir -p ${outdir}

        # Figure out the time part of the restart files (cf. CTIME on rcf files)
        # NOTE: Assuming that restarts are at full days (time=0000) only!
        nd="$(printf %06d $((leg_start_sec/(24*3600))))0000"

        mv srf${nd}.???? ${outdir}

    fi

    # -------------------------------------------------------------------------
    # *** Move LPJ-GUESS restart files to archive directory
    # -------------------------------------------------------------------------
    if $(has_config lpjg)
    then
        outdir="restart/lpjg/$(printf %03d $((leg_number)))"
        if [ -d ${outdir} ]
        then
            rm -rf  ${outdir}
        fi
        mkdir -p ${outdir}

        state_dir="./lpjg_state_$(printf %04d $((leg_end_date_yyyy)))"
        mv ${state_dir} ${outdir}
        # LPJG writes into run1 dir, so mv to main rundir
        mv -f run1/lpjgv.nc .

        # remove restart link
        if $leg_is_restart
        then
            old_state_dir="./lpjg_state_$(printf %04d $((leg_start_date_yyyy)))"
            if [ -L $old_state_dir ]
            then
                rm -f "$old_state_dir"
            fi
        fi
    fi

    # -------------------------------------------------------------------------
    # *** Move NEMO restart files to archive directory
    # -------------------------------------------------------------------------
    if $leg_is_restart && $(has_config nemo)
    then
        outdir="restart/nemo/$(printf %03d $((leg_number)))"
        mkdir -p ${outdir}

        ns=$(printf %08d $(( leg_start_sec / nem_time_step_sec - nem_restart_offset )))
        for f in oce ice
        do
            mv ${exp_name}_${ns}_restart_${f}_????.nc ${outdir}
        done
    fi

    # -------------------------------------------------------------------------
    # *** Move TM5 restart file to archive directory
    # -------------------------------------------------------------------------
    if $leg_is_restart && $(has_config tm5)
    then
        outdir="restart/tm5"
        mkdir -p ${outdir}

        case ${tm5_istart} in
            33) f=TM5_restart_${leg_start_date_yyyymmdd}_0000_glb300x200.nc
                ;;
            31) f=save_${leg_start_date_yyyymmdd}00_glb300x200.hdf
                ;;
        esac

        mv $f ${outdir}
    fi

    # -------------------------------------------------------------------------
    # *** Copy OASIS restart files to archive directory
    #     NOTE: These files are copied and not moved as they are used in the
    #           next leg!
    #           Note also that the OASIS restart files present at the end of
    #           the leg correspond to the start of the next leg!
    # -------------------------------------------------------------------------
    outdir="restart/oasis/$(printf %03d $((leg_number+1)))"
    mkdir -p ${outdir}

    for f in ${oas_rst_ifs_nemo} ${oas_rst_ifs_tm5} vegin.nc lpjgv.nc
    do
        test -f ${f} && cp ${f} ${outdir}
    done

    # -------------------------------------------------------------------------
    # *** Copy rcf files to the archive directory (of the next leg!)
    # -------------------------------------------------------------------------
    outdir="restart/ifs/$(printf %03d $((leg_number+1)))"
    mkdir -p ${outdir}

    for f in rcf
    do
        test -f ${f} && cp ${f} ${outdir}
    done

    # -------------------------------------------------------------------------
    # *** Move log files to archive directory
    # -------------------------------------------------------------------------
    outdir="log/$(printf %03d $((leg_number)))"
    mkdir -p ${outdir}

    for f in \
        ifs.log ifs.stat fort.4 ocean.output \
        time.step solver.stat guess.log run1/guess0.log \
        amip.log namelist.amip \
        nout.000000 debug.??.?????? A*_??.nc O*_??.nc lucia.??.?????? \
        ctm.tm5.log.0
    do
        test -f ${f} && mv ${f} ${outdir}
    done

    for f in ctm.tm5.log.*
    do
        test -f ${f} && \rm -f ${f}
    done

    # -------------------------------------------------------------------------
    # *** Write the restart control file
    # -------------------------------------------------------------------------

    # Compute CPMIP performance
    sypd="$(cpmip_sypd $leg_length_sec $(($t2 - $t1)))"
    ncores=0
    has_config nemo      && (( ncores+=${nem_numproc}  )) || :
    has_config ifs       && (( ncores+=${ifs_numproc}  )) || :
    has_config xios      && (( ncores+=${xio_numproc}  )) || :
    has_config rnfmapper && (( ncores+=${rnf_numproc}  )) || :
    has_config lpjg      && (( ncores+=${lpjg_numproc} )) || :
    has_config tm5       && (( ncores+=${tm5_numproc}  )) || :
    has_config amip      && (( ncores+=${amip_numproc} )) || :
    chpsy="$(cpmip_chpsy  $leg_length_sec $(($t2 - $t1)) $ncores)"

    echo "#"                                             | tee -a ${ece_info_file}
    echo "# Finished leg at `date '+%F %T'` after ${tr} (hh:mm:ss)" \
                                                         | tee -a ${ece_info_file}
    echo "# CPMIP performance: $sypd SYPD   $chpsy CHPSY"| tee -a ${ece_info_file}
    echo "leg_number=${leg_number}"                      | tee -a ${ece_info_file}
    echo "leg_start_date=\"${leg_start_date}\""          | tee -a ${ece_info_file}
    echo "leg_end_date=\"${leg_end_date}\""              | tee -a ${ece_info_file}

    # Need to reset force_run_from_scratch in order to avoid destroying the next leg
    force_run_from_scratch=false

done # loop over legs

# -----------------------------------------------------------------------------
# *** Platform dependent finalising of the run
# -----------------------------------------------------------------------------
finalise