ecearth_patch/util/special_restart_example_run.sh

#!/usr/bin/env bash

#PBS -q large
#PBS -r y
#PBS -W group_list=ecearth
#PBS -l walltime=23:50:00
#PBS -l select=6:ncpus=24:mem=63000mb:mpiprocs=24:ompthreads=1
#PBS -M pierre-yves.barriat@uclouvain.be
#PBS -m abe

exec > ${PBS_O_WORKDIR-$PWD}/${PBS_JOBNAME-"local"}_${PBS_JOBID-"id"}.log
echo "------------------ Job Info --------------------"
echo "jobid : $PBS_JOBID"
echo "jobname : $PBS_JOBNAME"
echo "job type : $PBS_ENVIRONMENT"
echo "submit dir : $PBS_O_WORKDIR"
echo "queue : $PBS_O_QUEUE"
echo "user : $PBS_O_LOGNAME"
echo "threads : $OMP_NUM_THREADS"

set -ue

# Directories
start_dir=${PBS_O_WORKDIR-$PWD}

# librunscript defines some helper functions
. ${start_dir}/librunscript.sh

# =============================================================================
# *** BEGIN User configuration
# =============================================================================

# -----------------------------------------------------------------------------
# *** General configuration
# -----------------------------------------------------------------------------

# Component configuration
# (for syntax of the $config variable, see librunscript.sh)
config="ifs nemo lim3 rnfmapper xios:detached oasis"

# Experiment name (exactly 4 letters!)
exp_name=ECN0

# Simulation start and end date. Use any (reasonable) syntax you want.
run_start_date="1990-01-01"
run_end_date="${run_start_date} + 10 years"

# Set $force_run_from_scratch to 'true' if you want to force this run to start
# from scratch, possibly ignoring any restart files present in the run
# directory. Leave set to 'false' otherwise.
# NOTE: If set to 'true' the run directory $run_dir is cleaned!
force_run_from_scratch=false
special_restart=true
special_restart_from=ECT0
special_restart_date="1995-01-01"

# Resolution
ifs_grid=T255L91
nem_grid=ORCA1L75

# Restart frequency. Use any (reasonable) number and time unit you want.
# For runs without restart, leave this variable empty
rst_freq="1 year"

# Number of restart legs to be run in one go
run_num_legs=4

# Coupling frequency between atmosphere and ocean (in seconds)
cpl_freq_atm_oce_sec=2700

# Directories
#start_dir=${PWD}
ctrl_file_dir=${start_dir}/ctrl

# Architecture
build_arch=ecconf

# This file is used to store information about restarts
ece_info_file="ece.info"

# -----------------------------------------------------------------------------
# *** Read platform dependent configuration
# -----------------------------------------------------------------------------
. ${start_dir}/ecconf.cfg

configure

# -----------------------------------------------------------------------------
# *** IFS configuration
# -----------------------------------------------------------------------------

ifs_version=36r4

ifs_time_step_sec=2700

ifs_output_freq=$(( 6 * 3600 / ifs_time_step_sec ))
ifs_di_freq=$(( 24 * 3600 / ifs_time_step_sec ))
ifs_ddh_freq=$(( 120 * 3600 / ifs_time_step_sec ))

ifs_res_hor=$(echo ${ifs_grid} | sed 's:T\([0-9]\+\)L\([0-9]\+\):\1:')
ifs_res_ver=$(echo ${ifs_grid} | sed 's:T\([0-9]\+\)L\([0-9]\+\):\2:')

ifs_numproc=78

ifs_exe_file=${ecearth_src_dir}/ifs-${ifs_version}/bin/ifsmaster-${build_arch}

ifs_lastout=false

ifs_cmip5=TRUE
ifs_cmip5_rcp=0
ifs_cmip_fixyear=2000
ifs_mac2sp=TRUE
ifs_cmip6piaer=TRUE

ifs_cmip6=TRUE

ifs_A4xCO2=FALSE
ifs_1PCTCO2=FALSE

# Time-varying orbital forcing (Qiong Zhang, SU-2013-09)
# https://dev.ec-earth.org/projects/ecearth3/wiki/Orbital_forcing_in_EC-Earth_3
#
#   ifs_orb_switch=FALSE, no orbital calculations applied
#   ifs_orb_switch=TRUE, use orbital calculations according to ifs_orb_mode
#   ifs_orb_mode="fixed_year", or "variable_year", or "fixed_parameters"
#     fixed_year: calculate the orbital parameters at ifs_orb_iyear, e.g.,1850
#     variable_year: calculate orbital parameters annually start from ifs_orb_iyear
#     fixed_parameters: prescribe orbital parameters for given year
ifs_orb_switch=FALSE
ifs_orb_mode="variable_year"
ifs_orb_iyear=$(date -u -d "${run_start_date}" +%Y)

# -----------------------------------------------------------------------------
# *** IFS tuning (the commented values are EC-Earth 3.2beta and IFS cy36r4)
# -----------------------------------------------------------------------------
                     # ECE32b     IFS cy36r4                     
RPRCON=1.51E-3       # 1.2E-3     1.4E-3
RVICE=0.125          # 0.13       0.15
RLCRITSNOW=4.2E-5    # 3.0E-5     5.0E-5
RSNOWLIN2=0.035      # 0.035      0.025
ENTRORG=1.66E-4       # 1.5E-4     1.8E-4
DETRPEN=0.69E-4       # 0.75E-4    0.75E-4
ENTRDD=3.6E-4        # 3.0E-4     2.0E-4
RMFDEPS=0.27          # 0.3        0.35
RCLDIFF=3.E-6        # 3.E-6
RCLDIFFC=5.0         # 5.0

# -----------------------------------------------------------------------------
# *** NEMO/LIM configuration
# -----------------------------------------------------------------------------

nem_time_step_sec=2700
lim_time_step_sec=2700

# This is only needed if the experiment is started from an existing set of NEMO
# restart files
nem_restart_file_path=${start_dir}/nemo-rst

nem_restart_offset=0

nem_res_hor=$(echo ${nem_grid} | sed 's:ORCA\([0-9]\+\)L[0-9]\+:\1:')

nem_exe_file=${ecearth_src_dir}/nemo-3.6/CONFIG/${nem_grid}_LIM3/BLD/bin/nemo.exe

nem_numproc=64

# -----------------------------------------------------------------------------
# *** Runoff mapper configuration
# -----------------------------------------------------------------------------

rnf_exe_file=${ecearth_src_dir}/runoff-mapper/bin/runoff-mapper.exe
rnf_numproc=1

# -----------------------------------------------------------------------------
# *** OASIS configuration
# -----------------------------------------------------------------------------

# Restart files for the coupling fields (note 8 character limit in OASIS)
#   rstas.nc : atmoshpere single-category fields
#   rstam.nc : atmoshpere multi-category fields
#   rstos.nc : ocean single-category fields
#   rstom.nc : ocean multi-category fields
oas_rst_files="rstas.nc rstos.nc"

# Decide whether the OASIS weight files for interpolation should be linked from
# the setup directory (true) or not (false). In the latter case, the weights
# are re-computed at the start of the run.
oas_link_weights=true

# Flux correction for runoff (not calving) sent from Oasis to ocean. 
# 1.07945 is computed to compensate for a P-E=-0.016 mm/day
oas_mb_fluxcorr=1.07945

# -----------------------------------------------------------------------------
# *** XIOS configuration
# -----------------------------------------------------------------------------

xio_exe_file=${ecearth_src_dir}/xios-2/bin/xios_server.exe

xio_numproc=1

# =============================================================================
# *** END of User configuration
# =============================================================================

# =============================================================================
# *** This is where the code begins ...
# =============================================================================

# -----------------------------------------------------------------------------
# *** Create the run dir if necessary and go there
#     Everything is done from here.
# -----------------------------------------------------------------------------
if [ ! -d ${run_dir} ]
then
    mkdir -p ${run_dir}
    if $special_restart
    then
        force_run_from_scratch=false
        rsync -av --delete ${run_dir}/../${special_restart_from}/ --exclude log --exclude output --exclude restart --exclude="${special_restart_from}_*" --exclude="srf*" --exclude="restart_*" --exclude="debug.*" --exclude="output.*" ${run_dir}
        cp -f ${nem_exe_file} ${run_dir}
        cp -f ${ifs_exe_file} ${run_dir}
        cp -f ${rnf_exe_file} ${run_dir}
        cp -f ${xio_exe_file} ${run_dir}
        special_year=${special_restart_date:0:4}
        sed -i "/$special_year/q" ${run_dir}/ece.info
        . ${run_dir}/ece.info
        special_restart_leg=$(printf %03d $((leg_number+1)))
        # PUT HERE THE INSTRUCTIONS TO COPY THE restart files
        rsync -av ${run_dir}/../${special_restart_from}/restart/ifs/${special_restart_leg}/ ${run_dir}
        rsync -av ${run_dir}/../${special_restart_from}/restart/oasis/${special_restart_leg}/ ${run_dir}
        cd ${run_dir}/../${special_restart_from}/restart/nemo/${special_restart_leg}
        for f in *.nc; do
            nf=${exp_name}${f:4}
            cp $f ${run_dir}/$nf
        done
        cd -
        cd ${run_dir}
        for f in ${exp_name}_????????_restart_???_????.nc; do
            nf=${f:14}
            ln -s $f $nf
        done
        cd -
        rm -f ${run_dir}/ICMCL${special_restart_from}INIT
        mv ${run_dir}/ICMGG${special_restart_from}INIUA ${run_dir}/ICMGG${exp_name}INIUA
        mv ${run_dir}/ICMGG${special_restart_from}INIT ${run_dir}/ICMGG${exp_name}INIT
        mv ${run_dir}/ICMSH${special_restart_from}INIT ${run_dir}/ICMSH${exp_name}INIT
    fi
else
    force_run_from_scratch=false
    special_restart=false
fi
cd ${run_dir}

# -----------------------------------------------------------------------------
# *** Determine the time span of this run and whether it's a restart leg
# -----------------------------------------------------------------------------

# Regularise the format of the start and end date of the simulation
run_start_date=$(date -uR -d "${run_start_date}")
run_end_date=$(date -uR -d "${run_end_date}")

# Loop over the number of legs
for (( ; run_num_legs>0 ; run_num_legs-- ))
do

    # Check for restart information file and set the current leg start date
    #   Ignore restart information file if force_run_from_scratch is true
    if ${force_run_from_scratch} || ! [ -r ${ece_info_file} ]
    then
        leg_is_restart=false
        leg_start_date=${run_start_date}
        leg_number=1
    else
        leg_is_restart=true
        . ./${ece_info_file}
        leg_start_date=${leg_end_date}
        leg_number=$((leg_number+1))
    fi

    # Compute the end date of the current leg
    if [ -n "${rst_freq}" ]
    then
        leg_end_date=$(date -uR -d "${leg_start_date} + ${rst_freq}")
    else
        leg_end_date=${run_end_date}
    fi

    if [ $(date -d "${leg_end_date}" +%s) -gt $(date -d "${run_end_date}" +%s) ]
    then
        leg_end_date=${run_end_date}
    fi

    # Some time variables needed later
    leg_length_sec=$(( $(date -d "${leg_end_date}" +%s) - $(date -d "${leg_start_date}" +%s) ))
    leg_start_sec=$(( $(date -d "${leg_start_date}" +%s) - $(date -d "${run_start_date}" +%s) ))
    leg_end_sec=$(( $(date -d "${leg_end_date}" +%s) - $(date -d "${run_start_date}" +%s) ))
    leg_start_date_yyyymmdd=$(date -u -d "${leg_start_date}" +%Y%m%d)
    leg_start_date_yyyy=$(date -u -d "${leg_start_date}" +%Y)
    leg_end_date_yyyy=$(date -u -d "${leg_end_date}" +%Y)

    # Check whether there's actually time left to simulate - exit otherwise
    if [ ${leg_length_sec} -le 0 ]
    then
        info "Leg start date equal to or after end of simulation."
        info "Nothing left to do. Exiting."
        exit 0
    fi

    # -------------------------------------------------------------------------
    # *** Prepare the run directory for a run from scratch
    # -------------------------------------------------------------------------
    if ! $leg_is_restart
    then
        # ---------------------------------------------------------------------
        # *** Check if run dir is empty. If not, and if we are allowed to do so
        #     by ${force_run_from_scratch}, remove everything
        # ---------------------------------------------------------------------
        if $(ls * >& /dev/null)
        then
            if ${force_run_from_scratch}
            then
                rm -fr ${run_dir}/*
            else
                error "Run directory not empty and \$force_run_from_scratch not set."
            fi
        fi

        # ---------------------------------------------------------------------
        # *** Copy executables of model components
        # *** Additionally, create symlinks to the original place for reference
        # ---------------------------------------------------------------------
        cp    ${ifs_exe_file} .
        ln -s ${ifs_exe_file} $(basename ${ifs_exe_file}).lnk

        cp    ${nem_exe_file} .
        ln -s ${nem_exe_file} $(basename ${nem_exe_file}).lnk

        cp    ${rnf_exe_file} .
        ln -s ${rnf_exe_file} $(basename ${rnf_exe_file}).lnk

        cp    ${xio_exe_file} .
        ln -s ${xio_exe_file} $(basename ${xio_exe_file}).lnk

        # ---------------------------------------------------------------------
        # *** Files needed for IFS (linked)
        # ---------------------------------------------------------------------

        # Initial data
        ln -s \
        ${ini_data_dir}/ifs/${ifs_grid}/${leg_start_date_yyyymmdd}/ICMGGECE3INIUA \
                                                            ICMGG${exp_name}INIUA
        ln -s \
        ${ini_data_dir}/ifs/${ifs_grid}/${leg_start_date_yyyymmdd}/ICMGGECE3INIT \
                                                            ICMGG${exp_name}INIT
        ln -s \
        ${ini_data_dir}/ifs/${ifs_grid}/${leg_start_date_yyyymmdd}/ICMSHECE3INIT \
                                                            ICMSH${exp_name}INIT
        # Other stuff
        ln -s ${ini_data_dir}/ifs/rtables/* .

        # Output control (ppt files)
        mkdir postins
        cp ${ctrl_file_dir}/ppt* postins/
        f_min=$(( ifs_output_freq * ifs_time_step_sec / 60 ))
        for t in $(seq $f_min $f_min 1439)
        do
            hh=$(printf %02d $((t/60)) )
            mm=$(printf %02d $((t%60)) )
            ln -s pptdddddd0000 postins/pptdddddd$hh$mm
        done
        /bin/ls -1 postins/* > dirlist

        # ---------------------------------------------------------------------
        # *** Files needed for NEMO (linked)
        # ---------------------------------------------------------------------

        # Link initialisation files for matching ORCA grid
        for f in \
            bathy_meter.nc coordinates.nc \
            ahmcoef.nc \
            K1rowdrg.nc M2rowdrg.nc mask_itf.nc \
            decay_scale_bot.nc decay_scale_cri.nc \
            mixing_power_bot.nc mixing_power_cri.nc mixing_power_pyc.nc \
            runoff_depth.nc
        do
            [ -f ${ini_data_dir}/nemo/initial/${nem_grid}/$f ] && ln -s ${ini_data_dir}/nemo/initial/${nem_grid}/$f
        done

        # Link geothermal heating file (independent of grid) and matching weight file
        ln -s ${ini_data_dir}/nemo/initial/Goutorbe_ghflux.nc
        ln -s ${ini_data_dir}/nemo/initial/weights_Goutorbe1_2_orca${nem_res_hor}_bilinear.nc

        # Link either restart files or climatology files for the initial state
        if $(has_config nemo:start_from_restart)
        then
            # When linking restart files, we accept three options:
            # (1) Merged files for ocean and ice, i.e.
            #     restart_oce.nc and restart_ice.nc
            # (2) One-file-per-MPI-rank, i.e.
            #     restart_oce_????.nc and restart_ice_????.nc
            #     No check is done whether the number of restart files agrees
            #     with the number of MPI ranks for NEMO!
            # (3) One-file-per-MPI-rank with a prefix, i.e.
            #     <exp_name>_<time_step>_restart_oce_????.nc (similar for the ice)
            #     The prefix is ignored.
            # The code assumes that one of the options can be applied! If more
            # options are applicable, the first is chosen. If none of the
            # options apply, NEMO will crash with missing restart file.
            if   ls -U ${nem_restart_file_path}/restart_[oi]ce.nc > /dev/null 2>&1
            then
                 ln -s ${nem_restart_file_path}/restart_[oi]ce.nc ./

            elif ls -U ${nem_restart_file_path}/restart_[oi]ce_????.nc > /dev/null 2>&1
            then
                 ln -s ${nem_restart_file_path}/restart_[oi]ce_????.nc ./

            else
                for f in ${nem_restart_file_path}/????_????????_restart_[oi]ce_????.nc
                do
                    ln -s $f $(echo $f | sed 's/.*_\(restart_[oi]ce_....\.nc\)/\1/')
                done
            fi
        else

            # Temperature and salinity files for initialisation
            ln -s ${ini_data_dir}/nemo/climatology/absolute_salinity_WOA13_decav_Reg1L75_clim.nc
            ln -s ${ini_data_dir}/nemo/climatology/conservative_temperature_WOA13_decav_Reg1L75_clim.nc
            ln -s ${ini_data_dir}/nemo/climatology/weights_WOA13d1_2_orca${nem_res_hor}_bilinear.nc

            # Grid dependent runoff files
            case ${nem_grid} in
            ORCA1*)   ln -s ${ini_data_dir}/nemo/climatology/runoff-icb_DaiTrenberth_Depoorter_ORCA1_JD.nc ;;
            ORCA025*) ln -s ${ini_data_dir}/nemo/climatology/ORCA_R025_runoff_v1.1.nc ;;
            esac
        fi


        # XIOS files
        . ${ctrl_file_dir}/iodef.xml.sh > iodef.xml
        ln -s ${ctrl_file_dir}/context_nemo.xml
        ln -s ${ctrl_file_dir}/domain_def_nemo.xml
        ln -s ${ctrl_file_dir}/field_def_nemo-lim.xml
        ln -s ${ctrl_file_dir}/field_def_nemo-opa.xml
        ln -s ${ctrl_file_dir}/field_def_nemo-pisces.xml
        ln -s ${ctrl_file_dir}/file_def_nemo-lim3.xml file_def_nemo-lim.xml
        ln -s ${ctrl_file_dir}/file_def_nemo-opa.xml
        ln -s ${ctrl_file_dir}/file_def_nemo-pisces.xml

        # ---------------------------------------------------------------------
        # *** Files needed for the Runoff mapper (linked)
        # ---------------------------------------------------------------------

        ln -s ${ini_data_dir}/runoff-mapper/runoff_maps.nc

        # ---------------------------------------------------------------------
        # *** Files needed for OASIS (linked)
        # ---------------------------------------------------------------------

        oas_grid_dir=${ini_data_dir}/oasis/T${ifs_res_hor}-ORCA${nem_res_hor}

        # Name table file
        ln -s ${ini_data_dir}/oasis/cf_name_table.txt

        # Grid definition files
        ln -s ${oas_grid_dir}/areas.nc
        ln -s ${oas_grid_dir}/grids.nc
        ln -s ${oas_grid_dir}/masks.nc

        # Weight files
        case ${ifs_res_hor} in
            159)  oas_agrd=080
                  ;;
            255)  oas_agrd=128
                  ;;
            511)  oas_agrd=256
                  ;;
            799)  oas_agrd=400
                  ;;
            *)    error "Unsupported horizontal resolution (IFS): ${ifs_res_hor}"
                  ;;
        esac

        case ${nem_res_hor} in
              1)  oas_ogrd=O1t0
                  ;;
            025)  oas_ogrd=Ot25
                  ;;
              *)  error "Unsupported horizontal resolution (NEMO): ${nem_res_hor}"
                  ;;
        esac

        if ${oas_link_weights}
        then
            for f in ${oas_grid_dir}/rmp_????_to_????_GAUSWGT.nc
            do
                ln -s $f
            done
        fi

        for f in ${oas_rst_files}
        do
            cp ${oas_grid_dir}/rst/$f .
        done

    else # i.e. $leg_is_restart == true

        # ---------------------------------------------------------------------
        # *** Remove all leftover output files from previous legs
        # ---------------------------------------------------------------------

        # IFS files
        rm -f ICM{SH,GG}${exp_name}+??????

        # NEMO files
        rm -f ${exp_name}_??_????????_????????_{grid_U,grid_V,grid_W,grid_T,icemod,SBC,scalar,SBC_scalar}.nc

    fi # ! $leg_is_restart

    # -------------------------------------------------------------------------
    # *** Create some control files
    # -------------------------------------------------------------------------

    # IFS, NEMO, LIM namelist and OASIS namcouple files
    . ${ctrl_file_dir}/namelist.ifs.sh                          > fort.4
    . ${ctrl_file_dir}/namelist.nemo.ref.sh                     > namelist_ref
    . ${ctrl_file_dir}/namelist.nemo-${nem_grid}-coupled.cfg.sh > namelist_cfg
    . ${ctrl_file_dir}/namelist.lim3.ref.sh                     > namelist_ice_ref
    . ${ctrl_file_dir}/namelist.lim3-${nem_grid}.cfg.sh         > namelist_ice_cfg

    . ${ctrl_file_dir}/namelist.runoffmapper.sh                 > namelist.runoffmapper

    . ${ctrl_file_dir}/namcouple.sh                             > namcouple

    # -------------------------------------------------------------------------
    # *** IFS climatology data
    # -------------------------------------------------------------------------
    icmclfile=${ini_data_dir}/ifs/${ifs_grid}/climate/ICMCL
    tempfile=tmp.$$
    grib_set=${GRIB_BIN_PATH}${GRIB_BIN_PATH:+/}grib_set

    # Create data for december, the year before the leg starts
    ${grib_set} \
    -s dataDate=$(printf "%04d" $((leg_start_date_yyyy-1)))1215 ${icmclfile}-12 \
                                                           ICMCL${exp_name}INIT

    # Create data for all month in the years of the leg
    for (( y=${leg_start_date_yyyy} ; y<=${leg_end_date_yyyy} ; y++ ))
    do
        yy=$(printf "%04d" $y)
        for m in {1..12}
        do
            mm=$(printf "%02d" $m)
            ${grib_set} -s dataDate=${yy}${mm}15 ${icmclfile}-${mm} ${tempfile}
            cat ${tempfile} >> ICMCL${exp_name}INIT
        done
    done

    # Create data for january, the year after the leg ends
    ${grib_set} -s dataDate=$(printf "%04d" $((leg_end_date_yyyy+1)))0115 ${icmclfile}-01 \
                                                                              ${tempfile}
    cat ${tempfile} >> ICMCL${exp_name}INIT

    # Clean up
    rm -f ${tempfile}

    # -------------------------------------------------------------------------
    # *** Link the appropriate NEMO restart files of the previous leg
    # -------------------------------------------------------------------------
    if $leg_is_restart
    then
        ns=$(printf %08d $(( leg_start_sec / nem_time_step_sec - nem_restart_offset )))
        for (( n=0 ; n<nem_numproc ; n++ ))
        do
            np=$(printf %04d ${n})
            ln -fs ${exp_name}_${ns}_restart_oce_${np}.nc restart_oce_${np}.nc
            ln -fs ${exp_name}_${ns}_restart_ice_${np}.nc restart_ice_${np}.nc
        done

        # Make sure there are no global restart files
        # If links are found, they will be removed. We are cautious and do
        # _not_ remove real files! However, if real global restart files are
        # present, NEMO/LIM will stop because time stamps will not match.
        [ -h restart_oce.nc ] && rm restart_oce.nc
        [ -h restart_ice.nc ] && rm restart_ice.nc
    fi

    # -------------------------------------------------------------------------
    # *** Remove some OASIS files of the previous leg
    # -------------------------------------------------------------------------
    if $leg_is_restart
    then
        rm -f anaisout_*
    fi

    # -------------------------------------------------------------------------
    # *** Start the run
    # -------------------------------------------------------------------------
    export DR_HOOK_IGNORE_SIGNALS='-1'

    export CPLNG='active'

    # Use the launch function from the platform configuration file
    t1=$(date +%s)
    launch \
        ${xio_numproc} ${xio_exe_file} -- \
        ${nem_numproc} ${nem_exe_file} -- \
        ${rnf_numproc} ${rnf_exe_file} -- \
        ${ifs_numproc} ${ifs_exe_file} -v ecmwf -e ${exp_name}
    t2=$(date +%s)

    tr=$(date -d "0 -$t1 sec + $t2 sec" +%T)

    # -------------------------------------------------------------------------
    # *** Check for signs of success
    #     Note the tests provide no guarantee that things went fine! They are
    #     just based on the IFS and NEMO log files. More tests (e.g. checking
    #     restart files) could be implemented.
    # -------------------------------------------------------------------------

    # Checking for IFS success
    if [ -f ifs.stat ]
    then
        if [ "$(awk 'END{print $3}' ifs.stat)" == "CNT0" ]
        then
            info "Leg successfully completed according to IFS log file 'ifs.stat'."
        else
            error "Leg not completed according to IFS log file 'ifs.stat'."
        fi
    else
        error "IFS log file 'ifs.stat' not found after run."
    fi

    # Check for NEMO success
    if [ -f ocean.output ]
    then
        if [ "$(awk '/New day/{d=$10}END{print d}' ocean.output)" == "$(date -d "${leg_end_date} - 1 day" +%Y/%m/%d)" ]
        then
            info "Leg successfully completed according to NEMO log file 'ocean.output'."
        else
            error "Leg not completed according to NEMO log file 'ocean.output'."
        fi

    else
        error "NEMO log file 'ocean.output' not found after run."
    fi

    # -------------------------------------------------------------------------
    # *** Move IFS output files to archive directory
    # -------------------------------------------------------------------------
    outdir="output/ifs/$(printf %03d $((leg_number)))"
    mkdir -p ${outdir}

    prv_leg=$(printf %03d $((leg_number-1)))

    # This takes care of a special IFS feature: The output for the last time
    # step of each leg is written at the first time step of the new leg. The
    # following code makes sure that the output is appended to the appropriate
    # file. Since GRIB files are just streams, its done with a simple cat
    # command.
    for f in ICMSH${exp_name}+?????? ICMGG${exp_name}+??????
    do
        if [ -f output/ifs/${prv_leg}/${f} ]
        then
            cat ${f} >> output/ifs/${prv_leg}/${f}
            rm -f ${f}
        else
            mv ${f} ${outdir}
        fi
    done

    # -------------------------------------------------------------------------
    # *** Move NEMO output files to archive directory
    # -------------------------------------------------------------------------
    outdir="output/nemo/$(printf %03d $((leg_number)))"
    mkdir -p ${outdir}

    for v in grid_U grid_V grid_W grid_T icemod SBC scalar SBC_scalar
    do
        for f in ${exp_name}_??_????????_????????_${v}.nc
        do
            test -f $f && mv $f $outdir/
        done
    done

    # -------------------------------------------------------------------------
    # *** Move IFS restart files to archive directory
    # -------------------------------------------------------------------------
    if $leg_is_restart
    then
        outdir="restart/ifs/$(printf %03d $((leg_number)))"
        mkdir -p ${outdir}

        # Figure out the time part of the restart files (cf. CTIME on rcf files)
        # NOTE: Assuming that restarts are at full days (time=0000) only!
        nd="$(printf %06d $((leg_start_sec/(24*3600))))0000"

        mv srf${nd}.???? ${outdir}
    fi

    # -------------------------------------------------------------------------
    # *** Move NEMO restart files to archive directory
    # -------------------------------------------------------------------------
    if $leg_is_restart
    then
        outdir="restart/nemo/$(printf %03d $((leg_number)))"
        mkdir -p ${outdir}

        ns=$(printf %08d $(( leg_start_sec / nem_time_step_sec - nem_restart_offset )))
        for f in oce ice
        do
            mv ${exp_name}_${ns}_restart_${f}_????.nc ${outdir}
        done
    fi

    # -------------------------------------------------------------------------
    # *** Copy OASIS restart files to archive directory
    #     NOTE: These files are copied and not moved as they are used in the
    #           next leg!
    #           Note also that the OASIS restart files present at the end of
    #           the leg correspond to the start of the next leg!
    # -------------------------------------------------------------------------
    outdir="restart/oasis/$(printf %03d $((leg_number+1)))"
    mkdir -p ${outdir}

    for f in ${oas_rst_files}
    do
        test -f ${f} && cp ${f} ${outdir}
    done

    # -------------------------------------------------------------------------
    # *** Copy rcf files to the archive directory (of the next leg!)
    # -------------------------------------------------------------------------
    outdir="restart/ifs/$(printf %03d $((leg_number+1)))"
    mkdir -p ${outdir}

    for f in rcf
    do
        test -f ${f} && cp ${f} ${outdir}
    done

    # -------------------------------------------------------------------------
    # *** Move log files to archive directory
    # -------------------------------------------------------------------------
    outdir="log/$(printf %03d $((leg_number)))"
    mkdir -p ${outdir}

    for f in \
        ifs.log ifs.stat fort.4 ocean.output \
        time.step solver.stat \
        nout.000000 debug.??.?????? A*_??.nc O*_??.nc
    do
        test -f ${f} && mv ${f} ${outdir}
    done

    # -------------------------------------------------------------------------
    # *** Write the restart control file
    # -------------------------------------------------------------------------

    # Compute CPMIP performance
    sypd="$(cpmip_sypd $leg_length_sec $(($t2 - $t1)))"
    chpsy="$(cpmip_chpsy  $leg_length_sec $(($t2 - $t1)) $(($ifs_numproc + $nem_numproc + $rnf_numproc + $xio_numproc)))"

    echo "#"                                             | tee -a ${ece_info_file}
    echo "# Finished leg at `date '+%F %T'` after ${tr} (hh:mm:ss)" \
                                                         | tee -a ${ece_info_file}
    echo "# CPMIP performance: $sypd SYPD   $chpsy CHPSY"| tee -a ${ece_info_file}
    echo "leg_number=${leg_number}"                      | tee -a ${ece_info_file}
    echo "leg_start_date=\"${leg_start_date}\""          | tee -a ${ece_info_file}
    echo "leg_end_date=\"${leg_end_date}\""              | tee -a ${ece_info_file}

    # Need to reset force_run_from_scratch in order to avoid destroying the next leg
    force_run_from_scratch=false
    special_restart=false

done # loop over legs

# -----------------------------------------------------------------------------
# *** Platform dependent finalising of the run
# -----------------------------------------------------------------------------
finalise

exit 0