Coral/templates/scripts/skeleton.sh

# Function leap days calculates the number of leap days (29th of Februrary) in
# a time intervall between two dates.
#
# Usage leap_days START_DATE END_DATE
#TODO: simplify this
function leap_days()
{
    local ld=0
    local frstYYYY=$(date -ud "$1" +%Y)
    local lastYYYY=$(date -ud "$2" +%Y)

    set +e

    # Check first year for leap day between start and end date
    $(date -ud "${frstYYYY}-02-29" > /dev/null 2>&1) \
	&& (( $(date -ud "$1" +%s) < $(date -ud "${frstYYYY}-03-01" +%s) )) \
	&& (( $(date -ud "$2" +%s) > $(date -ud "${lastYYYY}-02-28" +%s) )) \
	&& (( ld++ ))

    # Check intermediate years for leap day
    for (( y=(( ${frstYYYY}+1 )); y<=(( ${lastYYYY}-1 )); y++ ))
    do
        $(date -ud "$y-02-29" > /dev/null 2>&1) && (( ld++ ))
    done

    # Check last year (if different from first year) for leap day between start
    # and end date
    (( $lastYYYY > $frstYYYY )) \
	&& $(date -ud "${lastYYYY}-02-29" > /dev/null 2>&1) \
	&& (( $(date -ud "$1" +%s) < $(date -ud "${frstYYYY}-03-01" +%s) )) \
	&& (( $(date -ud "$2" +%s) > $(date -ud "${lastYYYY}-02-28" +%s) )) \
	&& (( ld++ ))

    set -e

    echo "$ld"
}


function ml_pp_ice () {
    if [[ ${USER} == "vsc"????? ]]; then
	if [[ "${name_icesheet_model}" == "fETISh" ]]; then
	    module purge
	    module load Python
	    PYTHONPATH="${HOME}/.local/bin:${HOME}/.local/lib/python3.7/site-packages:\${PYTHONPATH}"
	    module load CDO
	    echo ${PYTHONPATH} > ${start_dir}/ppath
	    echo "genius and fetish detected" > ${start_dir}/tmpout
	else
	    echo "On Genius, ice sheet models other than fETISh not implemented yet" > ${start_dir}/tmpout
	    exit 2
	fi
    elif [[ ${HOSTNAME} == "lm3"* ]]; then
	if [[ "${name_icesheet_model}" == "fETISh" ]]; then
	    echo "lemaitre3 and fetish detected" > ${start_dir}/tmpout
	    module purge
	    module load releases/2018a CDO/1.8.2-intel-2018a Python/3.6.4-intel-2018a
	    # PYTHONPATH="${HOME}/.local/lib/python3.6/site-packages:${PYTHONPATH}"
	    # module load NCO CDO
	else
	    echo "On lemaitre3, ice sheet models other than fETISh not implemented yet" > ${start_dir}/tmpout
	    exit 2
	fi

    else
	echo "hostname undetected. not implemented yet." > ${start_dir}/tmpout
	exit 2
    fi
}



[[ $@ == *verbose* ]] && set -x

module purge
module load ${module_list:?}
export LD_LIBRARY_PATH=${LD_LIBRARY_PATH:+${LD_LIBRARY_PATH}:}"${extralibs_list}"

#
# Create run directory
# Copy executables
# Copy XIOS XML files into run directory
# Copy namelist files into run directory
#
if [ ! -d ${run_dir:?} ]
then
    mkdir -p ${run_dir}
    if $special_restart
    then
        rsync -av --delete ${run_dir}/../${special_restart_from}/ --exclude log --exclude output --exclude restart --exclude="${special_restart_from}_*" --exclude="ocean*" --exclude="restart_*" --exclude="debug.*" --exclude="output.*" ${run_dir}
        cp -f ${nem_exe_file} ${run_dir}
        cp -f ${xio_exe_file} ${run_dir}
        special_year=${special_restart_date:0:4}
        sed -i "/$special_year/q" ${run_dir}/${info_file}
        . ${run_dir}/${info_file}
        special_restart_leg=$(printf %03d $((leg_number+1)))
        # PUT HERE THE INSTRUCTIONS TO COPY THE restart files
        cd ${run_dir}/../../archive/${special_restart_from}/restart/${special_restart_leg}
        for f in *.nc; do
            nf=${exp_name}${f:4}
            cp $f ${run_dir}/$nf
        done
        cd -
        cd ${run_dir}
        for f in ${exp_name}_????????_restart_???_????.nc; do
            nf=${f:14}
            ln -s $f $nf
        done
        cd -
    fi
fi
cd ${start_dir}
cp -u xios_config/*xml "${run_dir}"
cp -u namelists/* "${run_dir}"
cd "${run_dir}"
# cp -u "${nem_exe_file:?}" .
# cp -u "${xio_exe_file:?}" .

# nem_exe_bn=`basename ${nem_exe_file}`
# xio_exe_bn=`basename ${xio_exe_file}`


# Write fake file for previous fresh water budget adjustment (nn_fwb==2 in namelist)
[[ ! -f EMPave_old.dat ]] && echo "                               0  0.0000000000000000E+00  0.0000000000000000E+00" > EMPave_old.dat
#TODO: Enquiry about this
#
# Link data files into run directory
# Attention: no space in filenames allowed!
# (No protection against word splitting by design)
#
for file in "${ic_files[@]}"; do 
    [[ ! -e ${file#*> } ]] && ln -sf $(sed 's/ *=> */ /' <<< "${ini_data_dir}/${ic_subdir}/${nem_grid}/$file") 
done
for file in "${forcing_files[@]}"; do
    [[ ! -e ${file#*> } ||  "$file" == \** ]] && ln -sf $(sed 's/ *=> */ /' <<< "${ini_data_dir}/${forcing_subdir}/${nem_forcing_set}/$file")
done
for file in "${shared_files[@]}"; do
    [[ ! -e ${file#*> } ]] && ln -sf $(sed 's/ *=> */ /' <<< "${shared_dir}/$file")
done


if [ "${isfcpl}" == "1" ]; then

    if [[ ${USER} == "vsc"????? ]]; then
	echo "Genius and iscpl detected. OK."
    elif [[ ${HOSTNAME} == "lm3"* ]]; then
	echo "Lemaitre3 and iscpl detected. OK."
    else
	echo "You have asked for iscpl, but I have not recognized genius nor lemaitre3. Exiting."
	exit 2
    fi
    
    if [ "${name_icesheet_model}" != "fETISh" ]; then
	echo "only \"fETISh\" is known as an ice sheet model. exiting."
	exit 2
    fi
    
    length_cut_pref=$(( ${#exp_name} + 4 ))
    rebuild_nemo_exe=${nemo_src_dir}/TOOLS/REBUILD_NEMO/rebuild_nemo
    if [ -f ${start_dir}/isfcpl.track ]; then
	source ${start_dir}/isfcpl.track
	# printf "\n" >> isfcpl.track
	# rm -f isfcpl.track
    fi

    ln -sf ${start_dir}/isfcpl.track ${run_dir}
    
    cp -f ${icesheet_src_dir}/* ${run_dir}
    iscpl_nm=.true.
else
    iscpl_nm=.false.
fi

#
# Normalize date formats
#
run_start_date=$(date -uR -d "${run_start_date}")
run_end_date="${run_start_date} + ${run_duration:?}"
run_end_date=$(date -uR -d "${run_end_date}")

run_start_epoch=$(date -u -d"${run_start_date}" +%s)
run_end_epoch=$(date -u -d"${run_end_date}" +%s)




for (( ; run_num_legs>0 ; run_num_legs-- ))
do

    #
    # Initialize variables (using restart file if it exists)
    # Variables ending in '_date' are in RFC2822
    # Variables ending in '_epoch' are in nb of seconds since 1970-01-01
    #
    [[ -r "${info_file:?}" ]] && source "${info_file:?}"
    leg_start_date=${leg_end_date:-$run_start_date}
    leg_number=$((${leg_number:=0}+1))
    leg_start_epoch=$(date -u -d "${leg_start_date}" +%s)
    leg_end_epoch=$(date -u -d "${leg_start_date:?} + ${rst_freq:=$run_duration}" +%s)
    leg_end_date=$(date -uR -d@"${leg_end_epoch}")
    leg_length_sec=$(( leg_end_epoch - leg_start_epoch ))
    leg_start_sec=$(( leg_start_epoch - run_start_epoch ))
    leg_end_sec=$(( leg_end_epoch - run_start_epoch ))

    leg_beforestart_epoch=$(( ${leg_start_epoch} - 43200 ))
    leg_beforestart_date=$(date -uR -d@"${leg_beforestart_epoch}")
    
    leg_almostend_epoch=$(( ${leg_end_epoch} - 43200 ))
    leg_almostend_date=$(date -uR -d@"${leg_almostend_epoch}")

    nextleg_start_epoch=$(( ${leg_end_epoch} + 43200 ))
    nextleg_start_date=$(date -uR -d@"${nextleg_start_epoch}")
    
    run_start_date_yyyymmdd=$(date -u -d "${run_start_date}" +%Y%m%d) # FIXME appears unused
    run_end_date_yyyymmdd=$(date -u -d "${run_end_date}" +%Y%m%d) # FIXME appears unused
    leg_start_date_yyyymmdd=$(date -u -d "${leg_start_date}" +%Y%m%d) # FIXME appears unused
    leg_almostend_date_yyyymmdd=$(date -u -d "${leg_almostend_date}" +%Y%m%d) # FIXME appears unused
    nextleg_start_date_yyyymmdd=$(date -u -d "${nextleg_start_date}" +%Y%m%d) # FIXME appears unused
    leg_beforestart_date_yyyymmdd=$(date -u -d "${leg_beforestart_date}" +%Y%m%d) # FIXME appears unused

    # run_start_date_yyyymm="${run_start_date_yyyymmdd::-2}"
    # run_end_date_yyyymm="${run_end_date_yyyymmdd::-2}"
    # leg_start_date_yyyymm="${leg_start_date_yyyymmdd::-2}"
    # leg_almostend_date_yyyymm="${leg_almostend_date_yyyymmdd::-2}"
    # leg_beforestart_date_yyyymm="${leg_beforestart_date_yyyymmdd::-2}"

    
    if [ "${isfcpl}" == "1" ]; then
	
	# Check if a monthly "icesheet" output file has indeed been asked.
	tmp_check=$(grep -nri "file id=\"file.*\".*name_suffix=\"_icesheet\"" ${run_dir}/file_def.xml | wc -l)
	if (( tmp_check == 0 )); then
	    echo "file_def.xml does not look like you have asked for an icesheet output file! Exiting."
	    exit 2
	elif (( tmp_check > 1 )); then
	    echo "file_def.xml contains at least 2 icesheet output files! This is ambiguous. Exiting."
	    exit 2
	fi
	
	if (( leg_number == 1 )); then

	    mkdir -p ${archive_dir}/isfcpl
	    mkdir -p ${archive_dir}/${name_icesheet_model}
	    rm -f ${start_dir}/isfcpl.track
	    # Counter for the number of restart since last file exchange with the ice sheet model.
	    cnt_res_isfcpl=0

	    isfcpl_window_days=$(echo "(${n_res_perisfcpl} * ${leg_length_sec}) / 86400" | bc)
	    isfcpl_d30=$(echo "${isfcpl_window_days} / 30" | bc -l)	    
	    n_month_isfcpl=$(echo ${isfcpl_d30} | awk '{print int($1+0.5)}')
	    
	    # if (( isfcpl_window_days >= 28 && isfcpl_window_days <= 31 )); then
	    # 	n_month_isfcpl=1
	    # elif (( isfcpl_window_days >= 336 && isfcpl_window_days <= 372 )); then
	    # 	n_month_isfcpl=12
	    # else
	    # 	echo "Can't figure out how many month long is the coupling window."
	    # 	exit 2
	    # fi
	    
	    n_month_full_run=$(( $(( 12 * $(( ${run_end_date_yyyymmdd::4} - ${run_start_date_yyyymmdd::4} )) )) + $(( ${run_end_date_yyyymmdd:4:2} - ${run_start_date_yyyymmdd:4:2} )) ))
	    
	    if (( $(( ${n_month_full_run} % ${n_month_isfcpl} )) != 0 )); then
		echo "n_month_full_run=${n_month_full_run} is not a multiple of n_month_isfcpl=${n_month_isfcpl}."
		exit 2
	    else
		n_isfcpl_window=$(echo "${n_month_full_run} / ${n_month_isfcpl}" | bc)
	    fi

	    tmp=`readlink -f isf_draft_meter.nc`
	    tmp2=`readlink -f bathy_meter.nc`

	    if [ "${tmp}" != "${tmp2}" ]; then
		echo "Error. At initialization, isf_draft_meter.nc and bathy_meter.nc do not point to the same file."
		exit 1
	    fi
	    
	    init_bathy_file=`basename ${tmp}`
	    cp -f ${tmp} ${archive_dir}/isfcpl
	    cp -f ${tmp} ${run_dir}/init_NEMO_bathy.nc

	    date_beg_isfcpl="${run_start_date_yyyymmdd}"
	    init_geom=1
	    init_bathy=1
	    curr_bathy_file=${init_bathy_file}
	    ln -sf ${run_dir}/${icesheet_init_geom_file} ${run_dir}/${name_icesheet_model}_8km_geometry_curr.nc

	    
	    printf "# Information for the ice sheet model\n" > ${start_dir}/isfcpl.track
	    printf "run_name=${exp_name}\n" >> ${start_dir}/isfcpl.track
	    printf "initial_date=${run_start_date_yyyymmdd}\n" >> ${start_dir}/isfcpl.track
	    printf "n_month_isfcpl=${n_month_isfcpl}\n" >> ${start_dir}/isfcpl.track
	    printf "n_isfcpl_window=${n_isfcpl_window}\n" >> ${start_dir}/isfcpl.track
	    printf "date_beg_isfcpl=${date_beg_isfcpl}\n" >> ${start_dir}/isfcpl.track
	    printf "date_end_isfcpl=YYYYMMDD\n\n" >> ${start_dir}/isfcpl.track

	    printf "# Information for NEMO to keep up\n" >> ${start_dir}/isfcpl.track
	    printf "cnt_res_isfcpl=0\n" >> ${start_dir}/isfcpl.track	    
	    printf "init_geom=1\n" >> ${start_dir}/isfcpl.track
	    printf "init_bathy=1\n" >> ${start_dir}/isfcpl.track
	    printf "curr_bathy_file=${init_bathy_file}\n" >> ${start_dir}/isfcpl.track
	    
	    


	fi
	
	ln -sf ${curr_bathy_file} ${run_dir}/bathy_meter.nc
	ln -sf ${curr_bathy_file} ${run_dir}/isf_draft_meter.nc
	
    fi
    
    # Correct for leap days because NEMO standalone uses no-leap calendar
    if (( leap_years == 0 )); then
        leg_length_sec=$(( leg_length_sec - $(leap_days "${leg_start_date}" "${leg_end_date}")*24*3600 ))
        leg_start_sec=$(( leg_start_sec - $(leap_days "${run_start_date}" "${leg_start_date}")*24*3600 ))
        leg_end_sec=$(( leg_end_sec - $(leap_days "${run_start_date}" "${leg_end_date}")*24*3600 ))
    fi
    (( leg_number > 1 )) && leg_is_restart=true || leg_is_restart=false

    #
    # Compute leg end-date and trim if necessary
    #
    (( leg_end_epoch > run_end_epoch )) && leg_end_date=${run_end_epoch}

    #
    # Check whether there is some work left to do
    #
    if (( leg_start_epoch >= run_end_epoch ))
    then
        echo "Leg start date equal to or after end of simulation."
        echo "Nothing left to do. Cleaning and exiting."
	for (( n=0 ; n<nem_numproc ; n++ ))
	do
            np=$(printf %04d ${n})
	    rm -f "${run_dir}/restart_oce_${np}.nc"
	    rm -f "${run_dir}/restart_ice_${np}.nc"
	done
        exit 0
    fi

    #
    # Update namelist
    #
    source build_namelist_cfg.sh > namelist_cfg
    
    # Link the restart files
    ns=$(printf %08d $(( leg_start_sec / nem_time_step_sec - nem_restart_offset )))
    if ((leg_start_sec > 0 )); then
    	for (( n=0 ; n<nem_numproc ; n++ ))
    	do
            np=$(printf %04d ${n})
            [[ -f "${exp_name:?}_${ns}_restart_oce_${np}.nc" ]] || { echo "Error: restart file not found." ; exit 2 ; }
            ln -fs "${exp_name:?}_${ns}_restart_oce_${np}.nc" "restart_oce_${np}.nc"
            [[ -f "${exp_name:?}_${ns}_restart_ice_${np}.nc" ]] || { echo "Error: restart file not found." ; exit 2 ; }
            ln -fs "${exp_name:?}_${ns}_restart_ice_${np}.nc" "restart_ice_${np}.nc"
    	done
    fi

    # Stop here is preponly was specified
    [[ $@ == *preponly* ]] && exit 0

    #
    # Run nemo
    #

    if [[ "$@" == *"dbg"* ]]; then
	echo $HOSTNAME > ${start_dir}/log
	sleep 10000000000000
    fi
    
    time_begin=$(date +%s)
    mpirun -np "${xio_numproc:?}" "${xio_exe_file:?}" : -np "${nem_numproc:?}" "${nem_exe_file:?}"
    time_end=$(date +%s)

    echo "dbg cp made it past mpirun"
    
    if (( isfcpl == 1 )); then

	# NEMO restart index within ice sheet / ocean coupling window
	cnt_res_isfcpl=$(( $(( ${leg_number} - 1 )) % ${n_res_perisfcpl} ))
	sed -i "s/.*cnt_res_isfcpl=.*/cnt_res_isfcpl=${cnt_res_isfcpl}/" ${start_dir}/isfcpl.track

	
	# module purge
	# module load CDO
	fmt_res_isfcpl=$(printf "%02d" ${cnt_res_isfcpl})
	
	in_stencil="${exp_name}_1m_${leg_start_date_yyyymmdd}_${leg_almostend_date_yyyymmdd}_icesheet"
	out_stencil="${exp_name}_icesheet_res${fmt_res_isfcpl}"

	# Renaming.
	for((prc=0;prc<xio_numproc;prc++)); do
	    fmt_xio=$(printf "%04d" ${prc})
	    infile="${in_stencil}_${fmt_xio}.nc"
	    # echo "mv -f ${infile} ./${out_stencil}_${fmt_xio}.nc"
	    mv -f ${infile} ${out_stencil}_${fmt_xio}.nc
	done

	# Rebuilding it.
	module purge
	module load ${module_list:?}
	export LD_LIBRARY_PATH=${LD_LIBRARY_PATH:+${LD_LIBRARY_PATH}:}"${extralibs_list}"

	echo "dbg cp before rebuild"
	
	/bin/bash ${rebuild_nemo_exe} ${out_stencil} ${xio_numproc}
	rm -f ${out_stencil}_????.nc

	echo "dbg cp after rebuild"

	

	if (( ${leg_number} % ${n_res_perisfcpl} == 0 )); then
	    # We have just ended one NEMO leg which closes an ice sheet / ocean coupling window.
	    date_end_isfcpl=${leg_almostend_date_yyyymmdd}
	    sed -i "s/.*date_end_isfcpl=.*/date_end_isfcpl=${date_end_isfcpl}/" ${start_dir}/isfcpl.track

	    echo "dbg cp before rebuild mesh"
	    /bin/bash ${rebuild_nemo_exe} mesh_mask ${nem_numproc}
	    echo "dbg cp after rebuild mesh"
	    

	    module purge
	    ml_pp_ice

	    # Concatenate all icesheet files from all legs contained within this coupling window
	    tmpfile_eos80="curr_ocean2ice_EOS80.nc"
	    tmpfile_teos10="curr_ocean2ice_TEOS10.nc"
	    outfile="${exp_name}_ocean2ice_${date_beg_isfcpl}-${date_end_isfcpl}.nc"
	    
	    cdo -O copy ${out_stencil::-2}??.nc ${tmpfile_teos10}
	    rm -f ${out_stencil::-2}??.nc

	    echo "dbg cp after cdo copy, before pp nemo to fetish"
	    
	    
	    module purge
	    ml_pp_ice
	    ${pp_nemo_to_icesheet_call}

	    mv -f ${tmpfile_eos80} ${outfile}
	    mv -f mesh_mask.nc ${archive_dir}/isfcpl/${exp_name}_mesh_mask_${date_beg_isfcpl}-${date_end_isfcpl}.nc

	    echo "dbg cp after pp nemo to fetish, before fetish"
	    
	    # CALL ICE SHEET MODEL
	    module purge
	    module load ${icesheet_modules}
	    ${icesheet_call}

	    echo "dbg cp after fetish"
	    
	    find . -type f -name "*.mat" -exec mv -f {} ${archive_dir}/${name_icesheet_model} \;
	    ln -sf ${archive_dir}/${name_icesheet_model}/*.mat ./
	    module purge

	    # mv ocean2ice file to arch dir
	    mv -f ${outfile} ${archive_dir}/isfcpl
	    
	    new_geom_file="${exp_name}_ice2ocean_${date_end_isfcpl}.nc"

	    old_geom=`readlink -f ${name_icesheet_model}_8km_geometry_curr.nc`
	    if [ "${init_geom}" == 0 ]; then
		mv -f ${old_geom} ${archive_dir}/isfcpl
	    else
		init_geom=0
		sed -i "s/.*init_geom=.*/init_geom=0/" ${start_dir}/isfcpl.track
		cp -f ${old_geom} ${archive_dir}/isfcpl
	    fi

	    ln -sf ${new_geom_file} ${name_icesheet_model}_8km_geometry_curr.nc
	    
	    # leg number at which NEMO is supposed to receive updated geometry from the ice sheet model.
	    # We need to pre-treat fETISh outputs in order to make them readable by NEMO.

	    # new fETISh output should be ${run_dir}/fETISh_8km_geometry_curr.nc
	    module purge
	    ml_pp_ice
	    ${pp_icesheet_to_nemo_call}

	    # Copy (for initial) or move (non initial) previous bathymetry file to archive and link new one to what NEMO will read.
	    new_bathy_file="${name_icesheet_model}_eORCA025-SO_bathymetry_${date_end_isfcpl}.nc"
	    mv -f ${name_icesheet_model}_new_8km_eORCA025-bathymetry_curr.nc ${new_bathy_file}
		
	    old_bathy=`readlink -f ${run_dir}/bathy_meter.nc`
	    if [ "${init_bathy}" == 0 ]; then
		mv -f ${old_bathy} ${archive_dir}/isfcpl
	    else
		init_bathy=0
		sed -i "s/^init_bathy=.*$/init_bathy=0/g" ${start_dir}/isfcpl.track
		cp -f ${old_bathy} ${archive_dir}/isfcpl
	    fi
	    curr_bathy_file=${new_bathy_file}
	    sed -i "s/.*curr_bathy_file=.*/curr_bathy_file=${curr_bathy_file}/" ${start_dir}/isfcpl.track
	    
	    ln -sf ${new_bathy_file} ${run_dir}/bathy_meter.nc
	    ln -sf ${new_bathy_file} ${run_dir}/isf_draft_meter.nc
		
	    module purge
	    module load ${module_list:?}
	    export LD_LIBRARY_PATH=${LD_LIBRARY_PATH:+${LD_LIBRARY_PATH}:}"${extralibs_list}"

	    
	    printf "# Information for the ice sheet model\n" > ${start_dir}/isfcpl.track
	    printf "run_name=${exp_name}\n" >> ${start_dir}/isfcpl.track
	    printf "initial_date=${run_start_date_yyyymmdd}\n" >> ${start_dir}/isfcpl.track
	    printf "n_month_isfcpl=${n_month_isfcpl}\n" >> ${start_dir}/isfcpl.track
	    printf "n_isfcpl_window=${n_isfcpl_window}\n" >> ${start_dir}/isfcpl.track
	    printf "date_beg_isfcpl=${nextleg_start_date_yyyymmdd}\n" >> ${start_dir}/isfcpl.track
	    printf "date_end_isfcpl=YYYYMMDD\n\n" >> ${start_dir}/isfcpl.track
	    printf "# Information for NEMO to keep up\n" >> ${start_dir}/isfcpl.track	    
	    printf "cnt_res_isfcpl=0\n" >> ${start_dir}/isfcpl.track
	    printf "init_geom=${init_geom}\n" >> ${start_dir}/isfcpl.track
	    printf "init_bathy=${init_bathy}\n" >> ${start_dir}/isfcpl.track
	    printf "curr_bathy_file=${curr_bathy_file}\n" >> ${start_dir}/isfcpl.track
	    
	    module purge
	    module load ${module_list:?}
	    export LD_LIBRARY_PATH=${LD_LIBRARY_PATH:+${LD_LIBRARY_PATH}:}"${extralibs_list}"
	    
	fi
    fi
    
    #############################


    formatted_leg_number=$(printf %03d $((leg_number)))
    #
    # Move NEMO output files to archive directory
    # 
    outdir="${archive_dir:?}/output/${formatted_leg_number}"
    mkdir -p "${outdir}"

    shopt -s nullglob
    for v in grid_U grid_V grid_W grid_T icemod icemoa SBC SBC_scalar scalar fwfisf
    do
        for f in ${exp_name}_??_????????_????????_${v}_????.nc
        do
            mv "$f" "$outdir/"
        done
        for f in ${exp_name}_??_????????_????????_${v}.nc
        do
            mv "$f" "$outdir/"
        done
        for f in ${exp_name}_??_${v}.nc
        do
            mv "$f" "$outdir/"
        done
    done

    #
    # Move NEMO restart files from previous run to archive directory
    #
    outdir="$archive_dir/restart/${formatted_leg_number}"
    mkdir -p "${outdir}"

    for f in ${exp_name}_${ns}_restart_???_????.nc
    do
        [ -f "$f" ] && mv "$f" "${outdir}"
    done

    #
    # Move log files to archive directory
    #
    outdir="$archive_dir/log/${formatted_leg_number}"
    mkdir -p "${outdir}"
    for f in ocean.output time.step solver.stat ; do mv "${f}" "${outdir}"; done
    
    
    

    shopt -u nullglob
    # 
    # Write checkpoint control file
    # TODO: enquiry why 0 -1 +2 rather than 2- 1
    #
    tr=$(date -d "0 -$time_begin sec + $time_end sec" +%T) 
    current_date=$(date +'%F %T')
    {
	echo "#"
	echo "# Finished leg at ${current_date} after ${tr} (hh:mm:ss)" 
	echo "leg_number=${leg_number}"
	echo "leg_start_date=\"${leg_start_date}\""
	echo "leg_end_date=\"${leg_end_date}\""
    } | tee -a "${info_file}"

    special_restart=false

done

#
# Move back to submission directory
#
cd - >/dev/null


#
# Check whether there is some work left to do in a further job
#
[[ $@ == *noresubmit* ]] && exit 0
if (( leg_end_epoch < run_end_epoch )) ; then
    echo "Leg end earlier than end of simulation."
    echo "Submitting another job."
    
    #[[ $@ == *local* ]] && exec $0 $@ || scontrol requeue $SLURM_JOB_ID #TODO: factorize this
    if [[ "$@" == *"local"* ]] ; then
	cd ${start_dir}
	sleep 2
	./run.sh local && exit 0

	#exec "$0" "$@"
    elif [ "${submit_command}" == *sbatch* ] ; then
        sbatch $0 $@ | awk '{print $4}' >> ${run_dir}/.coral_jobs
    elif [[ $USER == "vsc"* ]]; then
        if [ -f ${PBS_O_WORKDIR-$PWD}/tmp/credits_file ]; then
            credits=`cat ${PBS_O_WORKDIR-$PWD}/tmp/credits_file`
        else
            #logfiletmp=`echo ${PBS_O_WORKDIR-$PWD} | rev | cut -d'/' -f1 | rev`
            #logfiletmp2=`echo $logfiletmp | cut -d'_' -f1`
            #logfile=`ls ${PBS_O_WORKDIR-$PWD}"/"$logfiletmp2*.o*`
            #credits=`grep "account string:" $logfile | sed 's/^.*: //'`
            credits_tmp=`grep "account string:" ${PBS_O_WORKDIR-$PWD}/*.o* -R | sed 's/^.*: //'`
            credits=`echo $credits_tmp | awk '{ print $1 }'`

            mkdir -p tmp
            echo ${credits} > tmp/credits_file
        fi
        script_name=`cat ${PBS_O_WORKDIR-$PWD}/tmp/script_name`
        run_path=`cat ${PBS_O_WORKDIR-$PWD}/tmp/address`
        ssh login2 "cd $run_path; qsub -A $credits $script_name | tee -a coral_jobs"
        sleep 2
        jobid=`cat coral_jobs`
        rm -f coral_jobs
        jobid=${jobid%%.*}
        echo "${jobid}" >> "${run_dir}"/.coral_jobs ;
    else
        "qsub" -v PBS_OPTIONS="$@" "$0" | tee -a coral_jobs
        sleep 2
        jobid=`cat coral_jobs`
        rm -f coral_jobs
        jobid=${jobid%.*}
        echo "${jobid}" >> "${run_dir}"/.coral_jobs
    fi
else
    if [[ $USER == "vsc"* ]]; then
        run_path=`cat ${PBS_O_WORKDIR-$PWD}/tmp/address`
        rm -rf $run_path/tmp
    fi

    echo "Nothing left to do. Cleaning and exiting." # FIXME Factorize this (we have two exit points)
    for (( n=0 ; n<nem_numproc ; n++ ))
    do
        np=$(printf %04d ${n})
        rm -f "restart_oce_${np}.nc"
        rm -f "restart_ice_${np}.nc"
    done
fi

exit 0