isfcpl skeleton

templates/scripts/skeleton.sh

@@ -13,9 +13,9 @@ function leap_days()
 
     # Check first year for leap day between start and end date
     $(date -ud "${frstYYYY}-02-29" > /dev/null 2>&1) \
-    && (( $(date -ud "$1" +%s) < $(date -ud "${frstYYYY}-03-01" +%s) )) \
-    && (( $(date -ud "$2" +%s) > $(date -ud "${lastYYYY}-02-28" +%s) )) \
-    && (( ld++ ))
+	&& (( $(date -ud "$1" +%s) < $(date -ud "${frstYYYY}-03-01" +%s) )) \
+	&& (( $(date -ud "$2" +%s) > $(date -ud "${lastYYYY}-02-28" +%s) )) \
+	&& (( ld++ ))
 
     # Check intermediate years for leap day
     for (( y=(( ${frstYYYY}+1 )); y<=(( ${lastYYYY}-1 )); y++ ))
@@ -26,10 +26,10 @@ function leap_days()
     # Check last year (if different from first year) for leap day between start
     # and end date
     (( $lastYYYY > $frstYYYY )) \
-    && $(date -ud "${lastYYYY}-02-29" > /dev/null 2>&1) \
-    && (( $(date -ud "$1" +%s) < $(date -ud "${frstYYYY}-03-01" +%s) )) \
-    && (( $(date -ud "$2" +%s) > $(date -ud "${lastYYYY}-02-28" +%s) )) \
-    && (( ld++ ))
+	&& $(date -ud "${lastYYYY}-02-29" > /dev/null 2>&1) \
+	&& (( $(date -ud "$1" +%s) < $(date -ud "${frstYYYY}-03-01" +%s) )) \
+	&& (( $(date -ud "$2" +%s) > $(date -ud "${lastYYYY}-02-28" +%s) )) \
+	&& (( ld++ ))
 
     set -e
 
@@ -37,9 +37,75 @@ function leap_days()
 }
 
 
+function ml_icesheet () {
+    if [[ ${USER} == "vsc"????? ]]; then
+	if [[ "${name_icesheet_model}" == "fETISh" ]]; then
+	    module purge
+	    module load Python
+	    PYTHONPATH="${HOME}/.local/bin:${HOME}/.local/lib/python3.7/site-packages:\${PYTHONPATH}"
+	    module load CDO
+	    echo ${PYTHONPATH} > ${start_dir}/ppath
+	    echo "genius and fetish detected" > ${start_dir}/tmpout
+	else
+	    echo "On Genius, ice sheet models other than fETISh not implemented yet" > ${start_dir}/tmpout
+	    exit 2
+	fi
+    elif [[ ${HOSTNAME} == "lm3"* ]]; then
+	if [[ "${name_icesheet_model}" == "fETISh" ]]; then
+	    echo "lemaitre3 and fetish detected" > ${start_dir}/tmpout
+	    module --force purge
+        module load tis/2018.01
+        module load releases/2018b
+        module --ignore-cache load MCR/R2018b
+	else
+	    echo "On lemaitre3, ice sheet models other than fETISh not implemented yet" > ${start_dir}/tmpout
+	    exit 2
+	fi
+
+    else
+	echo "hostname undetected. not implemented yet." > ${start_dir}/tmpout
+	exit 2
+    fi
+}
+
+
+# Function for loading modules 
+function ml_pp_ice () {
+    if [[ ${USER} == "vsc"????? ]]; then
+	if [[ "${name_icesheet_model}" == "fETISh" ]]; then
+	    module purge
+	    module load Python
+	    PYTHONPATH="${HOME}/.local/bin:${HOME}/.local/lib/python3.7/site-packages:\${PYTHONPATH}"
+	    module load CDO
+	    echo ${PYTHONPATH} > ${start_dir}/ppath
+	    echo "genius and fetish detected" > ${start_dir}/tmpout
+	else
+	    echo "On Genius, ice sheet models other than fETISh not implemented yet" > ${start_dir}/tmpout
+	    exit 2
+	fi
+    elif [[ ${HOSTNAME} == "lm3"* ]]; then
+	if [[ "${name_icesheet_model}" == "fETISh" ]]; then
+	    echo "lemaitre3 and fetish detected" > ${start_dir}/tmpout
+	    module purge
+	    module load releases/2018a CDO/1.8.2-intel-2018a Python/3.6.4-intel-2018a
+	    # PYTHONPATH="${HOME}/.local/lib/python3.6/site-packages:${PYTHONPATH}"
+	    # module load NCO CDO
+	else
+	    echo "On lemaitre3, ice sheet models other than fETISh not implemented yet" > ${start_dir}/tmpout
+	    exit 2
+	fi
+
+    else
+	echo "hostname undetected. not implemented yet." > ${start_dir}/tmpout
+	exit 2
+    fi
+}
+
+
 
 [[ $@ == *verbose* ]] && set -x
 
+module purge
 module load ${module_list:?}
 export LD_LIBRARY_PATH=${LD_LIBRARY_PATH:+${LD_LIBRARY_PATH}:}"${extralibs_list}"
 
@@ -51,37 +117,41 @@ export LD_LIBRARY_PATH=${LD_LIBRARY_PATH:+${LD_LIBRARY_PATH}:}"${extralibs_list}
 #
 if [ ! -d ${run_dir:?} ]
 then
-     mkdir -p ${run_dir}
-     if $special_restart
-     then
-          rsync -av --delete ${run_dir}/../${special_restart_from}/ --exclude log --exclude output --exclude restart --exclude="${special_restart_from}_*" --exclude="ocean*" --exclude="restart_*" --exclude="debug.*" --exclude="output.*" ${run_dir}
-          cp -f ${nem_exe_file} ${run_dir}
-          cp -f ${xio_exe_file} ${run_dir}
-          special_year=${special_restart_date:0:4}
-          sed -i "/$special_year/q" ${run_dir}/${info_file}
-          . ${run_dir}/${info_file}
-          special_restart_leg=$(printf %03d $((leg_number+1)))
-          # PUT HERE THE INSTRUCTIONS TO COPY THE restart files
-          cd ${run_dir}/../../archive/${special_restart_from}/restart/${special_restart_leg}
-          for f in *.nc; do
-              nf=${exp_name}${f:4}
-              cp $f ${run_dir}/$nf
-          done
-          cd -
-          cd ${run_dir}
-          for f in ${exp_name}_????????_restart_???_????.nc; do
-              nf=${f:14}
-              ln -s $f $nf
-          done
-          cd -
-     fi
+    mkdir -p ${run_dir}
+    if $special_restart
+    then
+        rsync -av --delete ${run_dir}/../${special_restart_from}/ --exclude log --exclude output --exclude restart --exclude="${special_restart_from}_*" --exclude="ocean*" --exclude="restart_*" --exclude="debug.*" --exclude="output.*" ${run_dir}
+        cp -f ${nem_exe_file} ${run_dir}
+        cp -f ${xio_exe_file} ${run_dir}
+        special_year=${special_restart_date:0:4}
+        sed -i "/$special_year/q" ${run_dir}/${info_file}
+        . ${run_dir}/${info_file}
+        special_restart_leg=$(printf %03d $((leg_number+1)))
+        # PUT HERE THE INSTRUCTIONS TO COPY THE restart files
+        cd ${run_dir}/../../archive/${special_restart_from}/restart/${special_restart_leg}
+        for f in *.nc; do
+            nf=${exp_name}${f:4}
+            cp $f ${run_dir}/$nf
+        done
+        cd -
+        cd ${run_dir}
+        for f in ${exp_name}_????????_restart_???_????.nc; do
+            nf=${f:14}
+            ln -s $f $nf
+        done
+        cd -
+    fi
 fi
 cd ${start_dir}
 cp -u xios_config/*xml "${run_dir}"
 cp -u namelists/* "${run_dir}"
 cd "${run_dir}"
-cp -u "${nem_exe_file:?}" .
-cp -u "${xio_exe_file:?}" .
+# cp -u "${nem_exe_file:?}" .
+# cp -u "${xio_exe_file:?}" .
+
+nem_exe_bn=`basename ${nem_exe_file}`
+xio_exe_bn=`basename ${xio_exe_file}`
+
 
 # Write fake file for previous fresh water budget adjustment (nn_fwb==2 in namelist)
 [[ ! -f EMPave_old.dat ]] && echo "                               0  0.0000000000000000E+00  0.0000000000000000E+00" > EMPave_old.dat
@@ -102,6 +172,48 @@ for file in "${shared_files[@]}"; do
 done
 
 
+if [ "${isfcpl}" == "1" ]; then
+
+    if [[ ${USER} == "vsc"????? ]]; then
+	echo "Genius and iscpl detected. OK."
+    elif [[ ${HOSTNAME} == "lm3"* ]]; then
+	echo "Lemaitre3 and iscpl detected. OK."
+    else
+	echo "You have asked for iscpl, but I have not recognized genius nor lemaitre3. Exiting."
+	exit 2
+    fi
+    
+    if [ "${name_icesheet_model}" != "fETISh" ]; then
+	echo "only \"fETISh\" is known as an ice sheet model. exiting."
+	exit 2
+    fi
+    
+    length_cut_pref=$(( ${#exp_name} + 4 ))
+    rebuild_nemo_exe=${nemo_src_dir}/TOOLS/REBUILD_NEMO/rebuild_nemo
+    if [ -f ${start_dir}/isfcpl.track ]; then
+	source ${start_dir}/isfcpl.track
+	# printf "\n" >> isfcpl.track
+	# rm -f isfcpl.track
+    fi
+
+    ln -sf ${start_dir}/isfcpl.track ${run_dir}
+    cp -f ${icesheet_src_dir}/* ${run_dir}
+    iscpl_nm=.true.
+
+    ice_atm_cpl=0
+    
+    if [[ ${xio_exe_file} == *"xios-2"* ]]; then
+	is_xios2=1
+	filedef_xios="${run_dir}/file_def_nemo-opa.xml"
+    else
+	is_xios2=0
+	filedef_xios="${run_dir}/file_def.xml"
+    fi
+     
+else
+    iscpl_nm=.false.
+fi
+
 #
 # Normalize date formats
 #
@@ -112,6 +224,9 @@ run_end_date=$(date -uR -d "${run_end_date}")
 run_start_epoch=$(date -u -d"${run_start_date}" +%s)
 run_end_epoch=$(date -u -d"${run_end_date}" +%s)
 
+
+
+
 for (( ; run_num_legs>0 ; run_num_legs-- ))
 do
 
@@ -129,11 +244,125 @@ do
     leg_length_sec=$(( leg_end_epoch - leg_start_epoch ))
     leg_start_sec=$(( leg_start_epoch - run_start_epoch ))
     leg_end_sec=$(( leg_end_epoch - run_start_epoch ))
+
+    leg_beforestart_epoch=$(( ${leg_start_epoch} - 43200 ))
+    leg_beforestart_date=$(date -uR -d@"${leg_beforestart_epoch}")
+    
+    leg_almostend_epoch=$(( ${leg_end_epoch} - 43200 ))
+    leg_almostend_date=$(date -uR -d@"${leg_almostend_epoch}")
+
+    nextleg_start_epoch=$(( ${leg_end_epoch} + 43200 ))
+    nextleg_start_date=$(date -uR -d@"${nextleg_start_epoch}")
+    
+    run_start_date_yyyymmdd=$(date -u -d "${run_start_date}" +%Y%m%d) # FIXME appears unused
+    run_end_date_yyyymmdd=$(date -u -d "${run_end_date}" +%Y%m%d) # FIXME appears unused
     leg_start_date_yyyymmdd=$(date -u -d "${leg_start_date}" +%Y%m%d) # FIXME appears unused
+    leg_almostend_date_yyyymmdd=$(date -u -d "${leg_almostend_date}" +%Y%m%d) # FIXME appears unused
+    nextleg_start_date_yyyymmdd=$(date -u -d "${nextleg_start_date}" +%Y%m%d) # FIXME appears unused
+    leg_beforestart_date_yyyymmdd=$(date -u -d "${leg_beforestart_date}" +%Y%m%d) # FIXME appears unused
+
+    # run_start_date_yyyymm="${run_start_date_yyyymmdd::-2}"
+    # run_end_date_yyyymm="${run_end_date_yyyymmdd::-2}"
+    # leg_start_date_yyyymm="${leg_start_date_yyyymmdd::-2}"
+    # leg_almostend_date_yyyymm="${leg_almostend_date_yyyymmdd::-2}"
+    # leg_beforestart_date_yyyymm="${leg_beforestart_date_yyyymmdd::-2}"
+
+    
+    if [ "${isfcpl}" == "1" ]; then
+	
+	# Check if a monthly "icesheet" output file has indeed been asked.
+	tmp_check=$(grep -nri "file id=\"file.*\".*name_suffix=\"_icesheet\"" ${filedef_xios} | wc -l)
+	if (( tmp_check == 0 )); then
+	    echo "${filedef_xios} does not look like you have asked for an icesheet output file! Exiting."
+	    exit 2
+	elif (( tmp_check > 1 )); then
+	    echo "${filedef_xios} contains at least 2 icesheet output files! This is ambiguous. Exiting."
+	    exit 2
+	fi
+	
+	if (( leg_number == 1 )); then
+
+	    mkdir -p ${archive_dir}/isfcpl
+	    mkdir -p ${archive_dir}/${name_icesheet_model}
+	    rm -f ${start_dir}/isfcpl.track
+	    # Counter for the number of restart since last file exchange with the ice sheet model.
+	    cnt_res_isfcpl=0
+
+	    isfcpl_window_days=$(echo "(${n_res_perisfcpl} * ${leg_length_sec}) / 86400" | bc)
+	    isfcpl_d30=$(echo "${isfcpl_window_days} / 30" | bc -l)	    
+	    n_month_isfcpl=$(echo ${isfcpl_d30} | awk '{print int($1+0.5)}')
+	    
+	    # if (( isfcpl_window_days >= 28 && isfcpl_window_days <= 31 )); then
+	    # 	n_month_isfcpl=1
+	    # elif (( isfcpl_window_days >= 336 && isfcpl_window_days <= 372 )); then
+	    # 	n_month_isfcpl=12
+	    # else
+	    # 	echo "Can't figure out how many month long is the coupling window."
+	    # 	exit 2
+	    # fi
+	    
+	    n_month_full_run=$(( $(( 12 * $(( ${run_end_date_yyyymmdd::4} - ${run_start_date_yyyymmdd::4} )) )) + $(( ${run_end_date_yyyymmdd:4:2} - ${run_start_date_yyyymmdd:4:2} )) ))
+	    
+	    if (( $(( ${n_month_full_run} % ${n_month_isfcpl} )) != 0 )); then
+		echo "n_month_full_run=${n_month_full_run} is not a multiple of n_month_isfcpl=${n_month_isfcpl}."
+		exit 2
+	    else
+		n_isfcpl_window=$(echo "${n_month_full_run} / ${n_month_isfcpl}" | bc)
+	    fi
+
+	    tmp=`readlink -f isf_draft_meter.nc`
+	    tmp2=`readlink -f bathy_meter.nc`
+
+	    if [ "${tmp}" != "${tmp2}" ]; then
+		echo "Error. At initialization, isf_draft_meter.nc and bathy_meter.nc do not point to the same file."
+		exit 1
+	    fi
+	    
+	    init_bathy_file=`basename ${tmp}`
+	    cp -f ${tmp} ${archive_dir}/isfcpl
+	    cp -f ${tmp} ${run_dir}/init_NEMO_bathy.nc
+
+	    date_beg_isfcpl="${run_start_date_yyyymmdd}"
+	    init_geom=1
+	    init_bathy=1
+	    curr_bathy_file=${init_bathy_file}
+	    ln -sf ${run_dir}/${icesheet_init_geom_file} ${run_dir}/${name_icesheet_model}_8km_geometry_curr.nc
+
+	    
+	    printf "# Information for the ice sheet model\n" > ${start_dir}/isfcpl.track
+	    printf "run_name=${exp_name}\n" >> ${start_dir}/isfcpl.track
+	    printf "initial_date=${run_start_date_yyyymmdd}\n" >> ${start_dir}/isfcpl.track
+	    printf "n_month_isfcpl=${n_month_isfcpl}\n" >> ${start_dir}/isfcpl.track
+	    printf "n_isfcpl_window=${n_isfcpl_window}\n" >> ${start_dir}/isfcpl.track
+	    printf "date_beg_isfcpl=${date_beg_isfcpl}\n" >> ${start_dir}/isfcpl.track
+	    printf "date_end_isfcpl=YYYYMMDD\n" >> ${start_dir}/isfcpl.track
+	    printf "ice_atm_cpl=${ice_atm_cpl}\n\n" >> ${start_dir}/isfcpl.track
+	    
+	    printf "# Information for NEMO to keep up\n" >> ${start_dir}/isfcpl.track
+	    printf "cnt_res_isfcpl=0\n" >> ${start_dir}/isfcpl.track	    
+	    printf "init_geom=1\n" >> ${start_dir}/isfcpl.track
+	    printf "init_bathy=1\n" >> ${start_dir}/isfcpl.track
+	    printf "curr_bathy_file=${init_bathy_file}\n" >> ${start_dir}/isfcpl.track
+
+	    
+	    init_geom=1
+	    init_bathy=1
+	    curr_bathy_file=${init_bathy_file}
+	    ln -sf ${run_dir}/${icesheet_init_geom_file} ${run_dir}/${name_icesheet_model}_8km_geometry_curr.nc
+
+	fi # leg_number == 1
+	
+	ln -sf ${curr_bathy_file} ${run_dir}/bathy_meter.nc
+	ln -sf ${curr_bathy_file} ${run_dir}/isf_draft_meter.nc
+	
+    fi # isfcpl == 1
+    
     # Correct for leap days because NEMO standalone uses no-leap calendar
-    leg_length_sec=$(( leg_length_sec - $(leap_days "${leg_start_date}" "${leg_end_date}")*24*3600 ))
-    leg_start_sec=$(( leg_start_sec - $(leap_days "${run_start_date}" "${leg_start_date}")*24*3600 ))
-    leg_end_sec=$(( leg_end_sec - $(leap_days "${run_start_date}" "${leg_end_date}")*24*3600 ))
+    if (( leap_years == 0 )); then
+        leg_length_sec=$(( leg_length_sec - $(leap_days "${leg_start_date}" "${leg_end_date}")*24*3600 ))
+        leg_start_sec=$(( leg_start_sec - $(leap_days "${run_start_date}" "${leg_start_date}")*24*3600 ))
+        leg_end_sec=$(( leg_end_sec - $(leap_days "${run_start_date}" "${leg_end_date}")*24*3600 ))
+    fi
     (( leg_number > 1 )) && leg_is_restart=true || leg_is_restart=false
 
     #
@@ -157,24 +386,52 @@ do
         exit 0
     fi
 
+   if ${special_rs_nemo}
+   then
+    if (( leg_number == 1 )); then
+
+	dir_rs=`dirname ${special_rs_nemo_stencil}`
+	base_rs=`basename ${special_rs_nemo_stencil}`
+	    
+	n_oce_rs=`find ${dir_rs} -name "${base_rs}oce_????.nc" | wc -l`
+	if(( n_oce_rs != nem_numproc )); then
+	    echo "in special_rs_nemo, not the right amount of ocean restart files."
+	    echo "${n_oce_rs} instead of ${nem_numproc}. exiting."
+	    exit 2
+	fi
+
+	n_ice_rs=`find ${dir_rs} -name "${base_rs}ice_????.nc" | wc -l`
+	if(( n_ice_rs != nem_numproc )); then
+	    echo "in special_rs_nemo, not the right amount of ice restart files."
+	    echo "${n_ice_rs} instead of ${nem_numproc}. exiting."
+	    exit 2
+	fi
+	    
+        for (( n=0 ; n<nem_numproc ; n++ ))
+        do
+	    np=$(printf %04d ${n})
+	    ln -fs ${special_rs_nemo_stencil}oce_${np}.nc restart_oce_${np}.nc
+	    ln -fs ${special_rs_nemo_stencil}ice_${np}.nc restart_ice_${np}.nc
+        done
+    fi # leg_number == 1
+   fi # if $special_rs_nemo
+
     #
     # Update namelist
     #
     source build_namelist_cfg.sh > namelist_cfg
-
-    #
+    
     # Link the restart files
-    #
     ns=$(printf %08d $(( leg_start_sec / nem_time_step_sec - nem_restart_offset )))
     if ((leg_start_sec > 0 )); then
-    for (( n=0 ; n<nem_numproc ; n++ ))
-    do
-        np=$(printf %04d ${n})
-        [[ -f "${exp_name:?}_${ns}_restart_oce_${np}.nc" ]] || { echo "Error: restart file not found." ; exit 2 ; }
-        ln -fs "${exp_name:?}_${ns}_restart_oce_${np}.nc" "restart_oce_${np}.nc"
-        [[ -f "${exp_name:?}_${ns}_restart_ice_${np}.nc" ]] || { echo "Error: restart file not found." ; exit 2 ; }
-        ln -fs "${exp_name:?}_${ns}_restart_ice_${np}.nc" "restart_ice_${np}.nc"
-    done
+    	for (( n=0 ; n<nem_numproc ; n++ ))
+    	do
+            np=$(printf %04d ${n})
+            [[ -f "${exp_name:?}_${ns}_restart_oce_${np}.nc" ]] || { echo "Error: restart file not found." ; exit 2 ; }
+            ln -fs "${exp_name:?}_${ns}_restart_oce_${np}.nc" "restart_oce_${np}.nc"
+            [[ -f "${exp_name:?}_${ns}_restart_ice_${np}.nc" ]] || { echo "Error: restart file not found." ; exit 2 ; }
+            ln -fs "${exp_name:?}_${ns}_restart_ice_${np}.nc" "restart_ice_${np}.nc"
+    	done
     fi
 
     # Stop here is preponly was specified
@@ -183,11 +440,171 @@ do
     #
     # Run nemo
     #
+
+    if [[ "$@" == *"dbg"* ]]; then
+	echo $HOSTNAME > ${start_dir}/log
+	sleep 10000000000000
+    fi
+    
     time_begin=$(date +%s)
     mpirun -np "${xio_numproc:?}" "${xio_exe_file:?}" : -np "${nem_numproc:?}" "${nem_exe_file:?}"
     time_end=$(date +%s)
 
-#############################
+    echo "dbg cp made it past mpirun"
+    
+    if (( isfcpl == 1 )); then
+
+	# NEMO restart index within ice sheet / ocean coupling window
+	cnt_res_isfcpl=$(( $(( ${leg_number} - 1 )) % ${n_res_perisfcpl} ))
+	sed -i "s/.*cnt_res_isfcpl=.*/cnt_res_isfcpl=${cnt_res_isfcpl}/" ${start_dir}/isfcpl.track
+
+	
+	# module purge
+	# module load CDO
+	fmt_res_isfcpl=$(printf "%02d" ${cnt_res_isfcpl})
+	
+	in_stencil="${exp_name}_1m_${leg_start_date_yyyymmdd}_${leg_almostend_date_yyyymmdd}_icesheet"
+	out_stencil="${exp_name}_icesheet_res${fmt_res_isfcpl}"
+
+	if (( is_xios2 == 0 )); then
+	    for((prc=0;prc<xio_numproc;prc++)); do
+		fmt_xio=$(printf "%04d" ${prc})
+		infile="${in_stencil}_${fmt_xio}.nc"
+		mv -f ${infile} ${out_stencil}_${fmt_xio}.nc
+	    done
+
+	    module purge
+	    module load ${module_list:?}
+	    export LD_LIBRARY_PATH=${LD_LIBRARY_PATH:+${LD_LIBRARY_PATH}:}"${extralibs_list}"
+
+	    echo "dbg cp before rebuild"
+	
+	    /bin/bash ${rebuild_nemo_exe} ${out_stencil} ${xio_numproc}
+	    rm -f ${out_stencil}_????.nc
+
+	    echo "dbg cp after rebuild"
+
+	else
+
+	    mv -f ${in_stencil}.nc ${out_stencil}.nc
+	    
+	fi
+	
+
+	if (( ${leg_number} % ${n_res_perisfcpl} == 0 )); then
+	    # We have just ended one NEMO leg which closes an ice sheet / ocean coupling window.
+	    date_end_isfcpl=${leg_almostend_date_yyyymmdd}
+	    sed -i "s/.*date_end_isfcpl=.*/date_end_isfcpl=${date_end_isfcpl}/" ${start_dir}/isfcpl.track
+
+	    /bin/bash ${rebuild_nemo_exe} mesh_mask ${nem_numproc}
+
+	    module purge
+	    ml_pp_ice
+
+	    # Concatenate all icesheet files from all legs contained within this coupling window
+	    tmpfile_eos80="curr_ocean2ice_EOS80.nc"
+	    tmpfile_teos10="curr_ocean2ice_TEOS10.nc"
+	    outfile="${exp_name}_ocean2ice_${date_beg_isfcpl}-${date_end_isfcpl}.nc"
+	    
+	    cdo -O copy ${out_stencil::-2}??.nc ${tmpfile_teos10}
+	    rm -f ${out_stencil::-2}??.nc
+	    
+	    
+	    module purge
+	    ml_pp_ice
+	    ${pp_nemo_to_icesheet_call}
+
+	    mv -f ${tmpfile_eos80} ${outfile}
+	    mv -f mesh_mask.nc ${archive_dir}/isfcpl/${exp_name}_mesh_mask_${date_beg_isfcpl}-${date_end_isfcpl}.nc
+	    
+	    # CALL ICE SHEET MODEL
+            # echo "dbg cp before icesheet modules"
+	    # ${icesheet_modules}
+            # echo "dbg cp after icesheet modules"
+            # module list
+	    ml_icesheet
+	    ${icesheet_call}
+
+	    last_toto_date=${date_end_isfcpl::-2}
+
+	    # move all toto files to archive
+	    find . -type f -name "${exp_name}_${name_icesheet_model}Run_??????_toto.mat" -exec mv -f {} ${archive_dir}/${name_icesheet_model} \;
+	    # remove all old symbolic links to toto files
+	    find . -type l -name "${exp_name}_${name_icesheet_model}Run_??????_toto.mat" -exec rm -f {}  \;
+	    # symbolic link to the new toto file
+	    ln -sf ${archive_dir}/${name_icesheet_model}/${exp_name}_${name_icesheet_model}Run_${last_toto_date}_toto.mat ./
+	    module purge
+
+	    # mv ocean2ice file to arch dir
+	    mv -f ${outfile} ${archive_dir}/isfcpl
+	    
+	    new_geom_file="${exp_name}_ice2ocean_${date_end_isfcpl}.nc"
+
+	    # old_geom=`readlink -f ${name_icesheet_model}_8km_geometry_curr.nc`
+	    # if [ "${init_geom}" == 0 ]; then
+	    # 	mv -f ${old_geom} ${archive_dir}/isfcpl
+	    # else
+	    # 	init_geom=0
+	    # 	sed -i "s/.*init_geom=.*/init_geom=0/" ${start_dir}/isfcpl.track
+	    # 	cp -f ${old_geom} ${archive_dir}/isfcpl
+	    # fi
+
+	    ln -sf ${new_geom_file} ${name_icesheet_model}_8km_geometry_curr.nc
+	    
+	    # We need to pre-treat fETISh outputs in order to make them readable by NEMO.
+	    # new fETISh output should be ${run_dir}/fETISh_8km_geometry_curr.nc
+	    module purge
+	    ml_pp_ice
+	    ${pp_icesheet_to_nemo_call}
+
+	    # Copy (for initial) or move (non initial) previous bathymetry file to archive and link new one to what NEMO will read.
+	    new_bathy_file="${name_icesheet_model}_eORCA025-SO_bathymetry_${date_end_isfcpl}.nc"
+	    mv -f ${name_icesheet_model}_new_8km_eORCA025-bathymetry_curr.nc ${new_bathy_file}
+	    mv -f ${new_geom_file} ${archive_dir}/isfcpl
+		
+	    old_bathy=`readlink -f ${run_dir}/bathy_meter.nc`
+	    if [ "${init_bathy}" == 0 ]; then
+		mv -f ${old_bathy} ${archive_dir}/isfcpl
+	    else
+		init_bathy=0
+		sed -i "s/^init_bathy=.*$/init_bathy=0/g" ${start_dir}/isfcpl.track
+		cp -f ${old_bathy} ${archive_dir}/isfcpl
+	    fi
+	    curr_bathy_file=${new_bathy_file}
+	    sed -i "s/.*curr_bathy_file=.*/curr_bathy_file=${curr_bathy_file}/" ${start_dir}/isfcpl.track
+	    
+	    ln -sf ${new_bathy_file} ${run_dir}/bathy_meter.nc
+	    ln -sf ${new_bathy_file} ${run_dir}/isf_draft_meter.nc
+		
+	    module purge
+	    module load ${module_list:?}
+	    export LD_LIBRARY_PATH=${LD_LIBRARY_PATH:+${LD_LIBRARY_PATH}:}"${extralibs_list}"
+
+	    
+	    printf "# Information for the ice sheet model\n" > ${start_dir}/isfcpl.track
+	    printf "run_name=${exp_name}\n" >> ${start_dir}/isfcpl.track
+	    printf "initial_date=${run_start_date_yyyymmdd}\n" >> ${start_dir}/isfcpl.track
+	    printf "n_month_isfcpl=${n_month_isfcpl}\n" >> ${start_dir}/isfcpl.track
+	    printf "n_isfcpl_window=${n_isfcpl_window}\n" >> ${start_dir}/isfcpl.track
+	    printf "date_beg_isfcpl=${nextleg_start_date_yyyymmdd}\n" >> ${start_dir}/isfcpl.track
+	    printf "date_end_isfcpl=YYYYMMDD\n" >> ${start_dir}/isfcpl.track
+	    printf "ice_atm_cpl=${ice_atm_cpl}\n\n" >> ${start_dir}/isfcpl.track
+	    
+	    printf "# Information for NEMO to keep up\n" >> ${start_dir}/isfcpl.track	    
+	    printf "cnt_res_isfcpl=0\n" >> ${start_dir}/isfcpl.track
+	    printf "init_geom=${init_geom}\n" >> ${start_dir}/isfcpl.track
+	    printf "init_bathy=${init_bathy}\n" >> ${start_dir}/isfcpl.track
+	    printf "curr_bathy_file=${curr_bathy_file}\n" >> ${start_dir}/isfcpl.track
+	    
+	    module purge
+	    module load ${module_list:?}
+	    export LD_LIBRARY_PATH=${LD_LIBRARY_PATH:+${LD_LIBRARY_PATH}:}"${extralibs_list}"
+	    
+	fi
+    fi
+    
+    #############################
+
 
     formatted_leg_number=$(printf %03d $((leg_number)))
     #
@@ -197,15 +614,20 @@ do
     mkdir -p "${outdir}"
 
     shopt -s nullglob
-    for v in grid_U grid_V grid_W grid_T icemod icemoa SBC SBC_scalar scalar
+    for v in grid_U grid_V grid_W grid_T icemod icemod_U icemod_V icemoa SBC SBC_scalar scalar fwfisf
     do
         for f in ${exp_name}_??_????????_????????_${v}_????.nc
         do
-             mv "$f" "$outdir/"
+            mv "$f" "$outdir/"
+        done
+        for f in ${exp_name}_???_????????_????????_${v}_????.nc
+        do
+            mv "$f" "$outdir/"
         done
+ 
         for f in ${exp_name}_??_????????_????????_${v}.nc
         do
-             mv "$f" "$outdir/"
+            mv "$f" "$outdir/"
         done
         for f in ${exp_name}_??_${v}.nc
         do
@@ -216,13 +638,13 @@ do
     #
     # Move NEMO restart files from previous run to archive directory
     #
-        outdir="$archive_dir/restart/${formatted_leg_number}"
-        mkdir -p "${outdir}"
+    outdir="$archive_dir/restart/${formatted_leg_number}"
+    mkdir -p "${outdir}"
 
-        for f in ${exp_name}_${ns}_restart_???_????.nc
-        do
-               [ -f "$f" ] && mv "$f" "${outdir}"
-        done
+    for f in ${exp_name}_${ns}_restart_???_????.nc
+    do
+        [ -f "$f" ] && mv "$f" "${outdir}"
+    done
 
     #
     # Move log files to archive directory
@@ -230,12 +652,11 @@ do
     outdir="$archive_dir/log/${formatted_leg_number}"
     mkdir -p "${outdir}"
     for f in ocean.output time.step solver.stat ; do mv "${f}" "${outdir}"; done
-#
-#
-##############################
+    
+    
+    
 
     shopt -u nullglob
-
     # 
     # Write checkpoint control file
     # TODO: enquiry why 0 -1 +2 rather than 2- 1
@@ -243,19 +664,23 @@ do
     tr=$(date -d "0 -$time_begin sec + $time_end sec" +%T) 
     current_date=$(date +'%F %T')
     {
-      echo "#"
-      echo "# Finished leg at ${current_date} after ${tr} (hh:mm:ss)" 
-      echo "leg_number=${leg_number}"
-      echo "leg_start_date=\"${leg_start_date}\""
-      echo "leg_end_date=\"${leg_end_date}\""
+	echo "#"
+	echo "# Finished leg at ${current_date} after ${tr} (hh:mm:ss)" 
+	echo "leg_number=${leg_number}"
+	echo "leg_start_date=\"${leg_start_date}\""
+	echo "leg_end_date=\"${leg_end_date}\""
     } | tee -a "${info_file}"
 
     special_restart=false
+
 done
+
 #
 # Move back to submission directory
 #
 cd - >/dev/null
+
+
 #
 # Check whether there is some work left to do in a further job
 #
@@ -263,9 +688,14 @@ cd - >/dev/null
 if (( leg_end_epoch < run_end_epoch )) ; then
     echo "Leg end earlier than end of simulation."
     echo "Submitting another job."
+    
     #[[ $@ == *local* ]] && exec $0 $@ || scontrol requeue $SLURM_JOB_ID #TODO: factorize this
     if [[ "$@" == *"local"* ]] ; then
-        exec "$0" "$@"
+	cd ${start_dir}
+	sleep 2
+	./run.sh local && exit 0
+
+	#exec "$0" "$@"
     elif [ "${submit_command}" == *sbatch* ] ; then
         sbatch $0 $@ | awk '{print $4}' >> ${run_dir}/.coral_jobs
     elif [[ $USER == "vsc"* ]]; then