First isfcpl commit.

templates/genius/experiment.cfg

@@ -4,13 +4,18 @@ exp_name=EXP_REF
 # Simulation start and end date. Use any (reasonable) syntax you want.
 run_start_date="1958-01-01"
 run_duration="4 years"
+leap_years=1
 
 # Restart frequency. Use any (reasonable) number and time unit you want.
 # For runs without restart, leave this variable empty or set to 0
 rst_freq="1 year"
 
 # Number of restart legs to be run in one go
-run_num_legs=2
+run_num_legs=4
+# Including (1) ice shelf coupling or not (0)
+isfcpl=1
+# Ice shelf coupling is performed every n_res_isfcpl NEMO restart legs
+n_res_perisfcpl=1
 
 # Special restart
 special_restart=false

templates/genius/namelists/build_namelist_cfg.sh

@@ -31,6 +31,10 @@ cat << EOF
    cn_ocerst_out = "restart_oce"  !  suffix of ocean restart name (output)
    nn_stock      =  0             !  frequency of creation of a restart file (modulo referenced to 1)
    nn_write      = -1             !  frequency of write in the output file   (modulo referenced to nn_it000)
+   ln_iscpl = ${iscpl_nm}
+   ln_clobber = .true.
+   nn_istate = 0
+   nn_leapy  = ${leap_years}
 /
 !-----------------------------------------------------------------------
 &namcfg        !   parameters of the configuration
@@ -56,7 +60,7 @@ cat << EOF
 !-----------------------------------------------------------------------
 &namdom        !   space and time domain (bathymetry, mesh, timestep)
 !-----------------------------------------------------------------------
-   nn_msh      =    0                  !  create (=1) a mesh file or not (=0)
+   nn_msh      =    ${isfcpl}                  !  create (=1) a mesh file or not (=0)
    rn_hmin     =    20.                !  min depth of the ocean (>0) or min number of ocean level (<0)
    rn_rdt      = ${nem_time_step_sec} !  time step for the dynamics (and tracer if nn_acc=0)
    ppglam0     =  999999.0             !  longitude of first raw and column T-point (jphgr_msh = 1)
@@ -190,6 +194,13 @@ cat << EOF
    sn_depmin_isf = 'clipped_runoff-icb_DaiTrenberth_Depoorter_eORCA1_JD.nc' ,       -12         ,'sodepmin_isf',    .false.     , .true.  , 'yearly'  ,  ''      ,   ''
 /
 !-----------------------------------------------------------------------
+&namsbc_iscpl  !   land ice / ocean coupling option
+!-----------------------------------------------------------------------
+   nn_drown  = 100       ! number of iteration of the extrapolation loop (fill the new wet cells)
+   ln_hsb    = .false.  ! activate conservation module (conservation exact after a time of rn_fiscpl)
+   nn_fiscpl = 43800    ! (number of time step) conservation period (maybe should be fix to the coupling frequencey of restart frequency)
+/
+!-----------------------------------------------------------------------
 &namsbc_apr    !   Atmospheric pressure used as ocean forcing or in bulk
 !-----------------------------------------------------------------------
 /

templates/genius/programs.cfg

@@ -18,3 +18,12 @@ nem_numproc=32
 xio_exe_file=${nemo_src_dir}/EXTERNAL/xios-1.0/bin/xios_server.exe
 xio_numproc=4
 
+# -----------------------------------------------------------------------------
+# *** fETISh configuration
+# -----------------------------------------------------------------------------
+
+fetish_src_dir=${nemo_src_dir}/EXTERNAL/isfcpl_scripts
+fetish_call="matlab -nodesktop -nojvm -r fETISh_Coupling_NEMO > toto"
+pp_fetish_call="python pp_fetish_toclust_nomovfront.py"
+fetish_modules="matlab/R2019a"
+pp_fetish_modules="mlpy; module load CDO"

templates/genius/xios_config/file_def.xml

@@ -158,6 +158,12 @@
       <field field_ref="snowthic_cat"   name="snthicat" />
     </file>
 
+
+    <file id="file9" name_suffix="_icesheet" >
+      <field field_ref="fwfisf"    name="fwfisf"   />
+    </file>
+
+
   </file_group>
 
   <file_group id="2m" output_freq="2mo" output_level="10" enabled=".TRUE."/> <!-- real 2m files -->

templates/scripts/skeleton.sh

@@ -13,9 +13,9 @@ function leap_days()
 
     # Check first year for leap day between start and end date
     $(date -ud "${frstYYYY}-02-29" > /dev/null 2>&1) \
-    && (( $(date -ud "$1" +%s) < $(date -ud "${frstYYYY}-03-01" +%s) )) \
-    && (( $(date -ud "$2" +%s) > $(date -ud "${lastYYYY}-02-28" +%s) )) \
-    && (( ld++ ))
+	&& (( $(date -ud "$1" +%s) < $(date -ud "${frstYYYY}-03-01" +%s) )) \
+	&& (( $(date -ud "$2" +%s) > $(date -ud "${lastYYYY}-02-28" +%s) )) \
+	&& (( ld++ ))
 
     # Check intermediate years for leap day
     for (( y=(( ${frstYYYY}+1 )); y<=(( ${lastYYYY}-1 )); y++ ))
@@ -26,10 +26,10 @@ function leap_days()
     # Check last year (if different from first year) for leap day between start
     # and end date
     (( $lastYYYY > $frstYYYY )) \
-    && $(date -ud "${lastYYYY}-02-29" > /dev/null 2>&1) \
-    && (( $(date -ud "$1" +%s) < $(date -ud "${frstYYYY}-03-01" +%s) )) \
-    && (( $(date -ud "$2" +%s) > $(date -ud "${lastYYYY}-02-28" +%s) )) \
-    && (( ld++ ))
+	&& $(date -ud "${lastYYYY}-02-29" > /dev/null 2>&1) \
+	&& (( $(date -ud "$1" +%s) < $(date -ud "${frstYYYY}-03-01" +%s) )) \
+	&& (( $(date -ud "$2" +%s) > $(date -ud "${lastYYYY}-02-28" +%s) )) \
+	&& (( ld++ ))
 
     set -e
 
@@ -37,9 +37,27 @@ function leap_days()
 }
 
 
+function mlpy () {
+
+    if [[ `hostname` == "ozone" ]]; then
+	module purge
+	module load Python
+	PYTHONPATH="/elic/home/pelletie/.local/lib/python3.6/site-packages:${PYTHONPATH}"
+    elif [[ ${USER} == "vsc"????? ]]; then
+	module purge
+	module load Python
+	PYTHONPATH="/user/leuven/327/vsc32749/.local/bin:${PYTHONPATH}"
+    else
+	echo "Nothing detected ${USER}" > ${start_dir}/tmpout
+	exit 0
+    fi
+}
+
+
 
 [[ $@ == *verbose* ]] && set -x
 
+module purge
 module load ${module_list:?}
 export LD_LIBRARY_PATH=${LD_LIBRARY_PATH:+${LD_LIBRARY_PATH}:}"${extralibs_list}"
 
@@ -51,37 +69,37 @@ export LD_LIBRARY_PATH=${LD_LIBRARY_PATH:+${LD_LIBRARY_PATH}:}"${extralibs_list}
 #
 if [ ! -d ${run_dir:?} ]
 then
-     mkdir -p ${run_dir}
-     if $special_restart
-     then
-          rsync -av --delete ${run_dir}/../${special_restart_from}/ --exclude log --exclude output --exclude restart --exclude="${special_restart_from}_*" --exclude="ocean*" --exclude="restart_*" --exclude="debug.*" --exclude="output.*" ${run_dir}
-          cp -f ${nem_exe_file} ${run_dir}
-          cp -f ${xio_exe_file} ${run_dir}
-          special_year=${special_restart_date:0:4}
-          sed -i "/$special_year/q" ${run_dir}/${info_file}
-          . ${run_dir}/${info_file}
-          special_restart_leg=$(printf %03d $((leg_number+1)))
-          # PUT HERE THE INSTRUCTIONS TO COPY THE restart files
-          cd ${run_dir}/../../archive/${special_restart_from}/restart/${special_restart_leg}
-          for f in *.nc; do
-              nf=${exp_name}${f:4}
-              cp $f ${run_dir}/$nf
-          done
-          cd -
-          cd ${run_dir}
-          for f in ${exp_name}_????????_restart_???_????.nc; do
-              nf=${f:14}
-              ln -s $f $nf
-          done
-          cd -
-     fi
+    mkdir -p ${run_dir}
+    if $special_restart
+    then
+        rsync -av --delete ${run_dir}/../${special_restart_from}/ --exclude log --exclude output --exclude restart --exclude="${special_restart_from}_*" --exclude="ocean*" --exclude="restart_*" --exclude="debug.*" --exclude="output.*" ${run_dir}
+        cp -f ${nem_exe_file} ${run_dir}
+        cp -f ${xio_exe_file} ${run_dir}
+        special_year=${special_restart_date:0:4}
+        sed -i "/$special_year/q" ${run_dir}/${info_file}
+        . ${run_dir}/${info_file}
+        special_restart_leg=$(printf %03d $((leg_number+1)))
+        # PUT HERE THE INSTRUCTIONS TO COPY THE restart files
+        cd ${run_dir}/../../archive/${special_restart_from}/restart/${special_restart_leg}
+        for f in *.nc; do
+            nf=${exp_name}${f:4}
+            cp $f ${run_dir}/$nf
+        done
+        cd -
+        cd ${run_dir}
+        for f in ${exp_name}_????????_restart_???_????.nc; do
+            nf=${f:14}
+            ln -s $f $nf
+        done
+        cd -
+    fi
 fi
 cd ${start_dir}
 cp -u xios_config/*xml "${run_dir}"
 cp -u namelists/* "${run_dir}"
 cd "${run_dir}"
-cp -u "${nem_exe_file:?}" .
-cp -u "${xio_exe_file:?}" .
+# cp -u "${nem_exe_file:?}" .
+# cp -u "${xio_exe_file:?}" .
 
 # Write fake file for previous fresh water budget adjustment (nn_fwb==2 in namelist)
 [[ ! -f EMPave_old.dat ]] && echo "                               0  0.0000000000000000E+00  0.0000000000000000E+00" > EMPave_old.dat
@@ -102,6 +120,22 @@ for file in "${shared_files[@]}"; do
 done
 
 
+if [ "${isfcpl}" == "1" ]; then
+
+    length_cut_pref=$(( ${#exp_name} + 4 ))
+    rebuild_nemo_exe=${nemo_src_dir}/TOOLS/REBUILD_NEMO/rebuild_nemo
+    if [ -f ${start_dir}/tmp/isfcpl.track ]; then
+	source ${start_dir}/tmp/isfcpl.track
+	# printf "\n" >> isfcpl.track
+	# rm -f isfcpl.track
+    fi
+    
+    cp -f ${fetish_src_dir}/* ${run_dir}
+    iscpl_nm=.true.
+else
+    iscpl_nm=.false.
+fi
+
 #
 # Normalize date formats
 #
@@ -112,6 +146,9 @@ run_end_date=$(date -uR -d "${run_end_date}")
 run_start_epoch=$(date -u -d"${run_start_date}" +%s)
 run_end_epoch=$(date -u -d"${run_end_date}" +%s)
 
+
+
+
 for (( ; run_num_legs>0 ; run_num_legs-- ))
 do
 
@@ -129,11 +166,151 @@ do
     leg_length_sec=$(( leg_end_epoch - leg_start_epoch ))
     leg_start_sec=$(( leg_start_epoch - run_start_epoch ))
     leg_end_sec=$(( leg_end_epoch - run_start_epoch ))
+
+    leg_beforestart_epoch=$(( ${leg_start_epoch} - 43200 ))
+    leg_beforestart_date=$(date -uR -d@"${leg_beforestart_epoch}")
+    
+    leg_almostend_epoch=$(( ${leg_end_epoch} - 43200 ))
+    leg_almostend_date=$(date -uR -d@"${leg_almostend_epoch}")
+
+    run_start_date_yyyymmdd=$(date -u -d "${run_start_date}" +%Y%m%d) # FIXME appears unused
+    run_end_date_yyyymmdd=$(date -u -d "${run_end_date}" +%Y%m%d) # FIXME appears unused
     leg_start_date_yyyymmdd=$(date -u -d "${leg_start_date}" +%Y%m%d) # FIXME appears unused
+    leg_almostend_date_yyyymmdd=$(date -u -d "${leg_almostend_date}" +%Y%m%d) # FIXME appears unused
+    leg_beforestart_date_yyyymmdd=$(date -u -d "${leg_beforestart_date}" +%Y%m%d) # FIXME appears unused
+
+    # run_start_date_yyyymm="${run_start_date_yyyymmdd::-2}"
+    # run_end_date_yyyymm="${run_end_date_yyyymmdd::-2}"
+    # leg_start_date_yyyymm="${leg_start_date_yyyymmdd::-2}"
+    # leg_almostend_date_yyyymm="${leg_almostend_date_yyyymmdd::-2}"
+    # leg_beforestart_date_yyyymm="${leg_beforestart_date_yyyymmdd::-2}"
+
+    
+    if [ "${isfcpl}" == "1" ]; then
+	
+	# Check if a monthly "icesheet" output file has indeed been asked.
+	tmp_check=$(grep -nri "file id=\"file.*\".*name_suffix=\"_icesheet\"" ${run_dir}/file_def.xml | wc -l)
+	if (( tmp_check == 0 )); then
+	    echo "file_def.xml does not look like you have asked for an icesheet output file! Exiting."
+	    exit 2
+	elif (( tmp_check > 1 )); then
+	    echo "file_def.xml contains at least 2 icesheet output files! This is ambiguous. Exiting."
+	    exit 2
+	fi
+	
+	if (( leg_number == 1 )); then
+
+	    mkdir -p ${archive_dir}/isfcpl
+	    rm -f ${start_dir}/tmp/isfcpl.track
+	    # Counter for the number of restart since last file exchange with fETISh.
+	    cnt_res_isfcpl=0
+
+	    isfcpl_window_days=$(echo "(${n_res_perisfcpl} * ${leg_length_sec}) / 86400" | bc)
+
+	    if (( isfcpl_window_days >= 28 && isfcpl_window_days <= 31 )); then
+		n_month_isfcpl=1
+	    elif (( isfcpl_window_days >= 336 && isfcpl_window_days <= 372 )); then
+		n_month_isfcpl=12
+	    else
+		echo "Can't figure out how many month long is the coupling window."
+		exit 2
+	    fi
+	    
+	    n_month_full_run=$(( $(( 12 * $(( ${run_end_date_yyyymmdd::4} - ${run_start_date_yyyymmdd::4} )) )) + $(( ${run_end_date_yyyymmdd:4:2} - ${run_start_date_yyyymmdd:4:2} )) ))
+	    
+	    if (( $(( ${n_month_full_run} % ${n_month_isfcpl} )) != 0 )); then
+		echo "n_month_full_run=${n_month_full_run} is not a multiple of n_month_isfcpl=${n_month_isfcpl}."
+		exit 2
+	    else
+		n_isfcpl_window=$(echo "${n_month_full_run} / ${n_month_isfcpl}" | bc)
+	    fi
+
+	    tmp=`readlink -f isf_draft_meter.nc`
+	    tmp2=`readlink -f bathy_meter.nc`
+
+	    if [ "${tmp}" != "${tmp2}" ]; then
+		echo "Error. At initialization, isf_draft_meter.nc and bathy_meter.nc do not point to the same file."
+		exit 1
+	    fi
+	    
+	    init_bathy_file=`basename ${tmp}`
+	    cp -f ${tmp} ${archive_dir}/isfcpl
+	    cp -f ${tmp} ${run_dir}/init_NEMO_bathy.nc
+	    
+	    printf "# Information for the ice sheet model\n" > ${start_dir}/tmp/isfcpl.track
+	    printf "run_name=${exp_name}\n" >> ${start_dir}/tmp/isfcpl.track
+	    printf "initial_date=${run_start_date_yyyymmdd}\n" >> ${start_dir}/tmp/isfcpl.track
+	    printf "n_month_isfcpl=${n_month_isfcpl}\n" >> ${start_dir}/tmp/isfcpl.track
+	    printf "n_isfcpl_window=${n_isfcpl_window}\n" >> ${start_dir}/tmp/isfcpl.track
+	    printf "date_beg_isfcpl=YYYYMMDD\n" >> ${start_dir}/tmp/isfcpl.track
+	    printf "date_end_isfcpl=YYYYMMDD\n\n" >> ${start_dir}/tmp/isfcpl.track
+
+	    printf "# Information for NEMO to keep up\n" >> ${start_dir}/tmp/isfcpl.track
+	    printf "cnt_res_isfcpl=0\n" >> ${start_dir}/tmp/isfcpl.track	    
+	    printf "init_geom=1\n" >> ${start_dir}/tmp/isfcpl.track
+	    printf "init_bathy=1\n" >> ${start_dir}/tmp/isfcpl.track
+	    printf "curr_bathy_file=${init_bathy_file}\n" >> ${start_dir}/tmp/isfcpl.track
+	    
+	    init_geom=1
+	    init_bathy=1
+	    
+	    curr_bathy_file=${init_bathy_file}
+	    
+	    ln -sf ${run_dir}/fETISh_8km_geometry_init.nc ${run_dir}/fETISh_8km_geometry_curr.nc
+
+	fi
+	
+	ln -sf ${curr_bathy_file} ${run_dir}/bathy_meter.nc
+	ln -sf ${curr_bathy_file} ${run_dir}/isf_draft_meter.nc
+	
+	if (( $(( ${leg_number} - 1 )) % ${n_res_perisfcpl} == 0 )); then
+	    
+	    date_beg_isfcpl=${leg_start_date_yyyymmdd}
+	    sed -i "s/.*date_beg_isfcpl=.*/date_beg_isfcpl=${date_beg_isfcpl}/" ${start_dir}/tmp/isfcpl.track
+	    
+	    if (( leg_number > 1 )); then
+		# leg number at which NEMO is supposed to receive updated geometry from fETISh.
+		# We need to pre-treat fETISh outputs in order to make them readable by NEMO.
+
+		# new fETISh output should be ${run_dir}/fETISh_8km_geometry_curr.nc
+		module purge
+		mlpy
+		module load CDO
+		${pp_fetish_call}
+
+		# Copy (for initial) or move (non initial) previous bathymetry file to archive and link new one to what NEMO will read.
+		new_bathy_file="fETISh_eORCA025-SO_bathymetry_${leg_beforestart_date_yyyymmdd}.nc"
+		mv -f fETISh_new_8km_eORCA025-bathymetry_curr.nc ${new_bathy_file}
+		
+		old_bathy=`readlink -f ${run_dir}/bathy_meter.nc`
+		if [ "${init_bathy}" == 0 ]; then
+		    mv -f ${old_bathy} ${archive_dir}/isfcpl
+		else
+		    init_bathy=0
+		    sed -i "s/^init_bathy=.*$/init_bathy=0/g" ${start_dir}/tmp/isfcpl.track
+		    cp -f ${old_bathy} ${archive_dir}/isfcpl
+		fi
+		curr_bathy_file=${new_bathy_file}
+		sed -i "s/.*curr_bathy_file=.*/curr_bathy_file=${curr_bathy_file}/" ${start_dir}/tmp/isfcpl.track
+		ln -sf ${new_bathy_file} ${run_dir}/bathy_meter.nc
+		ln -sf ${new_bathy_file} ${run_dir}/isf_draft_meter.nc
+		
+		module purge
+		module load ${module_list:?}
+		export LD_LIBRARY_PATH=${LD_LIBRARY_PATH:+${LD_LIBRARY_PATH}:}"${extralibs_list}"
+		
+	    fi
+	fi
+	
+    fi
+
+    
     # Correct for leap days because NEMO standalone uses no-leap calendar
-    leg_length_sec=$(( leg_length_sec - $(leap_days "${leg_start_date}" "${leg_end_date}")*24*3600 ))
-    leg_start_sec=$(( leg_start_sec - $(leap_days "${run_start_date}" "${leg_start_date}")*24*3600 ))
-    leg_end_sec=$(( leg_end_sec - $(leap_days "${run_start_date}" "${leg_end_date}")*24*3600 ))
+    if (( leap_years == 0 )); then
+        leg_length_sec=$(( leg_length_sec - $(leap_days "${leg_start_date}" "${leg_end_date}")*24*3600 ))
+        leg_start_sec=$(( leg_start_sec - $(leap_days "${run_start_date}" "${leg_start_date}")*24*3600 ))
+        leg_end_sec=$(( leg_end_sec - $(leap_days "${run_start_date}" "${leg_end_date}")*24*3600 ))
+    fi
     (( leg_number > 1 )) && leg_is_restart=true || leg_is_restart=false
 
     #
@@ -161,20 +338,18 @@ do
     # Update namelist
     #
     source build_namelist_cfg.sh > namelist_cfg
-
-    #
+    
     # Link the restart files
-    #
     ns=$(printf %08d $(( leg_start_sec / nem_time_step_sec - nem_restart_offset )))
     if ((leg_start_sec > 0 )); then
-    for (( n=0 ; n<nem_numproc ; n++ ))
-    do
-        np=$(printf %04d ${n})
-        [[ -f "${exp_name:?}_${ns}_restart_oce_${np}.nc" ]] || { echo "Error: restart file not found." ; exit 2 ; }
-        ln -fs "${exp_name:?}_${ns}_restart_oce_${np}.nc" "restart_oce_${np}.nc"
-        [[ -f "${exp_name:?}_${ns}_restart_ice_${np}.nc" ]] || { echo "Error: restart file not found." ; exit 2 ; }
-        ln -fs "${exp_name:?}_${ns}_restart_ice_${np}.nc" "restart_ice_${np}.nc"
-    done
+    	for (( n=0 ; n<nem_numproc ; n++ ))
+    	do
+            np=$(printf %04d ${n})
+            [[ -f "${exp_name:?}_${ns}_restart_oce_${np}.nc" ]] || { echo "Error: restart file not found." ; exit 2 ; }
+            ln -fs "${exp_name:?}_${ns}_restart_oce_${np}.nc" "restart_oce_${np}.nc"
+            [[ -f "${exp_name:?}_${ns}_restart_ice_${np}.nc" ]] || { echo "Error: restart file not found." ; exit 2 ; }
+            ln -fs "${exp_name:?}_${ns}_restart_ice_${np}.nc" "restart_ice_${np}.nc"
+    	done
     fi
 
     # Stop here is preponly was specified
@@ -183,11 +358,113 @@ do
     #
     # Run nemo
     #
+
+    if [[ "$@" == *"dbg"* ]]; then
+	echo $HOSTNAME
+	sleep 10000000000000
+    fi
+    
     time_begin=$(date +%s)
     mpirun -np "${xio_numproc:?}" "${xio_exe_file:?}" : -np "${nem_numproc:?}" "${nem_exe_file:?}"
     time_end=$(date +%s)
 
-#############################
+    if (( isfcpl == 1 )); then
+
+	module purge
+	module load CDO
+	fmt_res_isfcpl=$(printf "%02d" ${cnt_res_isfcpl})
+	
+	in_stencil="${exp_name}_1m_${leg_start_date_yyyymmdd}_${leg_almostend_date_yyyymmdd}_icesheet"
+	out_stencil="${exp_name}_icesheet_res${fmt_res_isfcpl}"
+
+	# Renaming.
+	for((prc=0;prc<xio_numproc;prc++)); do
+	    fmt_xio=$(printf "%04d" ${prc})
+	    infile="${in_stencil}_${fmt_xio}.nc"
+	    # echo "mv -f ${infile} ./${out_stencil}_${fmt_xio}.nc"
+	    mv -f ${infile} ${out_stencil}_${fmt_xio}.nc
+	done
+
+	# Rebuilding it.
+	module purge
+	module load ${module_list:?}
+	export LD_LIBRARY_PATH=${LD_LIBRARY_PATH:+${LD_LIBRARY_PATH}:}"${extralibs_list}"
+
+	${rebuild_nemo_exe} ${out_stencil} ${xio_numproc}
+	rm -f ${out_stencil}_????.nc
+	
+
+	# Counter increase (NEMO restart index within ice sheet / ocean coupling window).
+	cnt_res_isfcpl=$(( ${cnt_res_isfcpl} + 1 ))
+	sed -i "s/.*cnt_res_isfcpl=.*/cnt_res_isfcpl=${cnt_res_isfcpl}/" ${start_dir}/tmp/isfcpl.track
+
+	if (( ${leg_number} % ${n_res_perisfcpl} == 0 )); then
+	    # We have just ended one NEMO leg which closes an ice sheet / ocean coupling window.
+	    date_end_isfcpl=${leg_almostend_date_yyyymmdd}
+	    sed -i "s/.*date_end_isfcpl=.*/date_end_isfcpl=${date_end_isfcpl}/" ${start_dir}/tmp/isfcpl.track
+
+	    cp -f ${start_dir}/tmp/isfcpl.track ${start_dir}/tmp/isfcpl.old
+
+	    ${rebuild_nemo_exe} mesh_mask ${nem_numproc}
+	    mv -f mesh_mask.nc ${archive_dir}/isfcpl/mesh_mask_${date_beg_isfcpl}-${date_end_isfcpl}.nc
+	    
+	    module purge; module load CDO
+	    # Concatenate all icesheet files from all legs contained within this coupling window
+	    outfile="${exp_name}_ocean2ice_${date_beg_isfcpl}-${date_end_isfcpl}.nc"
+	    cdo -O copy "${out_stencil::-2}??.nc" ${outfile}
+	    
+	    # CALL FETISH
+	    module purge
+	    cp -f ${start_dir}/tmp/isfcpl.track ${run_dir}
+	    module load ${fetish_modules}
+	    ${fetish_call}
+	    mv -f ${exp_name}_fETISh*.mat ${archive_dir}/isfcpl
+	    module purge
+
+	    # mv ocean2ice file to arch dir
+	    mv -f ${outfile} ${archive_dir}/isfcpl
+	    
+	    new_geom_file="${exp_name}_ice2ocean_${date_end_isfcpl}.nc"
+
+            # module load NCO
+            # ncks -C -v B ${init_fetish_file} -A ${new_geom_file}
+            # module purge
+
+	    old_geom=`readlink -f fETISh_8km_geometry_curr.nc`
+	    if [ "${init_geom}" == 0 ]; then
+		mv -f ${old_geom} ${archive_dir}/isfcpl
+	    else
+		init_geom=0
+		sed -i "s/.*init_geom=.*/init_geom=0/" ${start_dir}/tmp/isfcpl.track
+		cp -f ${old_geom} ${archive_dir}/isfcpl
+	    fi
+
+	    ln -sf ${new_geom_file} fETISh_8km_geometry_curr.nc
+	    
+	    cnt_res_isfcpl=0
+
+	    printf "# Information for the ice sheet model\n" > ${start_dir}/tmp/isfcpl.track
+	    printf "run_name=${exp_name}\n" >> ${start_dir}/tmp/isfcpl.track
+	    printf "initial_date=${run_start_date_yyyymmdd}\n" >> ${start_dir}/tmp/isfcpl.track
+	    printf "n_month_isfcpl=${n_month_isfcpl}\n" >> ${start_dir}/tmp/isfcpl.track
+	    printf "n_isfcpl_window=${n_isfcpl_window}\n" >> ${start_dir}/tmp/isfcpl.track
+	    printf "date_beg_isfcpl=YYYYMMDD\n" >> ${start_dir}/tmp/isfcpl.track
+	    printf "date_end_isfcpl=YYYYMMDD\n\n" >> ${start_dir}/tmp/isfcpl.track
+	    printf "# Information for NEMO to keep up\n" >> ${start_dir}/tmp/isfcpl.track	    
+	    printf "cnt_res_isfcpl=0\n" >> ${start_dir}/tmp/isfcpl.track
+	    printf "init_geom=${init_geom}\n" >> ${start_dir}/tmp/isfcpl.track
+	    printf "init_bathy=${init_bathy}\n" >> ${start_dir}/tmp/isfcpl.track
+	    printf "curr_bathy_file=${curr_bathy_file}\n" >> ${start_dir}/tmp/isfcpl.track
+	    
+	    module purge
+	    module load ${module_list:?}
+	    export LD_LIBRARY_PATH=${LD_LIBRARY_PATH:+${LD_LIBRARY_PATH}:}"${extralibs_list}"
+	    
+	fi
+    fi
+    
+    #############################
+
 
     formatted_leg_number=$(printf %03d $((leg_number)))
     #
@@ -197,15 +474,15 @@ do
     mkdir -p "${outdir}"
 
     shopt -s nullglob
-    for v in grid_U grid_V grid_W grid_T icemod icemoa SBC SBC_scalar scalar
+    for v in grid_U grid_V grid_W grid_T icemod icemoa SBC SBC_scalar scalar fwfisf
     do
         for f in ${exp_name}_??_????????_????????_${v}_????.nc
         do
-             mv "$f" "$outdir/"
+            mv "$f" "$outdir/"
         done
         for f in ${exp_name}_??_????????_????????_${v}.nc
         do
-             mv "$f" "$outdir/"
+            mv "$f" "$outdir/"
         done
         for f in ${exp_name}_??_${v}.nc
         do
@@ -216,13 +493,13 @@ do
     #
     # Move NEMO restart files from previous run to archive directory
     #
-        outdir="$archive_dir/restart/${formatted_leg_number}"
-        mkdir -p "${outdir}"
+    outdir="$archive_dir/restart/${formatted_leg_number}"
+    mkdir -p "${outdir}"
 
-        for f in ${exp_name}_${ns}_restart_???_????.nc
-        do
-               [ -f "$f" ] && mv "$f" "${outdir}"
-        done
+    for f in ${exp_name}_${ns}_restart_???_????.nc
+    do
+        [ -f "$f" ] && mv "$f" "${outdir}"
+    done
 
     #
     # Move log files to archive directory
@@ -230,12 +507,11 @@ do
     outdir="$archive_dir/log/${formatted_leg_number}"
     mkdir -p "${outdir}"
     for f in ocean.output time.step solver.stat ; do mv "${f}" "${outdir}"; done
-#
-#
-##############################
+    
+    
+    
 
     shopt -u nullglob
-
     # 
     # Write checkpoint control file
     # TODO: enquiry why 0 -1 +2 rather than 2- 1
@@ -243,19 +519,23 @@ do
     tr=$(date -d "0 -$time_begin sec + $time_end sec" +%T) 
     current_date=$(date +'%F %T')
     {
-      echo "#"
-      echo "# Finished leg at ${current_date} after ${tr} (hh:mm:ss)" 
-      echo "leg_number=${leg_number}"
-      echo "leg_start_date=\"${leg_start_date}\""
-      echo "leg_end_date=\"${leg_end_date}\""
+	echo "#"
+	echo "# Finished leg at ${current_date} after ${tr} (hh:mm:ss)" 
+	echo "leg_number=${leg_number}"
+	echo "leg_start_date=\"${leg_start_date}\""
+	echo "leg_end_date=\"${leg_end_date}\""
     } | tee -a "${info_file}"
 
     special_restart=false
+
 done
+
 #
 # Move back to submission directory
 #
 cd - >/dev/null
+
+
 #
 # Check whether there is some work left to do in a further job
 #
@@ -263,9 +543,14 @@ cd - >/dev/null
 if (( leg_end_epoch < run_end_epoch )) ; then
     echo "Leg end earlier than end of simulation."
     echo "Submitting another job."
+    
     #[[ $@ == *local* ]] && exec $0 $@ || scontrol requeue $SLURM_JOB_ID #TODO: factorize this
     if [[ "$@" == *"local"* ]] ; then
-        exec "$0" "$@"
+	cd ${start_dir}
+	sleep 2
+	./run.sh local && exit 0
+
+	#exec "$0" "$@"
     elif [ "${submit_command}" == *sbatch* ] ; then
         sbatch $0 $@ | awk '{print $4}' >> ${run_dir}/.coral_jobs
     elif [[ $USER == "vsc"* ]]; then