9
0

2 Commitit 4eecd5b605 ... 9f080bba69

Tekijä SHA1 Viesti Päivämäärä
  Charles Pelletier 9f080bba69 Foko's coral 3 vuotta sitten
  Charles Pelletier 500b93c66f new coral isfcpl 3 vuotta sitten
3 muutettua tiedostoa jossa 1120 lisäystä ja 610 poistoa
  1. 49 10
      coral
  2. 287 600
      templates/scripts/skeleton.sh
  3. 784 0
      templates/scripts/skeleton_isfcpl.sh

+ 49 - 10
coral

@@ -77,6 +77,44 @@ EOT
   echo -e "Done. Run 'coral submit' to submit the job, or submit it manually." 
 }
 
+build_isfcpl() {
+doc "$@" <<EOT
+build - Build submission script from the configuraiton files in the current directory.
+EOT
+
+  [[ -f experiment.cfg ]] || die 1 "Error: configuration files not found. Run 'coral init' first"
+
+  echo -e "Generating job submission script '${output_script}'..."
+
+  echo "#!/bin/bash" >${output_script} ; (
+  set -e
+  source experiment.cfg
+  export exp_name
+  export nem_config_name
+  source programs.cfg
+  export nem_numproc
+  export xio_numproc
+  export total_nb_cpu=$((nem_numproc + xio_numproc))
+  export email=$USER_EMAIL
+  submit_command=$(grep -q SBATCH job.cfg && echo sbatch || echo qsub) ; export submit_command
+  echo -e  "\n#\n# Job options\n#"
+  <job.cfg egrep -v "(^\#|^$)" | sed "s/SBATCH/#SBATCH/" | sed "s/PBS -/#PBS -/" | envsubst \$exp_name,\$email,\$total_nb_cpu
+  echo -e  "\n#\n# Experiment options\n#"
+  <experiment.cfg egrep -v "(^\#|^$)" 
+  echo -e  "\n#\n# Data configuration\n#"
+  <data.cfg cat
+  echo -e  "\n#\n# Program configuration\n#"
+  <programs.cfg egrep -v "(^\#|^$)" 
+  echo -e  "\n#\n# Script logic\n#"
+  <"${CORAL_HOME}"/templates/scripts/skeleton_isfcpl.sh egrep  -v '^[[:space:]]*\#'  | envsubst \$submit_command
+  ) >>${output_script} 2>/dev/null \
+    || die 3 "Error: Syntax error in configuration files. Please review them."
+
+  echo -e "Done. Run 'coral submit' to submit the job, or submit it manually." 
+}
+
+
+
 submit() {
 doc "$@" <<EOT
 submit [<options>] - Submit the job based on the submission script present in the current directory.
@@ -97,17 +135,17 @@ EOT
   ( 
   set -e
   source experiment.cfg
-  eval "$(grep run_dir job.cfg)"
+  eval "$(grep out_dir job.cfg)"
 
-  [[ "$@" == *fromscratch* ]] && rm -rf "${run_dir:?}"
+  [[ "$@" == *fromscratch* ]] && rm -rf "${out_dir:?}"
   [[ "$@" == *local* ]] && { ./"${output_script}" "$@" ; exit 0; }
 
   which sbatch &> /dev/null  && {
      jobid="$(sbatch "${output_script}" "$@")"
      sleep 2
      jobid=${jobid##* }
-     mkdir -p "${run_dir:?}"
-     echo "${jobid}" >> "${run_dir}"/.coral_jobs ; 
+     mkdir -p "${out_dir:?}"
+     echo "${jobid}" >> "${out_dir}"/.coral_jobs ; 
      #squeue --start -j "${jobid}" # FIXME
      squeue -j "${jobid}"
      exit 0; 
@@ -129,8 +167,8 @@ EOT
         jobid=`cat coral_jobs`
         rm -f coral_jobs
         jobid=${jobid%%.*}
-        mkdir -p "${run_dir:?}"
-        echo "${jobid}" >> "${run_dir}"/.coral_jobs ;
+        mkdir -p "${out_dir:?}"
+        echo "${jobid}" >> "${out_dir}"/.coral_jobs ;
         qstat
      else
         qsub -v PBS_OPTIONS="$@" "${output_script}" | tee -a coral_jobs;
@@ -138,8 +176,8 @@ EOT
         jobid=`cat coral_jobs`
         rm -f coral_jobs
         jobid=${jobid%.*} 
-        mkdir -p "${run_dir:?}"
-        echo "${jobid}" >> "${run_dir}"/.coral_jobs ;
+        mkdir -p "${out_dir:?}"
+        echo "${jobid}" >> "${out_dir}"/.coral_jobs ;
         qstat -J "${jobid}"
      fi
 
@@ -302,6 +340,7 @@ help() {
 Usage: $0 <command> [<arguments>], where command is one of:
   init  - Initialize template in current directory. Default template is the machine name.
   build - Build submission script from template in current directory.
+  build_isfcpl - Build submission script from template in current directory, incl. isfcpl coupling.
   submit - Submit job from submission script in current directory.
   status - Displays the current status of the simulation.
   save - Save modifications to template in current directory
@@ -321,7 +360,7 @@ _coral ()   #  By convention, the function name
     _get_comp_words_by_ref cur prev words cword
     _split_long_opt
 
-    COMPREPLY=( $(compgen -W "init commit submit build edit share save import publish update list status" -- $cur) )
+    COMPREPLY=( $(compgen -W "init commit submit build build_isfcpl edit share save import publish update list status" -- $cur) )
 }
 
 complete -F _coral coral
@@ -329,7 +368,7 @@ EOTCOMPLETION
 }
 
 
-if [[ ! "$1" =~ ^(init|commit|submit|build|edit|share|save|import|publish|update|list|status|completion)$ ]]; then
+if [[ ! "$1" =~ ^(init|commit|submit|build|build_isfcpl|edit|share|save|import|publish|update|list|status|completion)$ ]]; then
   help >&2
   exit 1
 else

Tiedoston diff-näkymää rajattu, sillä se on liian suuri
+ 287 - 600
templates/scripts/skeleton.sh


+ 784 - 0
templates/scripts/skeleton_isfcpl.sh

@@ -0,0 +1,784 @@
+# Function leap days calculates the number of leap days (29th of Februrary) in
+# a time intervall between two dates.
+#
+# Usage leap_days START_DATE END_DATE
+function leap_days()
+{
+    local ld=0
+    local frstYYYY=$(date -ud "$1" +%Y)
+    local lastYYYY=$(date -ud "$2" +%Y)
+
+    set +e
+
+    # Check first year for leap day between start and end date
+    $(date -ud "${frstYYYY}-02-29" > /dev/null 2>&1) \
+    && (( $(date -ud "$1" +%s) < $(date -ud "${frstYYYY}-03-01" +%s) )) \
+    && (( $(date -ud "$2" +%s) > $(date -ud "${lastYYYY}-02-28" +%s) )) \
+    && (( ld++ ))
+
+    # Check intermediate years for leap day
+    for (( y=(( ${frstYYYY}+1 )); y<=(( ${lastYYYY}-1 )); y++ ))
+    do
+        $(date -ud "$y-02-29" > /dev/null 2>&1) && (( ld++ ))
+    done
+
+    # Check last year (if different from first year) for leap day between start
+    # and end date
+    (( $lastYYYY > $frstYYYY )) \
+    && $(date -ud "${lastYYYY}-02-29" > /dev/null 2>&1) \
+    && (( $(date -ud "$1" +%s) < $(date -ud "${frstYYYY}-03-01" +%s) )) \
+    && (( $(date -ud "$2" +%s) > $(date -ud "${lastYYYY}-02-28" +%s) )) \
+    && (( ld++ ))
+
+    set -e
+
+    echo "$ld"
+}
+
+# Function for loading modules 
+function load_modules_isfcpl () {
+    if [[ ${USER} == "vsc"????? ]]; then
+
+        # quick and dirty patch - only BRENIAC will work while waiting for something to distinguish between both machines (include on computing nodes)
+
+        #       if [[ ${HOSTNAME} == "tier2"* ]]; then
+
+
+
+            #     if [[ "${name_icesheet_model}" == "fETISh" ]]; then
+        #       module purge
+        #       module load Python
+        #       PYTHONPATH="${HOME}/.local/bin:${HOME}/.local/lib/python3.7/site-packages:\${PYTHONPATH}"
+        #       module load CDO
+        #       echo ${PYTHONPATH} > ${start_dir}/ppath
+        #       echo "genius and fetish detected" > ${start_dir}/tmpout
+        #     else
+        #       echo "On Genius, ice sheet models other than fETISh not implemented yet" > ${start_dir}/tmpout
+        #       exit 2
+        #     fi
+
+        # elif [[ ${HOSTNAME} == "login"? ]]; then
+            if [[ "${name_icesheet_model}" == "fETISh" ]]; then
+                module purge
+                module load Python/3.7.2-intel-2018a CDO/1.9.5-intel-2018a
+                [ ! -d "${VSC_HOME}/.local/lib/python3.7/site-packages/netCDF4" ] && pip install --user netCDF4
+                [ ! -d "${VSC_HOME}/.local/lib/python3.7/site-packages/numpy" ] && pip install --target=${VSC_HOME}/.local/lib/python3.7/site-packages numpy
+                [ ! -d "${VSC_HOME}/.local/lib/python3.7/site-packages/gsw" ] && pip install --user gsw
+
+                PYTHONPATH="${HOME}/.local/bin:${HOME}/.local/lib/python3.7/site-packages:\${PYTHONPATH}"
+                echo ${PYTHONPATH} > ${start_dir}/ppath
+                echo "breniac and fetish detected" > ${start_dir}/tmpout
+            else
+                echo "On Breniac, ice sheet models other than fETISh not implemented yet" > ${start_dir}/tmpout
+                exit 2
+            fi
+
+        # else
+        #     echo "You seem to be on a VSC machine that is neither Genius nor BRENIAC. This has not been implemented yet. Exiting"
+        #     exit 2
+        # fi
+
+    elif [[ ${HOSTNAME} == "lm3"* ]]; then
+        if [[ "${name_icesheet_model}" == "fETISh" ]]; then
+            echo "lemaitre3 and fetish detected" > ${start_dir}/tmpout
+            module purge
+            module load releases/2018a CDO/1.8.2-intel-2018a Python/3.6.4-intel-2018a
+        else
+            echo "On lemaitre3, ice sheet models other than fETISh not implemented yet" > ${start_dir}/tmpout
+            exit 2
+        fi
+
+    else
+        echo "hostname undetected. not implemented yet." > ${start_dir}/tmpout
+        exit 2
+    fi
+}
+
+[[ $@ == *verbose* ]] && set -x
+
+module load ${module_list:?}
+export LD_LIBRARY_PATH=${LD_LIBRARY_PATH:+${LD_LIBRARY_PATH}:}"${extralibs_list}"
+
+mkdir -p ${run_dir}
+cd ${run_dir}
+
+# Prepare iscpl
+if [ "${isfcpl}" == "1" ]; then
+
+    if [[ ${USER} == "vsc"????? ]]; then
+
+	if [[ ${VSC_INSTITUTE_CLUSTER} == "breniac" ]]; then
+            echo "BRENIAC and iscpl detected. OK."
+	else
+	    echo "VSC clusters other than BRENIAC not supported for isfcpl."
+	    exit 2
+	fi
+	
+    elif [[ ${HOSTNAME} == "lm3"* ]]; then
+        echo "Lemaitre3 and iscpl detected. OK."
+    else
+        echo "You have asked for iscpl, but I have not recognized genius nor lemaitre3. Exiting."
+        exit 2
+    fi
+
+    if [ "${name_icesheet_model}" != "fETISh" ]; then
+        echo "only \"fETISh\" is known as an ice sheet model. exiting."
+        exit 2
+    fi
+
+    length_cut_pref=$(( ${#exp_name} + 4 ))
+    rebuild_nemo_exe=${nemo_src_dir}/TOOLS/REBUILD_NEMO/rebuild_nemo
+    if [ -f ${start_dir}/isfcpl.track ]; then
+        source ${start_dir}/isfcpl.track
+    fi
+
+    iscpl_nm=.true.
+
+    if [[ ${xio_exe_file} == *"xios-2"* ]]; then
+        is_xios2=1
+    else
+        is_xios2=0
+    fi
+
+else
+    iscpl_nm=.false.
+
+fi
+
+run_start_epoch=$(date -u -d"${run_start_date}" +%s)
+run_start_date_cos="${run_start_date//-}00"
+run_start_date=$(date -uR -d "${run_start_date}")
+run_end_date="${run_start_date} + ${run_duration:?}"
+run_end_date=$(date -uR -d "${run_end_date}")
+run_end_epoch=$(date -u -d"${run_end_date}" +%s)
+run_end_date_cos="$(date -u -d"${run_end_date}" +%Y%m%d%H)"
+
+
+if [[ ${special_rs_nemo} == "true" && ${special_restart} == "true" ]]; then
+    echo "ERROR: both special_rs_nemo and special_restart are true. only can be true (at most). exit".
+    exit 2
+fi
+
+
+for (( ; run_num_legs>0 ; run_num_legs-- ))
+do
+
+    # Check for restart information file and set the current leg start date
+    #   Ignore restart information file if force_run_from_scratch is true
+    if ! [ -r ${info_file} ]
+    then
+        leg_is_restart=false
+        leg_start_date=${run_start_date}
+        leg_start_date_cos=${run_start_date_cos}
+        leg_number=1
+        echo "leg_is_restart=false"
+    else
+        leg_is_restart=true
+        . ./${info_file}
+        leg_start_date=${leg_end_date}
+        leg_start_date_cos="$(date -u -d"${leg_start_date}" +%Y%m%d%H)"
+        leg_number=$((leg_number+1))
+        echo "leg_is_restart=true"
+    fi
+
+    # Compute the end date of the current leg
+    if [ -n "${rst_freq}" ]
+    then
+        if (($(date -u -d "${leg_start_date:?} + ${rst_freq}" +%s) > $(date -u -d "${leg_start_date:?} + ${run_duration}" +%s) )); then
+           rst_freq=$run_duration
+        fi
+        leg_end_date=$(date -uR -d "${leg_start_date} + ${rst_freq}")
+        leg_end_date_cos="$(date -u -d"${leg_end_date}" +%Y%m%d%H)"
+    else
+        leg_end_date=${run_end_date}
+        leg_end_date_cos=${run_end_date_cos}
+    fi
+
+    if [ $(date -u -d "${leg_end_date}" +%s) -gt $(date -u -d "${run_end_date}" +%s) ]
+    then
+        leg_end_date=${run_end_date}
+        leg_end_date_cos=${run_end_date_cos}
+    fi
+
+    # Some time variables needed later
+    leg_start_epoch=$(date -u -d "${leg_start_date}" +%s)
+    leg_end_epoch=$(date -u -d "${leg_start_date:?} + ${rst_freq:=$run_duration}" +%s)
+    leg_length_sec=$(( $(date -u -d "${leg_end_date}" +%s) - $(date -u -d "${leg_start_date}" +%s) ))
+    leg_start_sec=$(( $(date -u -d "${leg_start_date}" +%s) - $(date -u -d "${run_start_date}" +%s) ))
+    leg_end_sec=$(( $(date -u -d "${leg_end_date}" +%s) - $(date -u -d "${run_start_date}" +%s) ))
+    leg_start_date_yyyymmdd=$(date -u -d "${leg_start_date}" +%Y%m%d)
+    leg_start_date_yyyy=$(date -u -d "${leg_start_date}" +%Y)
+    leg_end_date_yyyy=$(date -u -d "${leg_end_date}" +%Y)
+
+    echo "leg_number: $leg_number"
+    echo "leg_start_date: $leg_start_date"
+    echo "leg_end_date: $leg_end_date"
+
+    # Additional time variables for iscpl
+    if (( isfcpl == 1 )); then
+       leg_beforestart_epoch=$(( ${leg_start_epoch} - 43200 ))
+       leg_beforestart_date=$(date -uR -d@"${leg_beforestart_epoch}")
+       leg_almostend_epoch=$(( ${leg_end_epoch} - 43200 ))
+       leg_almostend_date=$(date -uR -d@"${leg_almostend_epoch}")
+       nextleg_start_epoch=$(( ${leg_end_epoch} + 43200 ))
+       nextleg_start_date=$(date -uR -d@"${nextleg_start_epoch}")
+
+       run_start_date_yyyymmdd=$(date -u -d "${run_start_date}" +%Y%m%d) # FIXME appears unused
+       run_end_date_yyyymmdd=$(date -u -d "${run_end_date}" +%Y%m%d) # FIXME appears unused
+       leg_start_date_yyyymmdd=$(date -u -d "${leg_start_date}" +%Y%m%d) # FIXME appears unused
+       leg_almostend_date_yyyymmdd=$(date -u -d "${leg_almostend_date}" +%Y%m%d) # FIXME appears unused
+       nextleg_start_date_yyyymmdd=$(date -u -d "${nextleg_start_date}" +%Y%m%d) # FIXME appears unused
+       leg_beforestart_date_yyyymmdd=$(date -u -d "${leg_beforestart_date}" +%Y%m%d) # FIXME appears unused
+    fi
+
+    # Check whether there's actually time left to simulate - exit otherwise
+    if [ ${leg_length_sec} -le 0 ]
+    then
+        info "Leg start date equal to or after end of simulation."
+        info "Nothing left to do. Exiting."
+        exit 0
+    fi
+
+    # -------------------------------------------------------------------------
+    # *** Prepare the run directory for a run from scratch
+    # -------------------------------------------------------------------------
+    if ! $leg_is_restart
+    then
+        # ---------------------------------------------------------------------
+        # *** Check if run dir is empty. If not, and if we are allowed to do so
+        #     by ${force_run_from_scratch}, remove everything
+        # ---------------------------------------------------------------------
+        echo "pwd: $PWD"
+        if $(ls * >& /dev/null)
+        then
+            if ${force_run_from_scratch}
+            then
+                rm -fr ${run_dir}/*
+            else
+                echo "error: run directory not empty and \$force_run_from_scratch not set."
+                exit 0
+            fi
+        fi
+
+        # ---------------------------------------------------------------------
+        # *** Creates some directories
+        # ---------------------------------------------------------------------
+        mkdir -p ${out_dir}
+	
+        # ---------------------------------------------------------------------
+        # *** Copy executables and coupling-related files
+        # ---------------------------------------------------------------------
+        cd ${start_dir}
+        cp -u xios_config/*xml "${run_dir}"
+        cp -u namelists/* "${run_dir}"
+        cd ${run_dir}
+        cp -u "${nem_exe_file:?}" .
+        cp -u "${xio_exe_file:?}" .
+	if (( isfcpl == 1 )) ; then
+	    cp -u ${icesheet_exe_file} .
+	    cp -u ${isfcpl_dir}/* .
+	    ln -sf ${start_dir}/isfcpl.track ${run_dir}
+	    mkdir -p ${out_dir}/isfcpl_tmp
+	fi
+
+        # ---------------------------------------------------------------------
+        # *** Link ic, forcing and shared files for NEMO
+        # ---------------------------------------------------------------------
+        [[ ! -f EMPave_old.dat ]] && echo "                               0  0.0000000000000000E+00  0.0000000000000000E+00" > EMPave_old.dat
+        for file in "${ic_files[@]}"; do
+            [[ ! -e ${file#*> } ]] && ln -sf $(sed 's/ *=> */ /' <<< "${ini_data_dir}/${ic_subdir}/${nem_grid}/$file")
+        done
+        for file in "${forcing_files[@]}"; do
+            [[ ! -e ${file#*> } ||  "$file" == \** ]] && ln -sf $(sed 's/ *=> */ /' <<< "${ini_data_dir}/${forcing_subdir}/${nem_forcing_set}/$file")
+        done
+        for file in "${shared_files[@]}"; do
+            [[ ! -e ${file#*> } ]] && ln -sf $(sed 's/ *=> */ /' <<< "${shared_dir}/$file")
+        done
+
+        # ---------------------------------------------------------------------
+        # *** Prepare NEMO - ice shelf coupling
+        # ---------------------------------------------------------------------
+        if [ "${isfcpl}" == "1" ]; then
+
+            if (( leg_number == 1 )); then
+
+                mkdir -p ${archive_dir}/isfcpl
+                mkdir -p ${archive_dir}/${name_icesheet_model}
+                rm -f ${start_dir}/isfcpl.track
+
+
+                isfcpl_window_days=$(echo "(${n_res_perisfcpl} * ${leg_length_sec}) / 86400" | bc)
+                isfcpl_d30=$(echo "${isfcpl_window_days} / 30" | bc -l)
+                n_month_isfcpl=$(echo ${isfcpl_d30} | awk '{print int($1+0.5)}')
+
+                n_month_full_run=$(( $(( 12 * $(( ${run_end_date_yyyymmdd::4} - ${run_start_date_yyyymmdd::4} )) )) + $(( ${run_end_date_yyyymmdd:4:2} - ${run_start_date_yyyymmdd:4:2} )) ))
+
+                if (( $(( ${n_month_full_run} % ${n_month_isfcpl} )) != 0 )); then
+                    echo "n_month_full_run=${n_month_full_run} is not a multiple of n_month_isfcpl=${n_month_isfcpl}."
+                    exit 2
+                else
+                    n_isfcpl_window=$(echo "${n_month_full_run} / ${n_month_isfcpl}" | bc)
+                fi
+
+                tmp=`readlink -f isf_draft_meter.nc`
+                tmp2=`readlink -f bathy_meter.nc`
+
+                if [ "${tmp}" != "${tmp2}" ]; then
+                    echo "Error. At initialization, isf_draft_meter.nc and bathy_meter.nc do not point to the same file."
+                    exit 1
+                fi
+
+                init_bathy_file=`basename ${tmp}`
+                cp -f ${tmp} ${archive_dir}/isfcpl
+                cp -f ${tmp} ${run_dir}/init_NEMO_bathy.nc
+
+		ln -sf ${icesheet_files_dir}/* ./
+		ln -sf ${isfcpl_files_dir}/* ./
+		
+                date_beg_isfcpl="${run_start_date_yyyymmdd}"
+
+                printf "# Information for the ice sheet model\n" > ${start_dir}/isfcpl.track
+                printf "run_name=${exp_name}\n" >> ${start_dir}/isfcpl.track
+                printf "initial_date=${run_start_date_yyyymmdd}\n" >> ${start_dir}/isfcpl.track
+                printf "n_month_isfcpl=${n_month_isfcpl}\n" >> ${start_dir}/isfcpl.track
+                printf "n_isfcpl_window=${n_isfcpl_window}\n" >> ${start_dir}/isfcpl.track
+                printf "date_beg_isfcpl=${date_beg_isfcpl}\n" >> ${start_dir}/isfcpl.track
+                printf "date_end_isfcpl=YYYYMMDD\n" >> ${start_dir}/isfcpl.track
+                printf "ice_atm_cpl=${ice_atm_cpl}\n\n" >> ${start_dir}/isfcpl.track
+
+                printf "# Information for NEMO to keep up\n" >> ${start_dir}/isfcpl.track
+                printf "init_bathy=1\n" >> ${start_dir}/isfcpl.track
+                printf "curr_bathy_file=${init_bathy_file}\n" >> ${start_dir}/isfcpl.track
+
+                init_bathy=1
+                curr_bathy_file=${init_bathy_file}
+
+            fi
+
+            ln -sf ${curr_bathy_file} ${run_dir}/bathy_meter.nc
+            ln -sf ${curr_bathy_file} ${run_dir}/isf_draft_meter.nc
+
+
+        fi # if [ "${iscpl}" == 1 ]
+
+        # ---------------------------------------------------------------------
+        # *** Copy potential special restarts 
+        # ---------------------------------------------------------------------
+        if $special_restart
+        then
+
+            ### Getting the right output folders 
+            leg_start_date_tmp=$leg_start_date
+            leg_end_date_tmp=$leg_end_date
+            leg_number_tmp=$leg_number
+            rsync -avz  ${run_dir}/../../${special_restart_from}/rundir/${info_file} ${run_dir}
+            special_date=$(date -uR -d "${special_restart_date}")
+            sed -i "/$special_date/q" ${run_dir}/${info_file}
+            . ${run_dir}/${info_file}
+            special_restart_leg=$(printf %03d $((leg_number)))
+
+            echo "special_date: $special_date"
+            echo "special_restart_leg: $special_restart_leg"
+            echo "special_restart_from: $special_restart_from"
+
+            ### NEMO
+            indir="${run_dir}/../../${special_restart_from}/archive/restart/nemo/$(printf %03d $((special_restart_leg)))"
+            cd ${indir}
+            for f in *.nc; do
+                nf=`echo "${f/$special_restart_from/$exp_name}"`
+                echo "copy $f to ${run_dir}/$nf" 
+               cp $f ${run_dir}/$nf
+            done
+            cd -
+            cd ${run_dir}
+            echo "pwd: $PWD"
+            for f in ${exp_name}_????????_restart_???_????.nc; do
+                nf=`echo ${f: -19}`
+                echo "linking $f to ${run_dir}/$nf" 
+                ln -s $f $nf
+            done
+
+            cd ${run_dir}
+            rm -rf ${info_file}
+            leg_start_date=$leg_start_date_tmp
+            leg_end_date=$leg_end_date_tmp
+            leg_number=$leg_number_tmp
+
+	elif ${special_rs_nemo}
+	then
+
+	    dir_rs=`dirname ${special_rs_nemo_stencil}`
+	    base_rs=`basename ${special_rs_nemo_stencil}`
+	    
+	    n_oce_rs=`find ${dir_rs} -name "${base_rs}oce_????.nc" | wc -l`
+	    if(( n_oce_rs != nem_numproc )); then
+		echo "in special_rs_nemo, not the right amount of ocean restart files."
+		echo "${n_oce_rs} instead of ${nem_numproc}. exiting."
+		exit 2
+	    fi
+
+	    n_ice_rs=`find ${dir_rs} -name "${base_rs}ice_????.nc" | wc -l`
+	    if(( n_ice_rs != nem_numproc )); then
+		echo "in special_rs_nemo, not the right amount of ice restart files."
+		echo "${n_ice_rs} instead of ${nem_numproc}. exiting."
+		exit 2
+	    fi
+	    
+            for (( n=0 ; n<nem_numproc ; n++ ))
+            do
+		np=$(printf %04d ${n})
+		ln -fs ${special_rs_nemo_stencil}oce_${np}.nc restart_oce_${np}.nc
+		ln -fs ${special_rs_nemo_stencil}ice_${np}.nc restart_ice_${np}.nc
+            done
+
+        fi # if $special_restart
+
+
+    else # if ! $leg_is_restart
+
+        # ---------------------------------------------------------------------
+        # *** Creates some directories
+        # ---------------------------------------------------------------------
+        mkdir -p ${out_dir}
+
+        if [ "${isfcpl}" == "1" ]; then
+            ln -sf ${curr_bathy_file} ${run_dir}/bathy_meter.nc
+            ln -sf ${curr_bathy_file} ${run_dir}/isf_draft_meter.nc
+        fi
+
+        # ---------------------------------------------------------------------
+        # *** Copy restart files in run directory
+        # ---------------------------------------------------------------------
+        cd ${run_dir}
+        ### NEMO
+        outdir="${archive_dir}/restart/nemo/$(printf %03d $((leg_number-1)))"
+
+	
+        ns=$(printf %08d $(( leg_start_sec / nem_time_step_sec )))
+        for f in oce ice
+        do
+	    echo "${outdir}/${exp_name}_${ns}_restart_${f}_????.nc"
+	    ln -sf ${outdir}/${exp_name}_${ns}_restart_${f}_????.nc ${run_dir}
+        done
+
+        for (( n=0 ; n<nem_numproc ; n++ ))
+        do
+	    np=$(printf %04d ${n})
+	    ln -fs ${exp_name}_${ns}_restart_oce_${np}.nc restart_oce_${np}.nc
+	    ln -fs ${exp_name}_${ns}_restart_ice_${np}.nc restart_ice_${np}.nc
+        done
+
+    fi # ! $leg_is_restart
+
+    # -------------------------------------------------------------------------
+    # *** Update ocean namelist
+    # -------------------------------------------------------------------------
+    source build_namelist_cfg.sh > namelist_cfg
+
+    # -------------------------------------------------------------------------
+    # *** Build ice sheet launch
+    # -------------------------------------------------------------------------
+    if (( isfcpl == 1)); then
+	source ${icesheet_launch_template} > launch_icesheet.sh
+	chmod +x launch_icesheet.sh
+    fi
+    
+
+    # -------------------------------------------------------------------------
+    # *** Launch experiment
+    # -------------------------------------------------------------------------
+
+    
+    echo "START of mpirun"
+    time_begin=$(date +%s)
+    mpirun -np "${xio_numproc:?}" "${xio_exe_file:?}" : -np "${nem_numproc:?}" "${nem_exe_file:?}"
+    time_end=$(date +%s)
+    echo "END of: mpirun"
+
+    # -------------------------------------------------------------------------
+    # *** Ice sheet coupling
+    # -------------------------------------------------------------------------
+
+    if (( isfcpl == 1 )); then
+
+        cnt_res_isfcpl=$(( $(( ${leg_number} - 1 )) % ${n_res_perisfcpl} ))
+        fmt_res_isfcpl=$(printf "%02d" ${cnt_res_isfcpl})
+
+        in_fwfisf_stencil="${exp_name}_${frequency_fwfisf}_${leg_start_date_yyyymmdd}_${leg_almostend_date_yyyymmdd}_${suffix_fwfisf}"
+        out_fwfisf_stencil="${exp_name}_fwfisf_res"
+
+	out_fwfisf_curr="${out_fwfisf_stencil}${fmt_res_isfcpl}"
+	
+        if (( is_xios2 == 0 )); then
+
+            module purge
+            module load ${module_list:?}
+            export LD_LIBRARY_PATH=${LD_LIBRARY_PATH:+${LD_LIBRARY_PATH}:}"${extralibs_list}"
+
+            /bin/bash ${rebuild_nemo_exe} ${in_fwfisf_stencil} ${xio_numproc}
+
+	    module purge
+	    load_modules_isfcpl
+	    
+	    cdo selname,${name_var_fwfisf} ${in_fwfisf_stencil}.nc tmp_fwfisf.nc
+	    cdo monmean tmp_fwfisf.nc ${out_fwfisf_curr}.nc
+
+	    rm -f ${in_fwfisf_stencil}.nc tmp_fwfisf.nc
+	
+        else
+	    
+	    cdo selname,${name_var_fwfisf} ${in_fwfisf_stencil}.nc tmp_fwfisf.nc
+	    cdo monmean tmp_fwfisf.nc ${out_fwfisf_curr}.nc
+
+	    rm -f tmp_fwfisf.nc
+
+        fi
+
+	mv -f ${out_fwfisf_curr}.nc ${out_dir}/isfcpl_tmp/
+	
+        if (( ${leg_number} % ${n_res_perisfcpl} == 0 )); then
+            date_end_isfcpl=${leg_almostend_date_yyyymmdd}
+            sed -i "s/.*date_end_isfcpl=.*/date_end_isfcpl=${date_end_isfcpl}/" ${start_dir}/isfcpl.track
+
+            module purge
+            module load ${module_list:?}
+            export LD_LIBRARY_PATH=${LD_LIBRARY_PATH:+${LD_LIBRARY_PATH}:}"${extralibs_list}"
+
+            /bin/bash ${rebuild_nemo_exe} mesh_mask ${nem_numproc}
+
+            module purge
+            load_modules_isfcpl
+
+            infile_ocean2ice="${exp_name}_ocean2ice_raw_${date_beg_isfcpl}-${date_end_isfcpl}.nc"
+            outfile_ocean2ice="${exp_name}_ocean2ice_${date_beg_isfcpl}-${date_end_isfcpl}.nc"
+	    
+	    fwfisf_records=""
+	    for((rs2=0; rs2<n_res_perisfcpl; rs2++)); do
+                fmt_rs2=$(printf "%02d" ${rs2})
+		curr_file="${out_dir}/isfcpl_tmp/${out_fwfisf_stencil}${fmt_rs2}.nc"
+		if [ -f ${curr_file} ] ; then
+		    fwfisf_records="${fwfisf_records} ${curr_file}"
+		else
+		    echo "${curr_file} is not in ${out_dir}/isfcpl_tmp. It is needed for NEMO - ice sheet coupling. Exiting"
+		    exit 2
+		fi
+	    done
+	    
+            cdo -O copy ${fwfisf_records} ${infile_ocean2ice}
+	    rm -f ${fwfisf_records}
+	    
+	    source build_pp_nemo_to_icesheet.sh > ./pp_nemo_to_${name_icesheet_model}.py
+	    python pp_nemo_to_${name_icesheet_model}.py
+
+	    mv -f ${infile_ocean2ice} ${archive_dir}/isfcpl/
+            mv -f mesh_mask.nc ${archive_dir}/isfcpl/${exp_name}_mesh_mask_${date_beg_isfcpl}-${date_end_isfcpl}.nc
+
+            module purge
+            module load ${icesheet_modules}
+            if [ "${icesheet_library_path}" != "" ]; then
+                export LD_LIBRARY_PATH="${icesheet_library_path}"
+            fi
+	    /bin/bash ./launch_icesheet.sh
+
+	    mv -f ${outfile_ocean2ice} ${archive_dir}/isfcpl
+	    if (( ice_atm_cpl == 1 )); then
+		mv -f ${outfile_atm2ice} ${archive_dir}/isfcpl
+	    fi
+
+	    if [[ "${name_icesheet_model}" == "fETISh" ]]; then
+		find . -type l -name "${exp_name}_${name_icesheet_model}Run_??????_toto.mat" -exec rm -f {} \;
+		mv -f ${exp_name}_${name_icesheet_model}Run_??????_toto.mat ${archive_dir}/${name_icesheet_model}
+		ln -sf ${archive_dir}/${name_icesheet_model}/${exp_name}_${name_icesheet_model}Run_${date_end_isfcpl::-2}_toto.mat
+	    else
+		echo "Ice-sheet models other than fETISh not implemented yet."
+		exit 2
+	    fi
+	    
+            curr_geom_file="${exp_name}_ice2ocean_${date_end_isfcpl}.nc"
+	    new_bathy_file="${exp_name}_bathymetry_isfdraft_${date_end_isfcpl}.nc"
+	    source build_pp_icesheet_to_nemo.sh > pp_${name_icesheet_model}_to_nemo.py
+	    
+	    if [ "${init_bathy}" == 0 ]; then
+		mv -f ${curr_bathy_file} ${archive_dir}/isfcpl
+	    else
+		init_bathy=0
+		cp -f ${curr_bathy_file} ${archive_dir}/isfcpl
+	    fi
+	    	    
+            module purge
+            load_modules_isfcpl
+	    python pp_${name_icesheet_model}_to_nemo.py
+
+	    mv -f ${curr_geom_file} ${archive_dir}/isfcpl
+
+	    curr_bathy_file=${new_bathy_file}
+            ln -sf ${curr_bathy_file} ${run_dir}/bathy_meter.nc
+            ln -sf ${curr_bathy_file} ${run_dir}/isf_draft_meter.nc
+
+            printf "# Information for the ice sheet model\n" > ${start_dir}/isfcpl.track
+            printf "run_name=${exp_name}\n" >> ${start_dir}/isfcpl.track
+            printf "initial_date=${run_start_date_yyyymmdd}\n" >> ${start_dir}/isfcpl.track
+            printf "n_month_isfcpl=${n_month_isfcpl}\n" >> ${start_dir}/isfcpl.track
+            printf "n_isfcpl_window=${n_isfcpl_window}\n" >> ${start_dir}/isfcpl.track
+            printf "date_beg_isfcpl=${nextleg_start_date_yyyymmdd}\n" >> ${start_dir}/isfcpl.track
+            printf "date_end_isfcpl=YYYYMMDD\n" >> ${start_dir}/isfcpl.track
+            printf "ice_atm_cpl=${ice_atm_cpl}\n\n" >> ${start_dir}/isfcpl.track
+
+            printf "# Information for NEMO to keep up\n" >> ${start_dir}/isfcpl.track
+            printf "init_bathy=${init_bathy}\n" >> ${start_dir}/isfcpl.track
+            printf "curr_bathy_file=${curr_bathy_file}\n" >> ${start_dir}/isfcpl.track
+
+            module purge
+            module load ${module_list:?}
+            export LD_LIBRARY_PATH=${LD_LIBRARY_PATH:+${LD_LIBRARY_PATH}:}"${extralibs_list}"
+
+        fi
+    fi
+
+    # -------------------------------------------------------------------------
+    # *** Move output files to archive directory
+    # -------------------------------------------------------------------------
+    ### NEMO
+    outdir="${archive_dir}/output/nemo/$(printf %03d $((leg_number)))"
+    mkdir -p ${outdir}
+
+    for v in grid_U grid_V grid_W grid_T icemod SBC scalar SBC_scalar diad_T \
+             grid_T_2D grid_U_2D grid_V_2D grid_W_2D grid_T_3D grid_U_3D grid_V_3D grid_W_3D \
+             grid_1point grid_T_3D_ncatice vert_sum \
+             grid_ptr_W_3basin_3D grid_ptr_T_3basin_2D grid_ptr_T_2D \
+             zoom_700_sum zoom_300_sum zoom_2000_sum
+    do
+        for f in ${exp_name}_*_????????_????????_*${v}*.nc
+        do
+            test -f $f && mv $f $outdir/
+        done
+    done
+
+    
+    # -------------------------------------------------------------------------
+    # *** Move restart files to archive directory
+    # -------------------------------------------------------------------------
+    ### NEMO
+    outdir="${archive_dir}/restart/nemo/$(printf %03d $((leg_number)))"
+    mkdir -p ${outdir}
+
+    ns=$(printf %08d $(( leg_end_sec / nem_time_step_sec )))
+    for f in oce ice
+    do
+        mv ${exp_name}_${ns}_restart_${f}_????.nc ${outdir}
+    done
+
+    # -------------------------------------------------------------------------
+    # *** Move log files to archive directory
+    # -------------------------------------------------------------------------
+    outdir="${archive_dir}/log/$(printf %03d $((leg_number)))"
+    mkdir -p ${outdir}
+
+    for f in \
+        ocean.output time.step solver.stat
+    do
+        test -f ${f} && mv ${f} ${outdir}
+    done
+
+    # ---------------------------------------------------------------------
+    # *** Remove all leftover output files from previous legs
+    # ---------------------------------------------------------------------
+
+    ### NEMO files
+    rm -f ${exp_name}_??_????????_????????_{grid_U,grid_V,grid_W,grid_T,icemod,SBC,scalar,SBC_scalar}.nc
+    rm -f ${exp_name}_*restart_*nc
+    ns=$(printf %08d $(( leg_start_sec / nem_time_step_sec )))
+    for (( n=0 ; n<nem_numproc ; n++ ))
+    do
+        np=$(printf %04d ${n})
+        rm -f restart_oce_${np}.nc
+        rm -f restart_ice_${np}.nc
+    done
+
+    # -------------------------------------------------------------------------
+    # *** Write the restart control file
+    # -------------------------------------------------------------------------
+    shopt -u nullglob
+
+    tr=$(date -d "0 -$time_begin sec + $time_end sec" +%T)
+    current_date=$(date +'%F %T')
+    {
+      echo "#"
+      echo "# Finished leg at ${current_date} after ${tr} (hh:mm:ss)" 
+      echo "leg_number=${leg_number}"
+      echo "leg_start_date=\"${leg_start_date}\""
+      echo "leg_end_date=\"${leg_end_date}\""
+    } | tee -a "${info_file}"
+
+    # Need to reset force_run_from_scratch in order to avoid destroying the next leg
+    force_run_from_scratch=false
+    special_restart=false
+
+done # Loop over legs
+
+cd ${start_dir}
+
+cd - >/dev/null
+[[ $@ == *noresubmit* ]] && exit 0
+if (( leg_end_epoch < run_end_epoch )) ; then
+    echo "Leg end earlier than end of simulation."
+    echo "Submitting another job."
+    if [[ "$@" == *"local"* ]] ; then
+        exec "$0" "$@"
+    elif [ "qsub" == *sbatch* ] ; then
+        sbatch $0 $@ | awk '{print $4}' >> ${run_dir}/.coral_jobs
+    elif [[ $USER == "vsc"* ]]; then
+        if [ -f ${PBS_O_WORKDIR-$PWD}/tmp/credits_file ]; then
+            credits=`cat ${PBS_O_WORKDIR-$PWD}/tmp/credits_file`
+        else
+            credits_tmp=`grep "account string:" ${PBS_O_WORKDIR-$PWD}/*.o* -R | sed 's/^.*: //'`
+            credits=`echo $credits_tmp | awk '{ print $1 }'`
+
+            cd ${start_dir}
+            mkdir -p tmp
+            echo ${credits} > tmp/credits_file
+        fi
+        if [ -f ${PBS_O_WORKDIR-$PWD}/tmp/script_name ]; then
+            script_name=`cat ${PBS_O_WORKDIR-$PWD}/tmp/script_name`
+        else
+            script_name="run.sh"
+        fi
+        ssh login2 "cd $start_dir; qsub -A $credits $script_name | tee -a coral_jobs"
+        sleep 2
+        jobid=`cat coral_jobs`
+        rm -f coral_jobs
+        jobid=${jobid%%.*}
+        echo "${jobid}" >> "${run_dir}"/.coral_jobs ;
+    else
+        "qsub" -v PBS_OPTIONS="$@" "$0" | tee -a coral_jobs
+        sleep 2
+        jobid=`cat coral_jobs`
+        rm -f coral_jobs
+        jobid=${jobid%.*}
+        echo "${jobid}" >> "${run_dir}"/.coral_jobs
+    fi
+else
+    if [[ $USER == "vsc"* ]]; then
+        rm -rf ${start_dir}/tmp
+    fi
+
+    rm -rf ${run_dir}/../cclm_out_tmp
+    echo "Nothing left to do. Exiting." 
+
+cat << "EOF" 
+   _____ _                 _       _   _                                         _      _           _ 
+  / ____(_)               | |     | | (_)                                       | |    | |         | |
+ | (___  _ _ __ ___  _   _| | __ _| |_ _  ___  _ __     ___ ___  _ __ ___  _ __ | | ___| |_ ___  __| |
+  \___ \| | '_ ` _ \| | | | |/ _` | __| |/ _ \| '_ \   / __/ _ \| '_ ` _ \| '_ \| |/ _ \ __/ _ \/ _` |
+  ____) | | | | | | | |_| | | (_| | |_| | (_) | | | | | (_| (_) | | | | | | |_) | |  __/ ||  __/ (_| |
+ |_____/|_|_| |_| |_|\__,_|_|\__,_|\__|_|\___/|_| |_|  \___\___/|_| |_| |_| .__/|_|\___|\__\___|\__,_|
+                                                                          | |                         
+                                                                          |_|                         
+EOF
+
+fi
+
+exit 0
+
+
+    

Kaikkia tiedostoja ei voida näyttää, sillä liian monta tiedostoa muuttui tässä diffissä