|
|
@@ -0,0 +1,784 @@
|
|
|
+# Function leap days calculates the number of leap days (29th of Februrary) in
|
|
|
+# a time intervall between two dates.
|
|
|
+#
|
|
|
+# Usage leap_days START_DATE END_DATE
|
|
|
+function leap_days()
|
|
|
+{
|
|
|
+ local ld=0
|
|
|
+ local frstYYYY=$(date -ud "$1" +%Y)
|
|
|
+ local lastYYYY=$(date -ud "$2" +%Y)
|
|
|
+
|
|
|
+ set +e
|
|
|
+
|
|
|
+ # Check first year for leap day between start and end date
|
|
|
+ $(date -ud "${frstYYYY}-02-29" > /dev/null 2>&1) \
|
|
|
+ && (( $(date -ud "$1" +%s) < $(date -ud "${frstYYYY}-03-01" +%s) )) \
|
|
|
+ && (( $(date -ud "$2" +%s) > $(date -ud "${lastYYYY}-02-28" +%s) )) \
|
|
|
+ && (( ld++ ))
|
|
|
+
|
|
|
+ # Check intermediate years for leap day
|
|
|
+ for (( y=(( ${frstYYYY}+1 )); y<=(( ${lastYYYY}-1 )); y++ ))
|
|
|
+ do
|
|
|
+ $(date -ud "$y-02-29" > /dev/null 2>&1) && (( ld++ ))
|
|
|
+ done
|
|
|
+
|
|
|
+ # Check last year (if different from first year) for leap day between start
|
|
|
+ # and end date
|
|
|
+ (( $lastYYYY > $frstYYYY )) \
|
|
|
+ && $(date -ud "${lastYYYY}-02-29" > /dev/null 2>&1) \
|
|
|
+ && (( $(date -ud "$1" +%s) < $(date -ud "${frstYYYY}-03-01" +%s) )) \
|
|
|
+ && (( $(date -ud "$2" +%s) > $(date -ud "${lastYYYY}-02-28" +%s) )) \
|
|
|
+ && (( ld++ ))
|
|
|
+
|
|
|
+ set -e
|
|
|
+
|
|
|
+ echo "$ld"
|
|
|
+}
|
|
|
+
|
|
|
+# Function for loading modules
|
|
|
+function load_modules_isfcpl () {
|
|
|
+ if [[ ${USER} == "vsc"????? ]]; then
|
|
|
+
|
|
|
+ # quick and dirty patch - only BRENIAC will work while waiting for something to distinguish between both machines (include on computing nodes)
|
|
|
+
|
|
|
+ # if [[ ${HOSTNAME} == "tier2"* ]]; then
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+ # if [[ "${name_icesheet_model}" == "fETISh" ]]; then
|
|
|
+ # module purge
|
|
|
+ # module load Python
|
|
|
+ # PYTHONPATH="${HOME}/.local/bin:${HOME}/.local/lib/python3.7/site-packages:\${PYTHONPATH}"
|
|
|
+ # module load CDO
|
|
|
+ # echo ${PYTHONPATH} > ${start_dir}/ppath
|
|
|
+ # echo "genius and fetish detected" > ${start_dir}/tmpout
|
|
|
+ # else
|
|
|
+ # echo "On Genius, ice sheet models other than fETISh not implemented yet" > ${start_dir}/tmpout
|
|
|
+ # exit 2
|
|
|
+ # fi
|
|
|
+
|
|
|
+ # elif [[ ${HOSTNAME} == "login"? ]]; then
|
|
|
+ if [[ "${name_icesheet_model}" == "fETISh" ]]; then
|
|
|
+ module purge
|
|
|
+ module load Python/3.7.2-intel-2018a CDO/1.9.5-intel-2018a
|
|
|
+ [ ! -d "${VSC_HOME}/.local/lib/python3.7/site-packages/netCDF4" ] && pip install --user netCDF4
|
|
|
+ [ ! -d "${VSC_HOME}/.local/lib/python3.7/site-packages/numpy" ] && pip install --target=${VSC_HOME}/.local/lib/python3.7/site-packages numpy
|
|
|
+ [ ! -d "${VSC_HOME}/.local/lib/python3.7/site-packages/gsw" ] && pip install --user gsw
|
|
|
+
|
|
|
+ PYTHONPATH="${HOME}/.local/bin:${HOME}/.local/lib/python3.7/site-packages:\${PYTHONPATH}"
|
|
|
+ echo ${PYTHONPATH} > ${start_dir}/ppath
|
|
|
+ echo "breniac and fetish detected" > ${start_dir}/tmpout
|
|
|
+ else
|
|
|
+ echo "On Breniac, ice sheet models other than fETISh not implemented yet" > ${start_dir}/tmpout
|
|
|
+ exit 2
|
|
|
+ fi
|
|
|
+
|
|
|
+ # else
|
|
|
+ # echo "You seem to be on a VSC machine that is neither Genius nor BRENIAC. This has not been implemented yet. Exiting"
|
|
|
+ # exit 2
|
|
|
+ # fi
|
|
|
+
|
|
|
+ elif [[ ${HOSTNAME} == "lm3"* ]]; then
|
|
|
+ if [[ "${name_icesheet_model}" == "fETISh" ]]; then
|
|
|
+ echo "lemaitre3 and fetish detected" > ${start_dir}/tmpout
|
|
|
+ module purge
|
|
|
+ module load releases/2018a CDO/1.8.2-intel-2018a Python/3.6.4-intel-2018a
|
|
|
+ else
|
|
|
+ echo "On lemaitre3, ice sheet models other than fETISh not implemented yet" > ${start_dir}/tmpout
|
|
|
+ exit 2
|
|
|
+ fi
|
|
|
+
|
|
|
+ else
|
|
|
+ echo "hostname undetected. not implemented yet." > ${start_dir}/tmpout
|
|
|
+ exit 2
|
|
|
+ fi
|
|
|
+}
|
|
|
+
|
|
|
+[[ $@ == *verbose* ]] && set -x
|
|
|
+
|
|
|
+module load ${module_list:?}
|
|
|
+export LD_LIBRARY_PATH=${LD_LIBRARY_PATH:+${LD_LIBRARY_PATH}:}"${extralibs_list}"
|
|
|
+
|
|
|
+mkdir -p ${run_dir}
|
|
|
+cd ${run_dir}
|
|
|
+
|
|
|
+# Prepare iscpl
|
|
|
+if [ "${isfcpl}" == "1" ]; then
|
|
|
+
|
|
|
+ if [[ ${USER} == "vsc"????? ]]; then
|
|
|
+
|
|
|
+ if [[ ${VSC_INSTITUTE_CLUSTER} == "breniac" ]]; then
|
|
|
+ echo "BRENIAC and iscpl detected. OK."
|
|
|
+ else
|
|
|
+ echo "VSC clusters other than BRENIAC not supported for isfcpl."
|
|
|
+ exit 2
|
|
|
+ fi
|
|
|
+
|
|
|
+ elif [[ ${HOSTNAME} == "lm3"* ]]; then
|
|
|
+ echo "Lemaitre3 and iscpl detected. OK."
|
|
|
+ else
|
|
|
+ echo "You have asked for iscpl, but I have not recognized genius nor lemaitre3. Exiting."
|
|
|
+ exit 2
|
|
|
+ fi
|
|
|
+
|
|
|
+ if [ "${name_icesheet_model}" != "fETISh" ]; then
|
|
|
+ echo "only \"fETISh\" is known as an ice sheet model. exiting."
|
|
|
+ exit 2
|
|
|
+ fi
|
|
|
+
|
|
|
+ length_cut_pref=$(( ${#exp_name} + 4 ))
|
|
|
+ rebuild_nemo_exe=${nemo_src_dir}/TOOLS/REBUILD_NEMO/rebuild_nemo
|
|
|
+ if [ -f ${start_dir}/isfcpl.track ]; then
|
|
|
+ source ${start_dir}/isfcpl.track
|
|
|
+ fi
|
|
|
+
|
|
|
+ iscpl_nm=.true.
|
|
|
+
|
|
|
+ if [[ ${xio_exe_file} == *"xios-2"* ]]; then
|
|
|
+ is_xios2=1
|
|
|
+ else
|
|
|
+ is_xios2=0
|
|
|
+ fi
|
|
|
+
|
|
|
+else
|
|
|
+ iscpl_nm=.false.
|
|
|
+
|
|
|
+fi
|
|
|
+
|
|
|
+run_start_epoch=$(date -u -d"${run_start_date}" +%s)
|
|
|
+run_start_date_cos="${run_start_date//-}00"
|
|
|
+run_start_date=$(date -uR -d "${run_start_date}")
|
|
|
+run_end_date="${run_start_date} + ${run_duration:?}"
|
|
|
+run_end_date=$(date -uR -d "${run_end_date}")
|
|
|
+run_end_epoch=$(date -u -d"${run_end_date}" +%s)
|
|
|
+run_end_date_cos="$(date -u -d"${run_end_date}" +%Y%m%d%H)"
|
|
|
+
|
|
|
+
|
|
|
+if [[ ${special_rs_nemo} == "true" && ${special_restart} == "true" ]]; then
|
|
|
+ echo "ERROR: both special_rs_nemo and special_restart are true. only can be true (at most). exit".
|
|
|
+ exit 2
|
|
|
+fi
|
|
|
+
|
|
|
+
|
|
|
+for (( ; run_num_legs>0 ; run_num_legs-- ))
|
|
|
+do
|
|
|
+
|
|
|
+ # Check for restart information file and set the current leg start date
|
|
|
+ # Ignore restart information file if force_run_from_scratch is true
|
|
|
+ if ! [ -r ${info_file} ]
|
|
|
+ then
|
|
|
+ leg_is_restart=false
|
|
|
+ leg_start_date=${run_start_date}
|
|
|
+ leg_start_date_cos=${run_start_date_cos}
|
|
|
+ leg_number=1
|
|
|
+ echo "leg_is_restart=false"
|
|
|
+ else
|
|
|
+ leg_is_restart=true
|
|
|
+ . ./${info_file}
|
|
|
+ leg_start_date=${leg_end_date}
|
|
|
+ leg_start_date_cos="$(date -u -d"${leg_start_date}" +%Y%m%d%H)"
|
|
|
+ leg_number=$((leg_number+1))
|
|
|
+ echo "leg_is_restart=true"
|
|
|
+ fi
|
|
|
+
|
|
|
+ # Compute the end date of the current leg
|
|
|
+ if [ -n "${rst_freq}" ]
|
|
|
+ then
|
|
|
+ if (($(date -u -d "${leg_start_date:?} + ${rst_freq}" +%s) > $(date -u -d "${leg_start_date:?} + ${run_duration}" +%s) )); then
|
|
|
+ rst_freq=$run_duration
|
|
|
+ fi
|
|
|
+ leg_end_date=$(date -uR -d "${leg_start_date} + ${rst_freq}")
|
|
|
+ leg_end_date_cos="$(date -u -d"${leg_end_date}" +%Y%m%d%H)"
|
|
|
+ else
|
|
|
+ leg_end_date=${run_end_date}
|
|
|
+ leg_end_date_cos=${run_end_date_cos}
|
|
|
+ fi
|
|
|
+
|
|
|
+ if [ $(date -u -d "${leg_end_date}" +%s) -gt $(date -u -d "${run_end_date}" +%s) ]
|
|
|
+ then
|
|
|
+ leg_end_date=${run_end_date}
|
|
|
+ leg_end_date_cos=${run_end_date_cos}
|
|
|
+ fi
|
|
|
+
|
|
|
+ # Some time variables needed later
|
|
|
+ leg_start_epoch=$(date -u -d "${leg_start_date}" +%s)
|
|
|
+ leg_end_epoch=$(date -u -d "${leg_start_date:?} + ${rst_freq:=$run_duration}" +%s)
|
|
|
+ leg_length_sec=$(( $(date -u -d "${leg_end_date}" +%s) - $(date -u -d "${leg_start_date}" +%s) ))
|
|
|
+ leg_start_sec=$(( $(date -u -d "${leg_start_date}" +%s) - $(date -u -d "${run_start_date}" +%s) ))
|
|
|
+ leg_end_sec=$(( $(date -u -d "${leg_end_date}" +%s) - $(date -u -d "${run_start_date}" +%s) ))
|
|
|
+ leg_start_date_yyyymmdd=$(date -u -d "${leg_start_date}" +%Y%m%d)
|
|
|
+ leg_start_date_yyyy=$(date -u -d "${leg_start_date}" +%Y)
|
|
|
+ leg_end_date_yyyy=$(date -u -d "${leg_end_date}" +%Y)
|
|
|
+
|
|
|
+ echo "leg_number: $leg_number"
|
|
|
+ echo "leg_start_date: $leg_start_date"
|
|
|
+ echo "leg_end_date: $leg_end_date"
|
|
|
+
|
|
|
+ # Additional time variables for iscpl
|
|
|
+ if (( isfcpl == 1 )); then
|
|
|
+ leg_beforestart_epoch=$(( ${leg_start_epoch} - 43200 ))
|
|
|
+ leg_beforestart_date=$(date -uR -d@"${leg_beforestart_epoch}")
|
|
|
+ leg_almostend_epoch=$(( ${leg_end_epoch} - 43200 ))
|
|
|
+ leg_almostend_date=$(date -uR -d@"${leg_almostend_epoch}")
|
|
|
+ nextleg_start_epoch=$(( ${leg_end_epoch} + 43200 ))
|
|
|
+ nextleg_start_date=$(date -uR -d@"${nextleg_start_epoch}")
|
|
|
+
|
|
|
+ run_start_date_yyyymmdd=$(date -u -d "${run_start_date}" +%Y%m%d) # FIXME appears unused
|
|
|
+ run_end_date_yyyymmdd=$(date -u -d "${run_end_date}" +%Y%m%d) # FIXME appears unused
|
|
|
+ leg_start_date_yyyymmdd=$(date -u -d "${leg_start_date}" +%Y%m%d) # FIXME appears unused
|
|
|
+ leg_almostend_date_yyyymmdd=$(date -u -d "${leg_almostend_date}" +%Y%m%d) # FIXME appears unused
|
|
|
+ nextleg_start_date_yyyymmdd=$(date -u -d "${nextleg_start_date}" +%Y%m%d) # FIXME appears unused
|
|
|
+ leg_beforestart_date_yyyymmdd=$(date -u -d "${leg_beforestart_date}" +%Y%m%d) # FIXME appears unused
|
|
|
+ fi
|
|
|
+
|
|
|
+ # Check whether there's actually time left to simulate - exit otherwise
|
|
|
+ if [ ${leg_length_sec} -le 0 ]
|
|
|
+ then
|
|
|
+ info "Leg start date equal to or after end of simulation."
|
|
|
+ info "Nothing left to do. Exiting."
|
|
|
+ exit 0
|
|
|
+ fi
|
|
|
+
|
|
|
+ # -------------------------------------------------------------------------
|
|
|
+ # *** Prepare the run directory for a run from scratch
|
|
|
+ # -------------------------------------------------------------------------
|
|
|
+ if ! $leg_is_restart
|
|
|
+ then
|
|
|
+ # ---------------------------------------------------------------------
|
|
|
+ # *** Check if run dir is empty. If not, and if we are allowed to do so
|
|
|
+ # by ${force_run_from_scratch}, remove everything
|
|
|
+ # ---------------------------------------------------------------------
|
|
|
+ echo "pwd: $PWD"
|
|
|
+ if $(ls * >& /dev/null)
|
|
|
+ then
|
|
|
+ if ${force_run_from_scratch}
|
|
|
+ then
|
|
|
+ rm -fr ${run_dir}/*
|
|
|
+ else
|
|
|
+ echo "error: run directory not empty and \$force_run_from_scratch not set."
|
|
|
+ exit 0
|
|
|
+ fi
|
|
|
+ fi
|
|
|
+
|
|
|
+ # ---------------------------------------------------------------------
|
|
|
+ # *** Creates some directories
|
|
|
+ # ---------------------------------------------------------------------
|
|
|
+ mkdir -p ${out_dir}
|
|
|
+
|
|
|
+ # ---------------------------------------------------------------------
|
|
|
+ # *** Copy executables and coupling-related files
|
|
|
+ # ---------------------------------------------------------------------
|
|
|
+ cd ${start_dir}
|
|
|
+ cp -u xios_config/*xml "${run_dir}"
|
|
|
+ cp -u namelists/* "${run_dir}"
|
|
|
+ cd ${run_dir}
|
|
|
+ cp -u "${nem_exe_file:?}" .
|
|
|
+ cp -u "${xio_exe_file:?}" .
|
|
|
+ if (( isfcpl == 1 )) ; then
|
|
|
+ cp -u ${icesheet_exe_file} .
|
|
|
+ cp -u ${isfcpl_dir}/* .
|
|
|
+ ln -sf ${start_dir}/isfcpl.track ${run_dir}
|
|
|
+ mkdir -p ${out_dir}/isfcpl_tmp
|
|
|
+ fi
|
|
|
+
|
|
|
+ # ---------------------------------------------------------------------
|
|
|
+ # *** Link ic, forcing and shared files for NEMO
|
|
|
+ # ---------------------------------------------------------------------
|
|
|
+ [[ ! -f EMPave_old.dat ]] && echo " 0 0.0000000000000000E+00 0.0000000000000000E+00" > EMPave_old.dat
|
|
|
+ for file in "${ic_files[@]}"; do
|
|
|
+ [[ ! -e ${file#*> } ]] && ln -sf $(sed 's/ *=> */ /' <<< "${ini_data_dir}/${ic_subdir}/${nem_grid}/$file")
|
|
|
+ done
|
|
|
+ for file in "${forcing_files[@]}"; do
|
|
|
+ [[ ! -e ${file#*> } || "$file" == \** ]] && ln -sf $(sed 's/ *=> */ /' <<< "${ini_data_dir}/${forcing_subdir}/${nem_forcing_set}/$file")
|
|
|
+ done
|
|
|
+ for file in "${shared_files[@]}"; do
|
|
|
+ [[ ! -e ${file#*> } ]] && ln -sf $(sed 's/ *=> */ /' <<< "${shared_dir}/$file")
|
|
|
+ done
|
|
|
+
|
|
|
+ # ---------------------------------------------------------------------
|
|
|
+ # *** Prepare NEMO - ice shelf coupling
|
|
|
+ # ---------------------------------------------------------------------
|
|
|
+ if [ "${isfcpl}" == "1" ]; then
|
|
|
+
|
|
|
+ if (( leg_number == 1 )); then
|
|
|
+
|
|
|
+ mkdir -p ${archive_dir}/isfcpl
|
|
|
+ mkdir -p ${archive_dir}/${name_icesheet_model}
|
|
|
+ rm -f ${start_dir}/isfcpl.track
|
|
|
+
|
|
|
+
|
|
|
+ isfcpl_window_days=$(echo "(${n_res_perisfcpl} * ${leg_length_sec}) / 86400" | bc)
|
|
|
+ isfcpl_d30=$(echo "${isfcpl_window_days} / 30" | bc -l)
|
|
|
+ n_month_isfcpl=$(echo ${isfcpl_d30} | awk '{print int($1+0.5)}')
|
|
|
+
|
|
|
+ n_month_full_run=$(( $(( 12 * $(( ${run_end_date_yyyymmdd::4} - ${run_start_date_yyyymmdd::4} )) )) + $(( ${run_end_date_yyyymmdd:4:2} - ${run_start_date_yyyymmdd:4:2} )) ))
|
|
|
+
|
|
|
+ if (( $(( ${n_month_full_run} % ${n_month_isfcpl} )) != 0 )); then
|
|
|
+ echo "n_month_full_run=${n_month_full_run} is not a multiple of n_month_isfcpl=${n_month_isfcpl}."
|
|
|
+ exit 2
|
|
|
+ else
|
|
|
+ n_isfcpl_window=$(echo "${n_month_full_run} / ${n_month_isfcpl}" | bc)
|
|
|
+ fi
|
|
|
+
|
|
|
+ tmp=`readlink -f isf_draft_meter.nc`
|
|
|
+ tmp2=`readlink -f bathy_meter.nc`
|
|
|
+
|
|
|
+ if [ "${tmp}" != "${tmp2}" ]; then
|
|
|
+ echo "Error. At initialization, isf_draft_meter.nc and bathy_meter.nc do not point to the same file."
|
|
|
+ exit 1
|
|
|
+ fi
|
|
|
+
|
|
|
+ init_bathy_file=`basename ${tmp}`
|
|
|
+ cp -f ${tmp} ${archive_dir}/isfcpl
|
|
|
+ cp -f ${tmp} ${run_dir}/init_NEMO_bathy.nc
|
|
|
+
|
|
|
+ ln -sf ${icesheet_files_dir}/* ./
|
|
|
+ ln -sf ${isfcpl_files_dir}/* ./
|
|
|
+
|
|
|
+ date_beg_isfcpl="${run_start_date_yyyymmdd}"
|
|
|
+
|
|
|
+ printf "# Information for the ice sheet model\n" > ${start_dir}/isfcpl.track
|
|
|
+ printf "run_name=${exp_name}\n" >> ${start_dir}/isfcpl.track
|
|
|
+ printf "initial_date=${run_start_date_yyyymmdd}\n" >> ${start_dir}/isfcpl.track
|
|
|
+ printf "n_month_isfcpl=${n_month_isfcpl}\n" >> ${start_dir}/isfcpl.track
|
|
|
+ printf "n_isfcpl_window=${n_isfcpl_window}\n" >> ${start_dir}/isfcpl.track
|
|
|
+ printf "date_beg_isfcpl=${date_beg_isfcpl}\n" >> ${start_dir}/isfcpl.track
|
|
|
+ printf "date_end_isfcpl=YYYYMMDD\n" >> ${start_dir}/isfcpl.track
|
|
|
+ printf "ice_atm_cpl=${ice_atm_cpl}\n\n" >> ${start_dir}/isfcpl.track
|
|
|
+
|
|
|
+ printf "# Information for NEMO to keep up\n" >> ${start_dir}/isfcpl.track
|
|
|
+ printf "init_bathy=1\n" >> ${start_dir}/isfcpl.track
|
|
|
+ printf "curr_bathy_file=${init_bathy_file}\n" >> ${start_dir}/isfcpl.track
|
|
|
+
|
|
|
+ init_bathy=1
|
|
|
+ curr_bathy_file=${init_bathy_file}
|
|
|
+
|
|
|
+ fi
|
|
|
+
|
|
|
+ ln -sf ${curr_bathy_file} ${run_dir}/bathy_meter.nc
|
|
|
+ ln -sf ${curr_bathy_file} ${run_dir}/isf_draft_meter.nc
|
|
|
+
|
|
|
+
|
|
|
+ fi # if [ "${iscpl}" == 1 ]
|
|
|
+
|
|
|
+ # ---------------------------------------------------------------------
|
|
|
+ # *** Copy potential special restarts
|
|
|
+ # ---------------------------------------------------------------------
|
|
|
+ if $special_restart
|
|
|
+ then
|
|
|
+
|
|
|
+ ### Getting the right output folders
|
|
|
+ leg_start_date_tmp=$leg_start_date
|
|
|
+ leg_end_date_tmp=$leg_end_date
|
|
|
+ leg_number_tmp=$leg_number
|
|
|
+ rsync -avz ${run_dir}/../../${special_restart_from}/rundir/${info_file} ${run_dir}
|
|
|
+ special_date=$(date -uR -d "${special_restart_date}")
|
|
|
+ sed -i "/$special_date/q" ${run_dir}/${info_file}
|
|
|
+ . ${run_dir}/${info_file}
|
|
|
+ special_restart_leg=$(printf %03d $((leg_number)))
|
|
|
+
|
|
|
+ echo "special_date: $special_date"
|
|
|
+ echo "special_restart_leg: $special_restart_leg"
|
|
|
+ echo "special_restart_from: $special_restart_from"
|
|
|
+
|
|
|
+ ### NEMO
|
|
|
+ indir="${run_dir}/../../${special_restart_from}/archive/restart/nemo/$(printf %03d $((special_restart_leg)))"
|
|
|
+ cd ${indir}
|
|
|
+ for f in *.nc; do
|
|
|
+ nf=`echo "${f/$special_restart_from/$exp_name}"`
|
|
|
+ echo "copy $f to ${run_dir}/$nf"
|
|
|
+ cp $f ${run_dir}/$nf
|
|
|
+ done
|
|
|
+ cd -
|
|
|
+ cd ${run_dir}
|
|
|
+ echo "pwd: $PWD"
|
|
|
+ for f in ${exp_name}_????????_restart_???_????.nc; do
|
|
|
+ nf=`echo ${f: -19}`
|
|
|
+ echo "linking $f to ${run_dir}/$nf"
|
|
|
+ ln -s $f $nf
|
|
|
+ done
|
|
|
+
|
|
|
+ cd ${run_dir}
|
|
|
+ rm -rf ${info_file}
|
|
|
+ leg_start_date=$leg_start_date_tmp
|
|
|
+ leg_end_date=$leg_end_date_tmp
|
|
|
+ leg_number=$leg_number_tmp
|
|
|
+
|
|
|
+ elif ${special_rs_nemo}
|
|
|
+ then
|
|
|
+
|
|
|
+ dir_rs=`dirname ${special_rs_nemo_stencil}`
|
|
|
+ base_rs=`basename ${special_rs_nemo_stencil}`
|
|
|
+
|
|
|
+ n_oce_rs=`find ${dir_rs} -name "${base_rs}oce_????.nc" | wc -l`
|
|
|
+ if(( n_oce_rs != nem_numproc )); then
|
|
|
+ echo "in special_rs_nemo, not the right amount of ocean restart files."
|
|
|
+ echo "${n_oce_rs} instead of ${nem_numproc}. exiting."
|
|
|
+ exit 2
|
|
|
+ fi
|
|
|
+
|
|
|
+ n_ice_rs=`find ${dir_rs} -name "${base_rs}ice_????.nc" | wc -l`
|
|
|
+ if(( n_ice_rs != nem_numproc )); then
|
|
|
+ echo "in special_rs_nemo, not the right amount of ice restart files."
|
|
|
+ echo "${n_ice_rs} instead of ${nem_numproc}. exiting."
|
|
|
+ exit 2
|
|
|
+ fi
|
|
|
+
|
|
|
+ for (( n=0 ; n<nem_numproc ; n++ ))
|
|
|
+ do
|
|
|
+ np=$(printf %04d ${n})
|
|
|
+ ln -fs ${special_rs_nemo_stencil}oce_${np}.nc restart_oce_${np}.nc
|
|
|
+ ln -fs ${special_rs_nemo_stencil}ice_${np}.nc restart_ice_${np}.nc
|
|
|
+ done
|
|
|
+
|
|
|
+ fi # if $special_restart
|
|
|
+
|
|
|
+
|
|
|
+ else # if ! $leg_is_restart
|
|
|
+
|
|
|
+ # ---------------------------------------------------------------------
|
|
|
+ # *** Creates some directories
|
|
|
+ # ---------------------------------------------------------------------
|
|
|
+ mkdir -p ${out_dir}
|
|
|
+
|
|
|
+ if [ "${isfcpl}" == "1" ]; then
|
|
|
+ ln -sf ${curr_bathy_file} ${run_dir}/bathy_meter.nc
|
|
|
+ ln -sf ${curr_bathy_file} ${run_dir}/isf_draft_meter.nc
|
|
|
+ fi
|
|
|
+
|
|
|
+ # ---------------------------------------------------------------------
|
|
|
+ # *** Copy restart files in run directory
|
|
|
+ # ---------------------------------------------------------------------
|
|
|
+ cd ${run_dir}
|
|
|
+ ### NEMO
|
|
|
+ outdir="${archive_dir}/restart/nemo/$(printf %03d $((leg_number-1)))"
|
|
|
+
|
|
|
+
|
|
|
+ ns=$(printf %08d $(( leg_start_sec / nem_time_step_sec )))
|
|
|
+ for f in oce ice
|
|
|
+ do
|
|
|
+ echo "${outdir}/${exp_name}_${ns}_restart_${f}_????.nc"
|
|
|
+ ln -sf ${outdir}/${exp_name}_${ns}_restart_${f}_????.nc ${run_dir}
|
|
|
+ done
|
|
|
+
|
|
|
+ for (( n=0 ; n<nem_numproc ; n++ ))
|
|
|
+ do
|
|
|
+ np=$(printf %04d ${n})
|
|
|
+ ln -fs ${exp_name}_${ns}_restart_oce_${np}.nc restart_oce_${np}.nc
|
|
|
+ ln -fs ${exp_name}_${ns}_restart_ice_${np}.nc restart_ice_${np}.nc
|
|
|
+ done
|
|
|
+
|
|
|
+ fi # ! $leg_is_restart
|
|
|
+
|
|
|
+ # -------------------------------------------------------------------------
|
|
|
+ # *** Update ocean namelist
|
|
|
+ # -------------------------------------------------------------------------
|
|
|
+ source build_namelist_cfg.sh > namelist_cfg
|
|
|
+
|
|
|
+ # -------------------------------------------------------------------------
|
|
|
+ # *** Build ice sheet launch
|
|
|
+ # -------------------------------------------------------------------------
|
|
|
+ if (( isfcpl == 1)); then
|
|
|
+ source ${icesheet_launch_template} > launch_icesheet.sh
|
|
|
+ chmod +x launch_icesheet.sh
|
|
|
+ fi
|
|
|
+
|
|
|
+
|
|
|
+ # -------------------------------------------------------------------------
|
|
|
+ # *** Launch experiment
|
|
|
+ # -------------------------------------------------------------------------
|
|
|
+
|
|
|
+
|
|
|
+ echo "START of mpirun"
|
|
|
+ time_begin=$(date +%s)
|
|
|
+ mpirun -np "${xio_numproc:?}" "${xio_exe_file:?}" : -np "${nem_numproc:?}" "${nem_exe_file:?}"
|
|
|
+ time_end=$(date +%s)
|
|
|
+ echo "END of: mpirun"
|
|
|
+
|
|
|
+ # -------------------------------------------------------------------------
|
|
|
+ # *** Ice sheet coupling
|
|
|
+ # -------------------------------------------------------------------------
|
|
|
+
|
|
|
+ if (( isfcpl == 1 )); then
|
|
|
+
|
|
|
+ cnt_res_isfcpl=$(( $(( ${leg_number} - 1 )) % ${n_res_perisfcpl} ))
|
|
|
+ fmt_res_isfcpl=$(printf "%02d" ${cnt_res_isfcpl})
|
|
|
+
|
|
|
+ in_fwfisf_stencil="${exp_name}_${frequency_fwfisf}_${leg_start_date_yyyymmdd}_${leg_almostend_date_yyyymmdd}_${suffix_fwfisf}"
|
|
|
+ out_fwfisf_stencil="${exp_name}_fwfisf_res"
|
|
|
+
|
|
|
+ out_fwfisf_curr="${out_fwfisf_stencil}${fmt_res_isfcpl}"
|
|
|
+
|
|
|
+ if (( is_xios2 == 0 )); then
|
|
|
+
|
|
|
+ module purge
|
|
|
+ module load ${module_list:?}
|
|
|
+ export LD_LIBRARY_PATH=${LD_LIBRARY_PATH:+${LD_LIBRARY_PATH}:}"${extralibs_list}"
|
|
|
+
|
|
|
+ /bin/bash ${rebuild_nemo_exe} ${in_fwfisf_stencil} ${xio_numproc}
|
|
|
+
|
|
|
+ module purge
|
|
|
+ load_modules_isfcpl
|
|
|
+
|
|
|
+ cdo selname,${name_var_fwfisf} ${in_fwfisf_stencil}.nc tmp_fwfisf.nc
|
|
|
+ cdo monmean tmp_fwfisf.nc ${out_fwfisf_curr}.nc
|
|
|
+
|
|
|
+ rm -f ${in_fwfisf_stencil}.nc tmp_fwfisf.nc
|
|
|
+
|
|
|
+ else
|
|
|
+
|
|
|
+ cdo selname,${name_var_fwfisf} ${in_fwfisf_stencil}.nc tmp_fwfisf.nc
|
|
|
+ cdo monmean tmp_fwfisf.nc ${out_fwfisf_curr}.nc
|
|
|
+
|
|
|
+ rm -f tmp_fwfisf.nc
|
|
|
+
|
|
|
+ fi
|
|
|
+
|
|
|
+ mv -f ${out_fwfisf_curr}.nc ${out_dir}/isfcpl_tmp/
|
|
|
+
|
|
|
+ if (( ${leg_number} % ${n_res_perisfcpl} == 0 )); then
|
|
|
+ date_end_isfcpl=${leg_almostend_date_yyyymmdd}
|
|
|
+ sed -i "s/.*date_end_isfcpl=.*/date_end_isfcpl=${date_end_isfcpl}/" ${start_dir}/isfcpl.track
|
|
|
+
|
|
|
+ module purge
|
|
|
+ module load ${module_list:?}
|
|
|
+ export LD_LIBRARY_PATH=${LD_LIBRARY_PATH:+${LD_LIBRARY_PATH}:}"${extralibs_list}"
|
|
|
+
|
|
|
+ /bin/bash ${rebuild_nemo_exe} mesh_mask ${nem_numproc}
|
|
|
+
|
|
|
+ module purge
|
|
|
+ load_modules_isfcpl
|
|
|
+
|
|
|
+ infile_ocean2ice="${exp_name}_ocean2ice_raw_${date_beg_isfcpl}-${date_end_isfcpl}.nc"
|
|
|
+ outfile_ocean2ice="${exp_name}_ocean2ice_${date_beg_isfcpl}-${date_end_isfcpl}.nc"
|
|
|
+
|
|
|
+ fwfisf_records=""
|
|
|
+ for((rs2=0; rs2<n_res_perisfcpl; rs2++)); do
|
|
|
+ fmt_rs2=$(printf "%02d" ${rs2})
|
|
|
+ curr_file="${out_dir}/isfcpl_tmp/${out_fwfisf_stencil}${fmt_rs2}.nc"
|
|
|
+ if [ -f ${curr_file} ] ; then
|
|
|
+ fwfisf_records="${fwfisf_records} ${curr_file}"
|
|
|
+ else
|
|
|
+ echo "${curr_file} is not in ${out_dir}/isfcpl_tmp. It is needed for NEMO - ice sheet coupling. Exiting"
|
|
|
+ exit 2
|
|
|
+ fi
|
|
|
+ done
|
|
|
+
|
|
|
+ cdo -O copy ${fwfisf_records} ${infile_ocean2ice}
|
|
|
+ rm -f ${fwfisf_records}
|
|
|
+
|
|
|
+ source build_pp_nemo_to_icesheet.sh > ./pp_nemo_to_${name_icesheet_model}.py
|
|
|
+ python pp_nemo_to_${name_icesheet_model}.py
|
|
|
+
|
|
|
+ mv -f ${infile_ocean2ice} ${archive_dir}/isfcpl/
|
|
|
+ mv -f mesh_mask.nc ${archive_dir}/isfcpl/${exp_name}_mesh_mask_${date_beg_isfcpl}-${date_end_isfcpl}.nc
|
|
|
+
|
|
|
+ module purge
|
|
|
+ module load ${icesheet_modules}
|
|
|
+ if [ "${icesheet_library_path}" != "" ]; then
|
|
|
+ export LD_LIBRARY_PATH="${icesheet_library_path}"
|
|
|
+ fi
|
|
|
+ /bin/bash ./launch_icesheet.sh
|
|
|
+
|
|
|
+ mv -f ${outfile_ocean2ice} ${archive_dir}/isfcpl
|
|
|
+ if (( ice_atm_cpl == 1 )); then
|
|
|
+ mv -f ${outfile_atm2ice} ${archive_dir}/isfcpl
|
|
|
+ fi
|
|
|
+
|
|
|
+ if [[ "${name_icesheet_model}" == "fETISh" ]]; then
|
|
|
+ find . -type l -name "${exp_name}_${name_icesheet_model}Run_??????_toto.mat" -exec rm -f {} \;
|
|
|
+ mv -f ${exp_name}_${name_icesheet_model}Run_??????_toto.mat ${archive_dir}/${name_icesheet_model}
|
|
|
+ ln -sf ${archive_dir}/${name_icesheet_model}/${exp_name}_${name_icesheet_model}Run_${date_end_isfcpl::-2}_toto.mat
|
|
|
+ else
|
|
|
+ echo "Ice-sheet models other than fETISh not implemented yet."
|
|
|
+ exit 2
|
|
|
+ fi
|
|
|
+
|
|
|
+ curr_geom_file="${exp_name}_ice2ocean_${date_end_isfcpl}.nc"
|
|
|
+ new_bathy_file="${exp_name}_bathymetry_isfdraft_${date_end_isfcpl}.nc"
|
|
|
+ source build_pp_icesheet_to_nemo.sh > pp_${name_icesheet_model}_to_nemo.py
|
|
|
+
|
|
|
+ if [ "${init_bathy}" == 0 ]; then
|
|
|
+ mv -f ${curr_bathy_file} ${archive_dir}/isfcpl
|
|
|
+ else
|
|
|
+ init_bathy=0
|
|
|
+ cp -f ${curr_bathy_file} ${archive_dir}/isfcpl
|
|
|
+ fi
|
|
|
+
|
|
|
+ module purge
|
|
|
+ load_modules_isfcpl
|
|
|
+ python pp_${name_icesheet_model}_to_nemo.py
|
|
|
+
|
|
|
+ mv -f ${curr_geom_file} ${archive_dir}/isfcpl
|
|
|
+
|
|
|
+ curr_bathy_file=${new_bathy_file}
|
|
|
+ ln -sf ${curr_bathy_file} ${run_dir}/bathy_meter.nc
|
|
|
+ ln -sf ${curr_bathy_file} ${run_dir}/isf_draft_meter.nc
|
|
|
+
|
|
|
+ printf "# Information for the ice sheet model\n" > ${start_dir}/isfcpl.track
|
|
|
+ printf "run_name=${exp_name}\n" >> ${start_dir}/isfcpl.track
|
|
|
+ printf "initial_date=${run_start_date_yyyymmdd}\n" >> ${start_dir}/isfcpl.track
|
|
|
+ printf "n_month_isfcpl=${n_month_isfcpl}\n" >> ${start_dir}/isfcpl.track
|
|
|
+ printf "n_isfcpl_window=${n_isfcpl_window}\n" >> ${start_dir}/isfcpl.track
|
|
|
+ printf "date_beg_isfcpl=${nextleg_start_date_yyyymmdd}\n" >> ${start_dir}/isfcpl.track
|
|
|
+ printf "date_end_isfcpl=YYYYMMDD\n" >> ${start_dir}/isfcpl.track
|
|
|
+ printf "ice_atm_cpl=${ice_atm_cpl}\n\n" >> ${start_dir}/isfcpl.track
|
|
|
+
|
|
|
+ printf "# Information for NEMO to keep up\n" >> ${start_dir}/isfcpl.track
|
|
|
+ printf "init_bathy=${init_bathy}\n" >> ${start_dir}/isfcpl.track
|
|
|
+ printf "curr_bathy_file=${curr_bathy_file}\n" >> ${start_dir}/isfcpl.track
|
|
|
+
|
|
|
+ module purge
|
|
|
+ module load ${module_list:?}
|
|
|
+ export LD_LIBRARY_PATH=${LD_LIBRARY_PATH:+${LD_LIBRARY_PATH}:}"${extralibs_list}"
|
|
|
+
|
|
|
+ fi
|
|
|
+ fi
|
|
|
+
|
|
|
+ # -------------------------------------------------------------------------
|
|
|
+ # *** Move output files to archive directory
|
|
|
+ # -------------------------------------------------------------------------
|
|
|
+ ### NEMO
|
|
|
+ outdir="${archive_dir}/output/nemo/$(printf %03d $((leg_number)))"
|
|
|
+ mkdir -p ${outdir}
|
|
|
+
|
|
|
+ for v in grid_U grid_V grid_W grid_T icemod SBC scalar SBC_scalar diad_T \
|
|
|
+ grid_T_2D grid_U_2D grid_V_2D grid_W_2D grid_T_3D grid_U_3D grid_V_3D grid_W_3D \
|
|
|
+ grid_1point grid_T_3D_ncatice vert_sum \
|
|
|
+ grid_ptr_W_3basin_3D grid_ptr_T_3basin_2D grid_ptr_T_2D \
|
|
|
+ zoom_700_sum zoom_300_sum zoom_2000_sum
|
|
|
+ do
|
|
|
+ for f in ${exp_name}_*_????????_????????_*${v}*.nc
|
|
|
+ do
|
|
|
+ test -f $f && mv $f $outdir/
|
|
|
+ done
|
|
|
+ done
|
|
|
+
|
|
|
+
|
|
|
+ # -------------------------------------------------------------------------
|
|
|
+ # *** Move restart files to archive directory
|
|
|
+ # -------------------------------------------------------------------------
|
|
|
+ ### NEMO
|
|
|
+ outdir="${archive_dir}/restart/nemo/$(printf %03d $((leg_number)))"
|
|
|
+ mkdir -p ${outdir}
|
|
|
+
|
|
|
+ ns=$(printf %08d $(( leg_end_sec / nem_time_step_sec )))
|
|
|
+ for f in oce ice
|
|
|
+ do
|
|
|
+ mv ${exp_name}_${ns}_restart_${f}_????.nc ${outdir}
|
|
|
+ done
|
|
|
+
|
|
|
+ # -------------------------------------------------------------------------
|
|
|
+ # *** Move log files to archive directory
|
|
|
+ # -------------------------------------------------------------------------
|
|
|
+ outdir="${archive_dir}/log/$(printf %03d $((leg_number)))"
|
|
|
+ mkdir -p ${outdir}
|
|
|
+
|
|
|
+ for f in \
|
|
|
+ ocean.output time.step solver.stat
|
|
|
+ do
|
|
|
+ test -f ${f} && mv ${f} ${outdir}
|
|
|
+ done
|
|
|
+
|
|
|
+ # ---------------------------------------------------------------------
|
|
|
+ # *** Remove all leftover output files from previous legs
|
|
|
+ # ---------------------------------------------------------------------
|
|
|
+
|
|
|
+ ### NEMO files
|
|
|
+ rm -f ${exp_name}_??_????????_????????_{grid_U,grid_V,grid_W,grid_T,icemod,SBC,scalar,SBC_scalar}.nc
|
|
|
+ rm -f ${exp_name}_*restart_*nc
|
|
|
+ ns=$(printf %08d $(( leg_start_sec / nem_time_step_sec )))
|
|
|
+ for (( n=0 ; n<nem_numproc ; n++ ))
|
|
|
+ do
|
|
|
+ np=$(printf %04d ${n})
|
|
|
+ rm -f restart_oce_${np}.nc
|
|
|
+ rm -f restart_ice_${np}.nc
|
|
|
+ done
|
|
|
+
|
|
|
+ # -------------------------------------------------------------------------
|
|
|
+ # *** Write the restart control file
|
|
|
+ # -------------------------------------------------------------------------
|
|
|
+ shopt -u nullglob
|
|
|
+
|
|
|
+ tr=$(date -d "0 -$time_begin sec + $time_end sec" +%T)
|
|
|
+ current_date=$(date +'%F %T')
|
|
|
+ {
|
|
|
+ echo "#"
|
|
|
+ echo "# Finished leg at ${current_date} after ${tr} (hh:mm:ss)"
|
|
|
+ echo "leg_number=${leg_number}"
|
|
|
+ echo "leg_start_date=\"${leg_start_date}\""
|
|
|
+ echo "leg_end_date=\"${leg_end_date}\""
|
|
|
+ } | tee -a "${info_file}"
|
|
|
+
|
|
|
+ # Need to reset force_run_from_scratch in order to avoid destroying the next leg
|
|
|
+ force_run_from_scratch=false
|
|
|
+ special_restart=false
|
|
|
+
|
|
|
+done # Loop over legs
|
|
|
+
|
|
|
+cd ${start_dir}
|
|
|
+
|
|
|
+cd - >/dev/null
|
|
|
+[[ $@ == *noresubmit* ]] && exit 0
|
|
|
+if (( leg_end_epoch < run_end_epoch )) ; then
|
|
|
+ echo "Leg end earlier than end of simulation."
|
|
|
+ echo "Submitting another job."
|
|
|
+ if [[ "$@" == *"local"* ]] ; then
|
|
|
+ exec "$0" "$@"
|
|
|
+ elif [ "qsub" == *sbatch* ] ; then
|
|
|
+ sbatch $0 $@ | awk '{print $4}' >> ${run_dir}/.coral_jobs
|
|
|
+ elif [[ $USER == "vsc"* ]]; then
|
|
|
+ if [ -f ${PBS_O_WORKDIR-$PWD}/tmp/credits_file ]; then
|
|
|
+ credits=`cat ${PBS_O_WORKDIR-$PWD}/tmp/credits_file`
|
|
|
+ else
|
|
|
+ credits_tmp=`grep "account string:" ${PBS_O_WORKDIR-$PWD}/*.o* -R | sed 's/^.*: //'`
|
|
|
+ credits=`echo $credits_tmp | awk '{ print $1 }'`
|
|
|
+
|
|
|
+ cd ${start_dir}
|
|
|
+ mkdir -p tmp
|
|
|
+ echo ${credits} > tmp/credits_file
|
|
|
+ fi
|
|
|
+ if [ -f ${PBS_O_WORKDIR-$PWD}/tmp/script_name ]; then
|
|
|
+ script_name=`cat ${PBS_O_WORKDIR-$PWD}/tmp/script_name`
|
|
|
+ else
|
|
|
+ script_name="run.sh"
|
|
|
+ fi
|
|
|
+ ssh login2 "cd $start_dir; qsub -A $credits $script_name | tee -a coral_jobs"
|
|
|
+ sleep 2
|
|
|
+ jobid=`cat coral_jobs`
|
|
|
+ rm -f coral_jobs
|
|
|
+ jobid=${jobid%%.*}
|
|
|
+ echo "${jobid}" >> "${run_dir}"/.coral_jobs ;
|
|
|
+ else
|
|
|
+ "qsub" -v PBS_OPTIONS="$@" "$0" | tee -a coral_jobs
|
|
|
+ sleep 2
|
|
|
+ jobid=`cat coral_jobs`
|
|
|
+ rm -f coral_jobs
|
|
|
+ jobid=${jobid%.*}
|
|
|
+ echo "${jobid}" >> "${run_dir}"/.coral_jobs
|
|
|
+ fi
|
|
|
+else
|
|
|
+ if [[ $USER == "vsc"* ]]; then
|
|
|
+ rm -rf ${start_dir}/tmp
|
|
|
+ fi
|
|
|
+
|
|
|
+ rm -rf ${run_dir}/../cclm_out_tmp
|
|
|
+ echo "Nothing left to do. Exiting."
|
|
|
+
|
|
|
+cat << "EOF"
|
|
|
+ _____ _ _ _ _ _ _ _
|
|
|
+ / ____(_) | | | | (_) | | | | | |
|
|
|
+ | (___ _ _ __ ___ _ _| | __ _| |_ _ ___ _ __ ___ ___ _ __ ___ _ __ | | ___| |_ ___ __| |
|
|
|
+ \___ \| | '_ ` _ \| | | | |/ _` | __| |/ _ \| '_ \ / __/ _ \| '_ ` _ \| '_ \| |/ _ \ __/ _ \/ _` |
|
|
|
+ ____) | | | | | | | |_| | | (_| | |_| | (_) | | | | | (_| (_) | | | | | | |_) | | __/ || __/ (_| |
|
|
|
+ |_____/|_|_| |_| |_|\__,_|_|\__,_|\__|_|\___/|_| |_| \___\___/|_| |_| |_| .__/|_|\___|\__\___|\__,_|
|
|
|
+ | |
|
|
|
+ |_|
|
|
|
+EOF
|
|
|
+
|
|
|
+fi
|
|
|
+
|
|
|
+exit 0
|
|
|
+
|
|
|
+
|
|
|
+
|
Charles Pelletier