# Function leap days calculates the number of leap days (29th of Februrary) in # a time intervall between two dates. # # Usage leap_days START_DATE END_DATE #TODO: simplify this function leap_days() { local ld=0 local frstYYYY=$(date -ud "$1" +%Y) local lastYYYY=$(date -ud "$2" +%Y) set +e # Check first year for leap day between start and end date $(date -ud "${frstYYYY}-02-29" > /dev/null 2>&1) \ && (( $(date -ud "$1" +%s) < $(date -ud "${frstYYYY}-03-01" +%s) )) \ && (( $(date -ud "$2" +%s) > $(date -ud "${lastYYYY}-02-28" +%s) )) \ && (( ld++ )) # Check intermediate years for leap day for (( y=(( ${frstYYYY}+1 )); y<=(( ${lastYYYY}-1 )); y++ )) do $(date -ud "$y-02-29" > /dev/null 2>&1) && (( ld++ )) done # Check last year (if different from first year) for leap day between start # and end date (( $lastYYYY > $frstYYYY )) \ && $(date -ud "${lastYYYY}-02-29" > /dev/null 2>&1) \ && (( $(date -ud "$1" +%s) < $(date -ud "${frstYYYY}-03-01" +%s) )) \ && (( $(date -ud "$2" +%s) > $(date -ud "${lastYYYY}-02-28" +%s) )) \ && (( ld++ )) set -e echo "$ld" } function ml_icesheet () { if [[ ${USER} == "vsc"????? ]]; then if [[ "${name_icesheet_model}" == "fETISh" ]]; then module purge module load Python PYTHONPATH="${HOME}/.local/bin:${HOME}/.local/lib/python3.7/site-packages:\${PYTHONPATH}" module load CDO echo ${PYTHONPATH} > ${start_dir}/ppath echo "genius and fetish detected" > ${start_dir}/tmpout else echo "On Genius, ice sheet models other than fETISh not implemented yet" > ${start_dir}/tmpout exit 2 fi elif [[ ${HOSTNAME} == "lm3"* ]]; then if [[ "${name_icesheet_model}" == "fETISh" ]]; then echo "lemaitre3 and fetish detected" > ${start_dir}/tmpout module --force purge module load tis/2018.01 module load releases/2018b module --ignore-cache load MCR/R2018b else echo "On lemaitre3, ice sheet models other than fETISh not implemented yet" > ${start_dir}/tmpout exit 2 fi else echo "hostname undetected. not implemented yet." > ${start_dir}/tmpout exit 2 fi } # Function for loading modules function ml_pp_ice () { if [[ ${USER} == "vsc"????? ]]; then if [[ "${name_icesheet_model}" == "fETISh" ]]; then module purge module load Python PYTHONPATH="${HOME}/.local/bin:${HOME}/.local/lib/python3.7/site-packages:\${PYTHONPATH}" module load CDO echo ${PYTHONPATH} > ${start_dir}/ppath echo "genius and fetish detected" > ${start_dir}/tmpout else echo "On Genius, ice sheet models other than fETISh not implemented yet" > ${start_dir}/tmpout exit 2 fi elif [[ ${HOSTNAME} == "lm3"* ]]; then if [[ "${name_icesheet_model}" == "fETISh" ]]; then echo "lemaitre3 and fetish detected" > ${start_dir}/tmpout module purge module load releases/2018a CDO/1.8.2-intel-2018a Python/3.6.4-intel-2018a # PYTHONPATH="${HOME}/.local/lib/python3.6/site-packages:${PYTHONPATH}" # module load NCO CDO else echo "On lemaitre3, ice sheet models other than fETISh not implemented yet" > ${start_dir}/tmpout exit 2 fi else echo "hostname undetected. not implemented yet." > ${start_dir}/tmpout exit 2 fi } [[ $@ == *verbose* ]] && set -x module purge module load ${module_list:?} export LD_LIBRARY_PATH=${LD_LIBRARY_PATH:+${LD_LIBRARY_PATH}:}"${extralibs_list}" # # Create run directory # Copy executables # Copy XIOS XML files into run directory # Copy namelist files into run directory # if [ ! -d ${run_dir:?} ] then mkdir -p ${run_dir} if $special_restart then rsync -av --delete ${run_dir}/../${special_restart_from}/ --exclude log --exclude output --exclude restart --exclude="${special_restart_from}_*" --exclude="ocean*" --exclude="restart_*" --exclude="debug.*" --exclude="output.*" ${run_dir} cp -f ${nem_exe_file} ${run_dir} cp -f ${xio_exe_file} ${run_dir} special_year=${special_restart_date:0:4} sed -i "/$special_year/q" ${run_dir}/${info_file} . ${run_dir}/${info_file} special_restart_leg=$(printf %03d $((leg_number+1))) # PUT HERE THE INSTRUCTIONS TO COPY THE restart files cd ${run_dir}/../../archive/${special_restart_from}/restart/${special_restart_leg} for f in *.nc; do nf=${exp_name}${f:4} cp $f ${run_dir}/$nf done cd - cd ${run_dir} for f in ${exp_name}_????????_restart_???_????.nc; do nf=${f:14} ln -s $f $nf done cd - fi fi cd ${start_dir} cp -u xios_config/*xml "${run_dir}" cp -u namelists/* "${run_dir}" cd "${run_dir}" # cp -u "${nem_exe_file:?}" . # cp -u "${xio_exe_file:?}" . nem_exe_bn=`basename ${nem_exe_file}` xio_exe_bn=`basename ${xio_exe_file}` # Write fake file for previous fresh water budget adjustment (nn_fwb==2 in namelist) [[ ! -f EMPave_old.dat ]] && echo " 0 0.0000000000000000E+00 0.0000000000000000E+00" > EMPave_old.dat #TODO: Enquiry about this # # Link data files into run directory # Attention: no space in filenames allowed! # (No protection against word splitting by design) # for file in "${ic_files[@]}"; do [[ ! -e ${file#*> } ]] && ln -sf $(sed 's/ *=> */ /' <<< "${ini_data_dir}/${ic_subdir}/${nem_grid}/$file") done for file in "${forcing_files[@]}"; do [[ ! -e ${file#*> } || "$file" == \** ]] && ln -sf $(sed 's/ *=> */ /' <<< "${ini_data_dir}/${forcing_subdir}/${nem_forcing_set}/$file") done for file in "${shared_files[@]}"; do [[ ! -e ${file#*> } ]] && ln -sf $(sed 's/ *=> */ /' <<< "${shared_dir}/$file") done if [ "${isfcpl}" == "1" ]; then if [[ ${USER} == "vsc"????? ]]; then echo "Genius and iscpl detected. OK." elif [[ ${HOSTNAME} == "lm3"* ]]; then echo "Lemaitre3 and iscpl detected. OK." else echo "You have asked for iscpl, but I have not recognized genius nor lemaitre3. Exiting." exit 2 fi if [ "${name_icesheet_model}" != "fETISh" ]; then echo "only \"fETISh\" is known as an ice sheet model. exiting." exit 2 fi length_cut_pref=$(( ${#exp_name} + 4 )) rebuild_nemo_exe=${nemo_src_dir}/TOOLS/REBUILD_NEMO/rebuild_nemo if [ -f ${start_dir}/isfcpl.track ]; then source ${start_dir}/isfcpl.track # printf "\n" >> isfcpl.track # rm -f isfcpl.track fi ln -sf ${start_dir}/isfcpl.track ${run_dir} cp -f ${icesheet_src_dir}/* ${run_dir} iscpl_nm=.true. ice_atm_cpl=0 if [[ ${xio_exe_file} == *"xios-2"* ]]; then is_xios2=1 filedef_xios="${run_dir}/file_def_nemo-opa.xml" else is_xios2=0 filedef_xios="${run_dir}/file_def.xml" fi else iscpl_nm=.false. fi # # Normalize date formats # run_start_date=$(date -uR -d "${run_start_date}") run_end_date="${run_start_date} + ${run_duration:?}" run_end_date=$(date -uR -d "${run_end_date}") run_start_epoch=$(date -u -d"${run_start_date}" +%s) run_end_epoch=$(date -u -d"${run_end_date}" +%s) for (( ; run_num_legs>0 ; run_num_legs-- )) do # # Initialize variables (using restart file if it exists) # Variables ending in '_date' are in RFC2822 # Variables ending in '_epoch' are in nb of seconds since 1970-01-01 # [[ -r "${info_file:?}" ]] && source "${info_file:?}" leg_start_date=${leg_end_date:-$run_start_date} leg_number=$((${leg_number:=0}+1)) leg_start_epoch=$(date -u -d "${leg_start_date}" +%s) leg_end_epoch=$(date -u -d "${leg_start_date:?} + ${rst_freq:=$run_duration}" +%s) leg_end_date=$(date -uR -d@"${leg_end_epoch}") leg_length_sec=$(( leg_end_epoch - leg_start_epoch )) leg_start_sec=$(( leg_start_epoch - run_start_epoch )) leg_end_sec=$(( leg_end_epoch - run_start_epoch )) leg_beforestart_epoch=$(( ${leg_start_epoch} - 43200 )) leg_beforestart_date=$(date -uR -d@"${leg_beforestart_epoch}") leg_almostend_epoch=$(( ${leg_end_epoch} - 43200 )) leg_almostend_date=$(date -uR -d@"${leg_almostend_epoch}") nextleg_start_epoch=$(( ${leg_end_epoch} + 43200 )) nextleg_start_date=$(date -uR -d@"${nextleg_start_epoch}") run_start_date_yyyymmdd=$(date -u -d "${run_start_date}" +%Y%m%d) # FIXME appears unused run_end_date_yyyymmdd=$(date -u -d "${run_end_date}" +%Y%m%d) # FIXME appears unused leg_start_date_yyyymmdd=$(date -u -d "${leg_start_date}" +%Y%m%d) # FIXME appears unused leg_almostend_date_yyyymmdd=$(date -u -d "${leg_almostend_date}" +%Y%m%d) # FIXME appears unused nextleg_start_date_yyyymmdd=$(date -u -d "${nextleg_start_date}" +%Y%m%d) # FIXME appears unused leg_beforestart_date_yyyymmdd=$(date -u -d "${leg_beforestart_date}" +%Y%m%d) # FIXME appears unused # run_start_date_yyyymm="${run_start_date_yyyymmdd::-2}" # run_end_date_yyyymm="${run_end_date_yyyymmdd::-2}" # leg_start_date_yyyymm="${leg_start_date_yyyymmdd::-2}" # leg_almostend_date_yyyymm="${leg_almostend_date_yyyymmdd::-2}" # leg_beforestart_date_yyyymm="${leg_beforestart_date_yyyymmdd::-2}" if [ "${isfcpl}" == "1" ]; then # Check if a monthly "icesheet" output file has indeed been asked. tmp_check=$(grep -nri "file id=\"file.*\".*name_suffix=\"_icesheet\"" ${filedef_xios} | wc -l) if (( tmp_check == 0 )); then echo "${filedef_xios} does not look like you have asked for an icesheet output file! Exiting." exit 2 elif (( tmp_check > 1 )); then echo "${filedef_xios} contains at least 2 icesheet output files! This is ambiguous. Exiting." exit 2 fi if (( leg_number == 1 )); then mkdir -p ${archive_dir}/isfcpl mkdir -p ${archive_dir}/${name_icesheet_model} rm -f ${start_dir}/isfcpl.track # Counter for the number of restart since last file exchange with the ice sheet model. cnt_res_isfcpl=0 isfcpl_window_days=$(echo "(${n_res_perisfcpl} * ${leg_length_sec}) / 86400" | bc) isfcpl_d30=$(echo "${isfcpl_window_days} / 30" | bc -l) n_month_isfcpl=$(echo ${isfcpl_d30} | awk '{print int($1+0.5)}') # if (( isfcpl_window_days >= 28 && isfcpl_window_days <= 31 )); then # n_month_isfcpl=1 # elif (( isfcpl_window_days >= 336 && isfcpl_window_days <= 372 )); then # n_month_isfcpl=12 # else # echo "Can't figure out how many month long is the coupling window." # exit 2 # fi n_month_full_run=$(( $(( 12 * $(( ${run_end_date_yyyymmdd::4} - ${run_start_date_yyyymmdd::4} )) )) + $(( ${run_end_date_yyyymmdd:4:2} - ${run_start_date_yyyymmdd:4:2} )) )) if (( $(( ${n_month_full_run} % ${n_month_isfcpl} )) != 0 )); then echo "n_month_full_run=${n_month_full_run} is not a multiple of n_month_isfcpl=${n_month_isfcpl}." exit 2 else n_isfcpl_window=$(echo "${n_month_full_run} / ${n_month_isfcpl}" | bc) fi tmp=`readlink -f isf_draft_meter.nc` tmp2=`readlink -f bathy_meter.nc` if [ "${tmp}" != "${tmp2}" ]; then echo "Error. At initialization, isf_draft_meter.nc and bathy_meter.nc do not point to the same file." exit 1 fi init_bathy_file=`basename ${tmp}` cp -f ${tmp} ${archive_dir}/isfcpl cp -f ${tmp} ${run_dir}/init_NEMO_bathy.nc date_beg_isfcpl="${run_start_date_yyyymmdd}" init_geom=1 init_bathy=1 curr_bathy_file=${init_bathy_file} ln -sf ${run_dir}/${icesheet_init_geom_file} ${run_dir}/${name_icesheet_model}_8km_geometry_curr.nc printf "# Information for the ice sheet model\n" > ${start_dir}/isfcpl.track printf "run_name=${exp_name}\n" >> ${start_dir}/isfcpl.track printf "initial_date=${run_start_date_yyyymmdd}\n" >> ${start_dir}/isfcpl.track printf "n_month_isfcpl=${n_month_isfcpl}\n" >> ${start_dir}/isfcpl.track printf "n_isfcpl_window=${n_isfcpl_window}\n" >> ${start_dir}/isfcpl.track printf "date_beg_isfcpl=${date_beg_isfcpl}\n" >> ${start_dir}/isfcpl.track printf "date_end_isfcpl=YYYYMMDD\n" >> ${start_dir}/isfcpl.track printf "ice_atm_cpl=${ice_atm_cpl}\n\n" >> ${start_dir}/isfcpl.track printf "# Information for NEMO to keep up\n" >> ${start_dir}/isfcpl.track printf "cnt_res_isfcpl=0\n" >> ${start_dir}/isfcpl.track printf "init_geom=1\n" >> ${start_dir}/isfcpl.track printf "init_bathy=1\n" >> ${start_dir}/isfcpl.track printf "curr_bathy_file=${init_bathy_file}\n" >> ${start_dir}/isfcpl.track init_geom=1 init_bathy=1 curr_bathy_file=${init_bathy_file} ln -sf ${run_dir}/${icesheet_init_geom_file} ${run_dir}/${name_icesheet_model}_8km_geometry_curr.nc fi # leg_number == 1 ln -sf ${curr_bathy_file} ${run_dir}/bathy_meter.nc ln -sf ${curr_bathy_file} ${run_dir}/isf_draft_meter.nc fi # isfcpl == 1 # Correct for leap days because NEMO standalone uses no-leap calendar if (( leap_years == 0 )); then leg_length_sec=$(( leg_length_sec - $(leap_days "${leg_start_date}" "${leg_end_date}")*24*3600 )) leg_start_sec=$(( leg_start_sec - $(leap_days "${run_start_date}" "${leg_start_date}")*24*3600 )) leg_end_sec=$(( leg_end_sec - $(leap_days "${run_start_date}" "${leg_end_date}")*24*3600 )) fi (( leg_number > 1 )) && leg_is_restart=true || leg_is_restart=false # # Compute leg end-date and trim if necessary # (( leg_end_epoch > run_end_epoch )) && leg_end_date=${run_end_epoch} # # Check whether there is some work left to do # if (( leg_start_epoch >= run_end_epoch )) then echo "Leg start date equal to or after end of simulation." echo "Nothing left to do. Cleaning and exiting." for (( n=0 ; n namelist_cfg # Link the restart files ns=$(printf %08d $(( leg_start_sec / nem_time_step_sec - nem_restart_offset ))) if ((leg_start_sec > 0 )); then for (( n=0 ; n ${start_dir}/log sleep 10000000000000 fi time_begin=$(date +%s) mpirun -np "${xio_numproc:?}" "${xio_exe_file:?}" : -np "${nem_numproc:?}" "${nem_exe_file:?}" time_end=$(date +%s) echo "dbg cp made it past mpirun" if (( isfcpl == 1 )); then # NEMO restart index within ice sheet / ocean coupling window cnt_res_isfcpl=$(( $(( ${leg_number} - 1 )) % ${n_res_perisfcpl} )) sed -i "s/.*cnt_res_isfcpl=.*/cnt_res_isfcpl=${cnt_res_isfcpl}/" ${start_dir}/isfcpl.track # module purge # module load CDO fmt_res_isfcpl=$(printf "%02d" ${cnt_res_isfcpl}) in_stencil="${exp_name}_1m_${leg_start_date_yyyymmdd}_${leg_almostend_date_yyyymmdd}_icesheet" out_stencil="${exp_name}_icesheet_res${fmt_res_isfcpl}" if (( is_xios2 == 0 )); then for((prc=0;prc ${start_dir}/isfcpl.track printf "run_name=${exp_name}\n" >> ${start_dir}/isfcpl.track printf "initial_date=${run_start_date_yyyymmdd}\n" >> ${start_dir}/isfcpl.track printf "n_month_isfcpl=${n_month_isfcpl}\n" >> ${start_dir}/isfcpl.track printf "n_isfcpl_window=${n_isfcpl_window}\n" >> ${start_dir}/isfcpl.track printf "date_beg_isfcpl=${nextleg_start_date_yyyymmdd}\n" >> ${start_dir}/isfcpl.track printf "date_end_isfcpl=YYYYMMDD\n" >> ${start_dir}/isfcpl.track printf "ice_atm_cpl=${ice_atm_cpl}\n\n" >> ${start_dir}/isfcpl.track printf "# Information for NEMO to keep up\n" >> ${start_dir}/isfcpl.track printf "cnt_res_isfcpl=0\n" >> ${start_dir}/isfcpl.track printf "init_geom=${init_geom}\n" >> ${start_dir}/isfcpl.track printf "init_bathy=${init_bathy}\n" >> ${start_dir}/isfcpl.track printf "curr_bathy_file=${curr_bathy_file}\n" >> ${start_dir}/isfcpl.track module purge module load ${module_list:?} export LD_LIBRARY_PATH=${LD_LIBRARY_PATH:+${LD_LIBRARY_PATH}:}"${extralibs_list}" fi fi ############################# formatted_leg_number=$(printf %03d $((leg_number))) # # Move NEMO output files to archive directory # outdir="${archive_dir:?}/output/${formatted_leg_number}" mkdir -p "${outdir}" shopt -s nullglob for v in grid_U grid_V grid_W grid_T icemod icemod_U icemod_V icemoa SBC SBC_scalar scalar fwfisf do for f in ${exp_name}_??_????????_????????_${v}_????.nc do mv "$f" "$outdir/" done for f in ${exp_name}_???_????????_????????_${v}_????.nc do mv "$f" "$outdir/" done for f in ${exp_name}_??_????????_????????_${v}.nc do mv "$f" "$outdir/" done for f in ${exp_name}_??_${v}.nc do mv "$f" "$outdir/" done done # # Move NEMO restart files from previous run to archive directory # outdir="$archive_dir/restart/${formatted_leg_number}" mkdir -p "${outdir}" for f in ${exp_name}_${ns}_restart_???_????.nc do [ -f "$f" ] && mv "$f" "${outdir}" done # # Move log files to archive directory # outdir="$archive_dir/log/${formatted_leg_number}" mkdir -p "${outdir}" for f in ocean.output time.step solver.stat ; do mv "${f}" "${outdir}"; done shopt -u nullglob # # Write checkpoint control file # TODO: enquiry why 0 -1 +2 rather than 2- 1 # tr=$(date -d "0 -$time_begin sec + $time_end sec" +%T) current_date=$(date +'%F %T') { echo "#" echo "# Finished leg at ${current_date} after ${tr} (hh:mm:ss)" echo "leg_number=${leg_number}" echo "leg_start_date=\"${leg_start_date}\"" echo "leg_end_date=\"${leg_end_date}\"" } | tee -a "${info_file}" special_restart=false done # # Move back to submission directory # cd - >/dev/null # # Check whether there is some work left to do in a further job # [[ $@ == *noresubmit* ]] && exit 0 if (( leg_end_epoch < run_end_epoch )) ; then echo "Leg end earlier than end of simulation." echo "Submitting another job." #[[ $@ == *local* ]] && exec $0 $@ || scontrol requeue $SLURM_JOB_ID #TODO: factorize this if [[ "$@" == *"local"* ]] ; then cd ${start_dir} sleep 2 ./run.sh local && exit 0 #exec "$0" "$@" elif [ "${submit_command}" == *sbatch* ] ; then sbatch $0 $@ | awk '{print $4}' >> ${run_dir}/.coral_jobs elif [[ $USER == "vsc"* ]]; then if [ -f ${PBS_O_WORKDIR-$PWD}/tmp/credits_file ]; then credits=`cat ${PBS_O_WORKDIR-$PWD}/tmp/credits_file` else #logfiletmp=`echo ${PBS_O_WORKDIR-$PWD} | rev | cut -d'/' -f1 | rev` #logfiletmp2=`echo $logfiletmp | cut -d'_' -f1` #logfile=`ls ${PBS_O_WORKDIR-$PWD}"/"$logfiletmp2*.o*` #credits=`grep "account string:" $logfile | sed 's/^.*: //'` credits_tmp=`grep "account string:" ${PBS_O_WORKDIR-$PWD}/*.o* -R | sed 's/^.*: //'` credits=`echo $credits_tmp | awk '{ print $1 }'` mkdir -p tmp echo ${credits} > tmp/credits_file fi script_name=`cat ${PBS_O_WORKDIR-$PWD}/tmp/script_name` run_path=`cat ${PBS_O_WORKDIR-$PWD}/tmp/address` ssh login2 "cd $run_path; qsub -A $credits $script_name | tee -a coral_jobs" sleep 2 jobid=`cat coral_jobs` rm -f coral_jobs jobid=${jobid%%.*} echo "${jobid}" >> "${run_dir}"/.coral_jobs ; else "qsub" -v PBS_OPTIONS="$@" "$0" | tee -a coral_jobs sleep 2 jobid=`cat coral_jobs` rm -f coral_jobs jobid=${jobid%.*} echo "${jobid}" >> "${run_dir}"/.coral_jobs fi else if [[ $USER == "vsc"* ]]; then run_path=`cat ${PBS_O_WORKDIR-$PWD}/tmp/address` rm -rf $run_path/tmp fi echo "Nothing left to do. Cleaning and exiting." # FIXME Factorize this (we have two exit points) for (( n=0 ; n