Coral/templates/scripts/skeleton.sh

# Function leap days calculates the number of leap days (29th of Februrary) in
# a time intervall between two dates.
#
# Usage leap_days START_DATE END_DATE
#TODO: simplify this
function leap_days()
{
    local ld=0
    local frstYYYY=$(date -ud "$1" +%Y)
    local lastYYYY=$(date -ud "$2" +%Y)

    set +e

    # Check first year for leap day between start and end date
    $(date -ud "${frstYYYY}-02-29" > /dev/null 2>&1) \
    && (( $(date -ud "$1" +%s) < $(date -ud "${frstYYYY}-03-01" +%s) )) \
    && (( $(date -ud "$2" +%s) > $(date -ud "${lastYYYY}-02-28" +%s) )) \
    && (( ld++ ))

    # Check intermediate years for leap day
    for (( y=(( ${frstYYYY}+1 )); y<=(( ${lastYYYY}-1 )); y++ ))
    do
        $(date -ud "$y-02-29" > /dev/null 2>&1) && (( ld++ ))
    done

    # Check last year (if different from first year) for leap day between start
    # and end date
    (( $lastYYYY > $frstYYYY )) \
    && $(date -ud "${lastYYYY}-02-29" > /dev/null 2>&1) \
    && (( $(date -ud "$1" +%s) < $(date -ud "${frstYYYY}-03-01" +%s) )) \
    && (( $(date -ud "$2" +%s) > $(date -ud "${lastYYYY}-02-28" +%s) )) \
    && (( ld++ ))

    set -e

    echo "$ld"
}



[[ $@ == *verbose* ]] && set -x

module load ${module_list:?}
export LD_LIBRARY_PATH=${LD_LIBRARY_PATH:+${LD_LIBRARY_PATH}:}"${extralibs_list}"

#
# Create run directory
# Copy executables
# Copy XIOS XML files into run directory
# Copy namelist files into run directory
#
if [ ! -d ${run_dir:?} ]
then
     mkdir -p ${run_dir}
     if $special_restart
     then
          rsync -av --delete ${run_dir}/../${special_restart_from}/ --exclude log --exclude output --exclude restart --exclude="${special_restart_from}_*" --exclude="ocean*" --exclude="restart_*" --exclude="debug.*" --exclude="output.*" ${run_dir}
          cp -f ${nem_exe_file} ${run_dir}
          cp -f ${xio_exe_file} ${run_dir}
          special_year=${special_restart_date:0:4}
          sed -i "/$special_year/q" ${run_dir}/${info_file}
          . ${run_dir}/${info_file}
          special_restart_leg=$(printf %03d $((leg_number+1)))
          # PUT HERE THE INSTRUCTIONS TO COPY THE restart files
          cd ${run_dir}/../../archive/${special_restart_from}/restart/${special_restart_leg}
          for f in *.nc; do
              nf=${exp_name}${f:4}
              cp $f ${run_dir}/$nf
          done
          cd -
          cd ${run_dir}
          for f in ${exp_name}_????????_restart_???_????.nc; do
              nf=${f:14}
              ln -s $f $nf
          done
          cd -
     fi
fi
cd ${start_dir}
cp -u xios_config/*xml "${run_dir}"
cp -u namelists/* "${run_dir}"
cd "${run_dir}"
cp -u "${nem_exe_file:?}" .
cp -u "${xio_exe_file:?}" .

# Write fake file for previous fresh water budget adjustment (nn_fwb==2 in namelist)
[[ ! -f EMPave_old.dat ]] && echo "                               0  0.0000000000000000E+00  0.0000000000000000E+00" > EMPave_old.dat
#TODO: Enquiry about this
#
# Link data files into run directory
# Attention: no space in filenames allowed!
# (No protection against word splitting by design)
#
for file in "${ic_files[@]}"; do 
    [[ ! -e ${file#*> } ]] && ln -sf $(sed 's/ *=> */ /' <<< "${ini_data_dir}/${ic_subdir}/${nem_grid}/$file") 
done
for file in "${forcing_files[@]}"; do
    [[ ! -e ${file#*> } ||  "$file" == \** ]] && ln -sf $(sed 's/ *=> */ /' <<< "${ini_data_dir}/${forcing_subdir}/${nem_forcing_set}/$file")
done
for file in "${shared_files[@]}"; do
    [[ ! -e ${file#*> } ]] && ln -sf $(sed 's/ *=> */ /' <<< "${shared_dir}/$file")
done


#
# Normalize date formats
#
run_start_date=$(date -uR -d "${run_start_date}")
run_end_date="${run_start_date} + ${run_duration:?}"
run_end_date=$(date -uR -d "${run_end_date}")

run_start_epoch=$(date -u -d"${run_start_date}" +%s)
run_end_epoch=$(date -u -d"${run_end_date}" +%s)

for (( ; run_num_legs>0 ; run_num_legs-- ))
do

    #
    # Initialize variables (using restart file if it exists)
    # Variables ending in '_date' are in RFC2822
    # Variables ending in '_epoch' are in nb of seconds since 1970-01-01
    #
    [[ -r "${info_file:?}" ]] && source "${info_file:?}"
    leg_start_date=${leg_end_date:-$run_start_date}
    leg_number=$((${leg_number:=0}+1))
    leg_start_epoch=$(date -u -d "${leg_start_date}" +%s)
    leg_end_epoch=$(date -u -d "${leg_start_date:?} + ${rst_freq:=$run_duration}" +%s)
    leg_end_date=$(date -uR -d@"${leg_end_epoch}")
    leg_length_sec=$(( leg_end_epoch - leg_start_epoch ))
    leg_start_sec=$(( leg_start_epoch - run_start_epoch ))
    leg_end_sec=$(( leg_end_epoch - run_start_epoch ))
    leg_start_date_yyyymmdd=$(date -u -d "${leg_start_date}" +%Y%m%d) # FIXME appears unused
    # Correct for leap days because NEMO standalone uses no-leap calendar
    leg_length_sec=$(( leg_length_sec - $(leap_days "${leg_start_date}" "${leg_end_date}")*24*3600 ))
    leg_start_sec=$(( leg_start_sec - $(leap_days "${run_start_date}" "${leg_start_date}")*24*3600 ))
    leg_end_sec=$(( leg_end_sec - $(leap_days "${run_start_date}" "${leg_end_date}")*24*3600 ))
    (( leg_number > 1 )) && leg_is_restart=true || leg_is_restart=false

    #
    # Compute leg end-date and trim if necessary
    #
    (( leg_end_epoch > run_end_epoch )) && leg_end_date=${run_end_epoch}

    #
    # Check whether there is some work left to do
    #
    if (( leg_start_epoch >= run_end_epoch ))
    then
        echo "Leg start date equal to or after end of simulation."
        echo "Nothing left to do. Cleaning and exiting."
	for (( n=0 ; n<nem_numproc ; n++ ))
	do
            np=$(printf %04d ${n})
	    rm -f "${run_dir}/restart_oce_${np}.nc"
	    rm -f "${run_dir}/restart_ice_${np}.nc"
	done
        exit 0
    fi

    #
    # Update namelist
    #
    source build_namelist_cfg.sh > namelist_cfg

    #
    # Link the restart files
    #
    ns=$(printf %08d $(( leg_start_sec / nem_time_step_sec - nem_restart_offset )))
    if ((leg_start_sec > 0 )); then
    for (( n=0 ; n<nem_numproc ; n++ ))
    do
        np=$(printf %04d ${n})
        [[ -f "${exp_name:?}_${ns}_restart_oce_${np}.nc" ]] || { echo "Error: restart file not found." ; exit 2 ; }
        ln -fs "${exp_name:?}_${ns}_restart_oce_${np}.nc" "restart_oce_${np}.nc"
        [[ -f "${exp_name:?}_${ns}_restart_ice_${np}.nc" ]] || { echo "Error: restart file not found." ; exit 2 ; }
        ln -fs "${exp_name:?}_${ns}_restart_ice_${np}.nc" "restart_ice_${np}.nc"
    done
    fi

    # Stop here is preponly was specified
    [[ $@ == *preponly* ]] && exit 0

    #
    # Run nemo
    #
    time_begin=$(date +%s)
    mpirun -np "${xio_numproc:?}" "${xio_exe_file:?}" : -np "${nem_numproc:?}" "${nem_exe_file:?}"
    time_end=$(date +%s)

#############################

    formatted_leg_number=$(printf %03d $((leg_number)))
    #
    # Move NEMO output files to archive directory
    # 
    outdir="${archive_dir:?}/output/${formatted_leg_number}"
    mkdir -p "${outdir}"

    shopt -s nullglob
    for v in grid_U grid_V grid_W grid_T icemod icemoa SBC SBC_scalar scalar
    do
        for f in ${exp_name}_??_????????_????????_${v}_????.nc
        do
             mv "$f" "$outdir/"
        done
        for f in ${exp_name}_??_????????_????????_${v}.nc
        do
             mv "$f" "$outdir/"
        done
        for f in ${exp_name}_??_${v}.nc
        do
            mv "$f" "$outdir/"
        done
    done

    #
    # Move NEMO restart files from previous run to archive directory
    #
        outdir="$archive_dir/restart/${formatted_leg_number}"
        mkdir -p "${outdir}"

        for f in ${exp_name}_${ns}_restart_???_????.nc
        do
               [ -f "$f" ] && mv "$f" "${outdir}"
        done

    #
    # Move log files to archive directory
    #
    outdir="$archive_dir/log/${formatted_leg_number}"
    mkdir -p "${outdir}"
    for f in ocean.output time.step solver.stat ; do mv "${f}" "${outdir}"; done
#
#
##############################

    shopt -u nullglob

    # 
    # Write checkpoint control file
    # TODO: enquiry why 0 -1 +2 rather than 2- 1
    #
    tr=$(date -d "0 -$time_begin sec + $time_end sec" +%T) 
    current_date=$(date +'%F %T')
    {
      echo "#"
      echo "# Finished leg at ${current_date} after ${tr} (hh:mm:ss)" 
      echo "leg_number=${leg_number}"
      echo "leg_start_date=\"${leg_start_date}\""
      echo "leg_end_date=\"${leg_end_date}\""
    } | tee -a "${info_file}"

    special_restart=false
done
#
# Move back to submission directory
#
cd - >/dev/null
#
# Check whether there is some work left to do in a further job
#
[[ $@ == *noresubmit* ]] && exit 0
if (( leg_end_epoch < run_end_epoch )) ; then
    echo "Leg end earlier than end of simulation."
    echo "Submitting another job."
    #[[ $@ == *local* ]] && exec $0 $@ || scontrol requeue $SLURM_JOB_ID #TODO: factorize this
    if [[ "$@" == *"local"* ]] ; then
        exec "$0" "$@"
    elif [ "${submit_command}" == *sbatch* ] ; then
        sbatch $0 $@ | awk '{print $4}' >> ${run_dir}/.coral_jobs
    elif [[ $USER == "vsc"* ]]; then
        if [ -f ${PBS_O_WORKDIR-$PWD}/tmp/credits_file ]; then
            credits=`cat ${PBS_O_WORKDIR-$PWD}/tmp/credits_file`
        else
            #logfiletmp=`echo ${PBS_O_WORKDIR-$PWD} | rev | cut -d'/' -f1 | rev`
            #logfiletmp2=`echo $logfiletmp | cut -d'_' -f1`
            #logfile=`ls ${PBS_O_WORKDIR-$PWD}"/"$logfiletmp2*.o*`
            #credits=`grep "account string:" $logfile | sed 's/^.*: //'`
            credits_tmp=`grep "account string:" ${PBS_O_WORKDIR-$PWD}/*.o* -R | sed 's/^.*: //'`
            credits=`echo $credits_tmp | awk '{ print $1 }'`

            mkdir -p tmp
            echo ${credits} > tmp/credits_file
        fi
        script_name=`cat ${PBS_O_WORKDIR-$PWD}/tmp/script_name`
        run_path=`cat ${PBS_O_WORKDIR-$PWD}/tmp/address`
        ssh login2 "cd $run_path; qsub -A $credits $script_name | tee -a coral_jobs"
        sleep 2
        jobid=`cat coral_jobs`
        rm -f coral_jobs
        jobid=${jobid%%.*}
        echo "${jobid}" >> "${run_dir}"/.coral_jobs ;
    else
        "qsub" -v PBS_OPTIONS="$@" "$0" | tee -a coral_jobs
        sleep 2
        jobid=`cat coral_jobs`
        rm -f coral_jobs
        jobid=${jobid%.*}
        echo "${jobid}" >> "${run_dir}"/.coral_jobs
    fi
else
    if [[ $USER == "vsc"* ]]; then
        run_path=`cat ${PBS_O_WORKDIR-$PWD}/tmp/address`
        rm -rf $run_path/tmp
    fi

    echo "Nothing left to do. Cleaning and exiting." # FIXME Factorize this (we have two exit points)
    for (( n=0 ; n<nem_numproc ; n++ ))
    do
        np=$(printf %04d ${n})
        rm -f "restart_oce_${np}.nc"
        rm -f "restart_ice_${np}.nc"
    done
fi

exit 0