ecearth3/runtime/classic/ece-lsm.sh.tmpl

#!/usr/bin/env bash

set -ue

# autosubmit stuff - this is ignored if using the classic runtime
if [[ "%%_%%" == "%_%" ]]
then
    use_autosubmit=true
    set -xuve
    # go to autosubmit root directory
    cd %CURRENT_ROOTDIR%
    # Simulation member. Use any (reasonable) syntax you want.
    member="%MEMBER%"
else
    use_autosubmit=false
    # Simulation member. Use any (reasonable) syntax you want.
    member="fc0"
fi

# librunscript defines some helper functions
source ./librunscript.sh

# =============================================================================
# *** BEGIN User configuration
# =============================================================================

# -----------------------------------------------------------------------------
# *** General configuration
# -----------------------------------------------------------------------------
# Component configuration (for syntax of the $config variable, see librunscript.sh)
#
if $use_autosubmit
then
    [[ "%LPJG%" = TRUE ]] && config="%LPJG_CONFIG% lpjg_forcing:IFS oasis" ||  config=""
else
    config="lpjg lpjg_forcing:IFS oasis"
fi

# minimum sanity
has_config lpjg_forcing ifs && error "Cannot have both ifs and lpjg_forcing in config!!"

# Experiment name (exactly 4 letters!)
export exp_name=[[[MOD:GENERAL:EXP_NAME]]]

# Simulation start and end date. Use any (reasonable) syntax you want.
run_start_date="[[[MOD:GENERAL:RUN_START_DATE]]]"
run_end_date="[[[MOD:GENERAL:RUN_END_DATE]]]"

# Set $force_run_from_scratch to 'true' if you want to force this run to start
# from scratch, possibly ignoring any restart files present in the run
# directory. Leave set to 'false' otherwise.
# NOTE: If set to 'true' the run directory $run_dir is cleaned!
force_run_from_scratch=[[[MOD:GENERAL:FORCE_RUN_FROM_SCRATCH]]]

# Resolution (only T255L91 supported for now)
ifs_grid=T255L91 #[[[MOD:IFS:GRID]]]
#atm_grid=A128

# Restart frequency. Use any (reasonable) number and time unit you want.
# For runs without restart, leave this variable empty
rst_freq="[[[MOD:GENERAL:RST_FREQ]]]"

# Number of restart legs to be run in one go
run_num_legs=[[[MOD:GENERAL:RUN_NUM_LEGS]]]

# Coupling frequencies
has_config lpjg && cpl_freq_atm_lpjg_hrs=[[[MOD:OASIS:CPL_FREQ_ATM_LPJG_HRS]]] #24

# Directories
start_dir=${PWD}
ctrl_file_dir=${start_dir}/ctrl

# Architecture
build_arch=[[[PLT:ACTIVE:BUILD_ARCH]]]

# This file is used to store information about restarts
ece_info_file="ece.info"

# -----------------------------------------------------------------------------
# *** Read platform dependent configuration
# -----------------------------------------------------------------------------
source ./ecconf.cfg

configure

# -----------------------------------------------------------------------------
# *** IFS configuration - these variables are needed by LPJG
# -----------------------------------------------------------------------------

export ifs_res_hor=$(echo ${ifs_grid} | sed 's:T\([0-9]\+\)L\([0-9]\+\):\1:')
ifs_res_ver=$(echo ${ifs_grid} | sed 's:T\([0-9]\+\)L\([0-9]\+\):\2:')

ifs_cmip_fixyear=[[[MOD:IFS:CMIP_FIXYEAR]]]
ifs_A4xCO2=[[[MOD:IFS:CMIP6_A4xCO2]]]
ifs_1PCTCO2=[[[MOD:IFS:CMIP6_1PCTCO2]]]


# -----------------------------------------------------------------------------
# *** LPJ-GUESS offline forcing configuration
# -----------------------------------------------------------------------------

lpjg_forcing_exe_file=${ecearth_src_dir}/lpjg/offline/lpjg_forcing
has_config lpjg_forcing:IFS && lpjg_forcing_exe_file=${ecearth_src_dir}/lpjg/offline/lpjg_forcing_ifs
lpjg_forcing_numproc=1
# TODO add this to config-run.xml ?
lpjg_forcing_files_path="/gpfs/projects/bsc32/bsc32051/lpjg/forcing/a0pq/out"

# -----------------------------------------------------------------------------
# *** LPJ-GUESS configuration
# -----------------------------------------------------------------------------

lpjg_time_step_sec=[[[MOD:LPJG:TIME_STEP_SEC]]]
lpjg_numproc=[[[MOD:LPJG:NUMPROC]]]

has_config lpjg       && lpjg_on=1
has_config lpjg:fdbck && lpjg_fdbck=1
has_config tm5        && lpjg_fdbck_tm5=1 || lpjg_fdbck_tm5=0

export lpjg_fixNdepafter=[[[MOD:LPJG:LPJG_FIXNDEPAFTER]]]
export lpjg_fixLUafter=[[[MOD:LPJG:LPJG_FIXLUAFTER]]]

lpjg_exe_file=${ecearth_src_dir}/lpjg/build/guess

# -----------------------------------------------------------------------------

# =============================================================================
# *** END of User configuration
# =============================================================================

# =============================================================================
# *** This is where the code begins ...
# =============================================================================

# -----------------------------------------------------------------------------
# *** Create the run dir if necessary and go there
#     Everything is done from here.
# -----------------------------------------------------------------------------
if [ ! -d ${run_dir} ]
then
    mkdir -p ${run_dir}
fi
cd ${run_dir}

# -----------------------------------------------------------------------------
# *** Determine the time span of this run and whether it's a restart leg
# -----------------------------------------------------------------------------

# Regularise the format of the start and end date of the simulation
run_start_date=$(date -uR -d "${run_start_date}")
run_end_date=$(date -uR -d "${run_end_date}")

# Loop over the number of legs
for (( ; run_num_legs>0 ; run_num_legs-- ))
do

    # Check for restart information file and set the current leg start date
    #   Ignore restart information file if force_run_from_scratch is true
    if ${force_run_from_scratch} || ! [ -r ${ece_info_file} ]
    then
        leg_is_restart=false
        leg_start_date=${run_start_date}
        leg_number=1
    else
        leg_is_restart=true
        . ./${ece_info_file}
        leg_start_date=${leg_end_date}
        leg_number=$((leg_number+1))
    fi

    # Compute the end date of the current leg
    if [ -n "${rst_freq}" ]
    then
        leg_end_date=$(date -uR -d "${leg_start_date} + ${rst_freq}")
    else
        leg_end_date=${run_end_date}
    fi

    if [ $(date -d "${leg_end_date}" +%s) -gt $(date -d "${run_end_date}" +%s) ]
    then
        leg_end_date=${run_end_date}
    fi

    # Some time variables needed later
    leg_length_sec=$(( $(date -d "${leg_end_date}" +%s) - $(date -d "${leg_start_date}" +%s) ))
    leg_start_sec=$(( $(date -d "${leg_start_date}" +%s) - $(date -d "${run_start_date}" +%s) ))
    leg_end_sec=$(( $(date -d "${leg_end_date}" +%s) - $(date -d "${run_start_date}" +%s) ))
    leg_start_date_yyyymmdd=$(date -u -d "${leg_start_date}" +%Y%m%d)
    leg_start_date_yyyy=$(date -u -d "${leg_start_date}" +%Y)
    leg_end_date_yyyy=$(date -u -d "${leg_end_date}" +%Y)
    leg_length_years=$(( ${leg_end_date_yyyy} - ${leg_start_date_yyyy} ))

    # Check whether there's actually time left to simulate - exit otherwise
    if [ ${leg_length_sec} -le 0 ]
    then
        info "Leg start date equal to or after end of simulation."
        info "Nothing left to do. Exiting."
        exit 0
    fi

    # -------------------------------------------------------------------------
    # *** Prepare the run directory for a run from scratch
    # -------------------------------------------------------------------------
    if ! $leg_is_restart
    then
        # ---------------------------------------------------------------------
        # *** Check if run dir is empty. If not, and if we are allowed to do so
        #     by ${force_run_from_scratch}, remove everything
        # ---------------------------------------------------------------------
        if $(ls * >& /dev/null)
        then
            if ${force_run_from_scratch}
            then
                rm -fr ${run_dir}/*
            else
                error "Run directory ${run_dir} not empty and \$force_run_from_scratch not set."
            fi
        fi

        # ---------------------------------------------------------------------
        # *** Copy executables of model components
        # *** Additionally, create symlinks to the original place for reference
        # ---------------------------------------------------------------------
        if $(has_config lpjg)
        then
            cp    ${lpjg_exe_file} .
            ln -s ${lpjg_exe_file} $(basename ${lpjg_exe_file}).lnk
        fi

        if $(has_config lpjg_forcing)
        then
          cp ${lpjg_forcing_exe_file} .
        fi

        # ---------------------------------------------------------------------
        # *** Files needed for LPJ-GUESS
        # ---------------------------------------------------------------------
        if $(has_config lpjg)
        then
            # Initial data - saved state for LPJ-GUESS (.bin format)
            lpjgstartdir=$(printf "lpjg_state_%04d" $leg_start_date_yyyy)
            ln -sf ${ini_data_dir}/lpjg/${lpjgstartdir}/ ${run_dir}/${lpjgstartdir}

            # Control files (i.e. .ins, landuse, N deposition, soil type files etc.)
#FIXME: the ins files should not come from two sources and even if, the source dir should be the first that is copied
#and then insfiles out of the ini or some experiment data dir could overwrite some of the defaults.
            #cp  ${ini_data_dir}/lpjg/insfiles/* ${run_dir}
            cp  -f ${ecearth_src_dir}/lpjg/data/ins/*.ins .
            mkdir -p ${run_dir}/landuse
            cp  ${ini_data_dir}/lpjg/landuse/* ${run_dir}/landuse
            mkdir -p ${run_dir}/ndep
            cp  ${ini_data_dir}/lpjg/ndep/* ${run_dir}/ndep
            cp  ${ini_data_dir}/lpjg/ndep/* ${run_dir}
            cp  ${ini_data_dir}/ifs/cmip6-data/CMIP6_histo_mole_fraction_of_carbon_dioxide_in_air_input4MIPs_gr1-GMNHSH.nc ${run_dir}

        fi

#TODO ET ask Anthoni for this file
#        if $(has_config GSWP3)
#        then
#           cp -f ${ecearth_src_dir}/lpjg/offline/guess_GSWP3.ins ${run_dir}/guess.ins
#        fi


    else # i.e. $leg_is_restart == true

        # ---------------------------------------------------------------------
        # *** Remove all leftover output files from previous legs
        # ---------------------------------------------------------------------

        # OSM lpjg files ????

        echo nothing to do

    fi # ! $leg_is_restart

    # -------------------------------------------------------------------------
    # *** Create some control files
    # -------------------------------------------------------------------------

    # namelist and namcouple files

    has_config lpjg_forcing  && . ${ctrl_file_dir}/namelist.lpjg_forcing.sh                     > namelist.lpjg_forcing

    . ${ctrl_file_dir}/namcouple.sh > namcouple

    # -------------------------------------------------------------------------
    # *** LPJ-GUESS initial data
    # -------------------------------------------------------------------------
    if $(has_config lpjg)
    then
        # LPJG runtime rcfile - update with leg dates
        . ${ctrl_file_dir}/namelist.lpjg.sh > lpjg_steps.rc

        # Populate or update LPJG run directories
        for (( n=1; n<=${lpjg_numproc}; n++ ))
        do
            if ! $leg_is_restart
            then
                # Make output directories
                # add the ioopt config option if running on one node to speed up output
                if $(has_config lpjg:ioopt)
                then
                    mkdir -p $TMPDIR/lpjg/run${n}/output
                    mkdir -p ${run_dir}/run${n}
                    ln -sf $TMPDIR/lpjg/run${n}/output ${run_dir}/run${n}/output
                else
                    mkdir -p ${run_dir}/run${n}/output
                fi

                # Copy *.ins, lpjg_steps.rc and OASIS-MCT grid and restart files
                cp ${run_dir}/*.ins ${run_dir}/run${n}
                cp ${ini_data_dir}/lpjg/oasismct/*.nc ${run_dir}
                cp ${ini_data_dir}/lpjg/oasismct/*.txt ${run_dir}/run${n}

                # Link OASIS-MCT grid definition files
                # would normally come from oasis/T255-... dir
                for f in areas grids masks
                do
                    ln -sf ${f}lpjg.nc ${f}.nc
                done

            # If num_proc has increased provide additional rundirs.
            elif [ ! -d ${run_dir}/run${n} ]
            then
                mkdir -p ${run_dir}/run${n}/output
                cp ${run_dir}/*.ins ${run_dir}/run${n}
                cp ${ini_data_dir}/lpjg/oasismct/*.txt ${run_dir}/run${n}
            fi
            # Refresh dirs after they hav been removed after end of leg
            mkdir -p ${run_dir}/run${n}/output/CMIP6
        done

        if $leg_is_restart
        then
            lpjg_restart_dir="restart/lpjg/$(printf %03d $((leg_number-1)))"
            lpjg_rst_state="${lpjg_restart_dir}/lpjg_state_${leg_start_date_yyyy}"
            if [ -d "$lpjg_rst_state" ]
            then
                ln -sf $lpjg_rst_state
            else
                echo "lpjg restart dir $lpjg_rst_state not available"
                exit -1
            fi
            # get oasis restart files
            oasis_restart_dir="restart/oasis/$(printf %03d $((leg_number)))"
            cp -f ${oasis_restart_dir}/*.nc ${run_dir}/

        fi
    fi

    # -------------------------------------------------------------------------
    # *** Remove some OASIS files of the previous leg
    # -------------------------------------------------------------------------
    # TODO check oasis restart files
    #if $leg_is_restart
    #then
    #    rm -f anaisout_*
    #fi

    # -------------------------------------------------------------------------
    # *** Start the run
    # -------------------------------------------------------------------------
#needed for osm???
#    export DR_HOOK_IGNORE_SIGNALS='-1'
#    export CPLNG='active'

    has_config lpjg && cmd="${lpjg_numproc} ${lpjg_exe_file} guess.ins -parallel" || error "no lpjg in config"
    has_config lpjg_forcing && cmd=${cmd}" -- ${lpjg_forcing_numproc} ${lpjg_forcing_exe_file}"

    t1=$(date +%s)
    launch $cmd
    t2=$(date +%s)

    tr=$(date -d "0 -$t1 sec + $t2 sec" +%T)

    # -------------------------------------------------------------------------
    # *** Check for signs of success
    #     Note the tests provide no guarantee that things went fine! They are
    #     just based on the IFS, NEMO and TM5 log files. More tests (e.g. checking
    #     restart files) could be implemented.
    # -------------------------------------------------------------------------

    # Check for LPJG success ???

    # -------------------------------------------------------------------------
    # *** Move LPJ-GUESS output files to archive directory
    # -------------------------------------------------------------------------
    if $(has_config lpjg)
    then
        outdir="output/lpjg/$(printf %03d $((leg_number)))"
    if [ -d ${outdir} ]
    then
        rm -rf  ${outdir}
    fi
        mkdir -p ${outdir}

        # LPJG run directories
        # concatenate *.out files from each run* into output dir
        # DMG: adding escaped % for autosubmit, undo when issue #42 is resolved
        if $use_autosubmit
        then
            flist=$(find ${run_dir}/run1/output/ -type f | sed "s%${run_dir}/run1/output\/%%%%")
        else
            flist=$(find ${run_dir}/run1/output/ -type f | sed "s%${run_dir}/run1/output\/%%")
        fi
        mkdir ${outdir}/CMIP6
        for (( n=1; n<=${lpjg_numproc}; n++ ))
    do
        for ofile in $flist
            do
        if (( n == 1 ))
        then
            cat ${run_dir}/run${n}/output/${ofile} > ${outdir}/$ofile
        else
            awk '(FNR!=1){print $0}' ${run_dir}/run${n}/output/${ofile} >> ${outdir}/$ofile
        fi
        done
        rm -rf ${run_dir}/run${n}/output
        done
    # move monthly file if available
    if [ -f ${run_dir}/LPJ-GUESS_monthlyoutput.txt ]
    then
        mv ${run_dir}/LPJ-GUESS_monthlyoutput.txt ${outdir}
    fi
    fi

        #the output might be large, lets gzip those text files
        # this is not in the esm runscript anymore
        find ${outdir} -type f | xargs -I %% gzip %%

    # -------------------------------------------------------------------------
    # *** Move LPJ-GUESS restart files to archive directory
    # -------------------------------------------------------------------------
    if $(has_config lpjg)
    then
        outdir="restart/lpjg/$(printf %03d $((leg_number)))"
        if [ -d ${outdir} ]
        then
            rm -rf  ${outdir}
        fi
        mkdir -p ${outdir}

        state_dir="./lpjg_state_$(printf %04d $((leg_end_date_yyyy)))"
        mv ${state_dir} ${outdir}
        # LPJG writes into run1 dir, so mv to main rundir
        mv -f run1/lpjgv.nc .

        # remove restart link
        if $leg_is_restart
        then
            old_state_dir="./lpjg_state_$(printf %04d $((leg_start_date_yyyy)))"
            if [ -L $old_state_dir ]
            then
                rm -f "$old_state_dir"
            fi
        fi
    fi

    # -------------------------------------------------------------------------
    # *** Move log files to archive directory
    # -------------------------------------------------------------------------
    outdir="log/$(printf %03d $((leg_number)))"
    mkdir -p ${outdir}

    for f in \
        guess.log \
        nout.000000 debug.??.?????? A*_??.nc O*_??.nc
    do
        test -f ${f} && mv ${f} ${outdir}
    done

    # lpjg run logs
    # this is not in the esm runscript anymore
    # other (lpjg_forcing osm) logs ???
    if $(has_config lpjg)
    then
        mkdir -p ${outdir}/lpjg

        for (( n=1; n<=${lpjg_numproc}; n++ ))
        do
            #concat the lpjg run logs
            if (( n == 1 ))
            then
              echo "--------- lpjg run${n} log --------" > ${outdir}/lpjg/guess_runs.log
            else
              echo "--------- lpjg run${n} log --------" >> ${outdir}/lpjg/guess_runs.log
            fi
            cat ${run_dir}/run${n}/guess$((n-1)).log >> ${outdir}/lpjg/guess_runs.log
        done
    fi

    # -------------------------------------------------------------------------
    # *** Copy OASIS restart files to archive directory
    #     NOTE: These files are copied and not moved as they are used in the
    #           next leg!
    #           Note also that the OASIS restart files present at the end of
    #           the leg correspond to the start of the next leg!
    # -------------------------------------------------------------------------
    outdir="restart/oasis/$(printf %03d $((leg_number+1)))"
    mkdir -p ${outdir}

    for f in vegin.nc lpjgv.nc
    do
        test -f ${f} && cp ${f} ${outdir}
    done

    # -------------------------------------------------------------------------
    # *** Write the restart control file
    # -------------------------------------------------------------------------

    # Compute CPMIP performance
    sypd="$(cpmip_sypd $leg_length_sec $(($t2 - $t1)))"
    ncores=0
#    has_config osm       && (( ncores+=${osm_numproc}  )) || :
    has_config lpjg      && (( ncores+=${lpjg_numproc} )) || :
    has_config lpjg_forcing  && (( ncores+=${lpjg_forcing_numproc} )) || :
    chpsy="$(cpmip_chpsy  $leg_length_sec $(($t2 - $t1 +1)) $ncores)" #t_run+1 prevent div zero, just in case

    echo "#"                                             | tee -a ${ece_info_file}
    echo "# Finished leg at `date '+%F %T'` after ${tr} (hh:mm:ss)" \
                                                         | tee -a ${ece_info_file}
    echo "# CPMIP performance: $sypd SYPD   $chpsy CHPSY"| tee -a ${ece_info_file}
    echo "leg_number=${leg_number}"                      | tee -a ${ece_info_file}
    echo "leg_start_date=\"${leg_start_date}\""          | tee -a ${ece_info_file}
    echo "leg_end_date=\"${leg_end_date}\""              | tee -a ${ece_info_file}

    # Need to reset force_run_from_scratch in order to avoid destroying the next leg
    force_run_from_scratch=false

done # loop over legs

# -----------------------------------------------------------------------------
# *** Platform dependent finalising of the run
# -----------------------------------------------------------------------------
finalise

$use_autosubmit || exit 0