ecearth3/runtime/classic/ece-ifs.sh.tmpl

#!/usr/bin/env bash

set -ue

# librunscript defines some helper functions
source ./librunscript.sh

# =============================================================================
# *** BEGIN User configuration
# =============================================================================

# -----------------------------------------------------------------------------
# *** General configuration
# -----------------------------------------------------------------------------

# Component configuration
# (for syntax of the $config variable, see librunscript.sh)
config="ifs amip"

# Experiment name (exactly 4 letters!)
exp_name=[[[MOD:GENERAL:EXP_NAME]]]

# Simulation start and end date. Use any (reasonable) syntax you want.
run_start_date="[[[MOD:GENERAL:RUN_START_DATE]]]"
run_end_date="[[[MOD:GENERAL:RUN_END_DATE]]]"

# Set $force_run_from_scratch to 'true' if you want to force this run to start
# from scratch, possibly ignoring any restart files present in the run
# directory. Leave set to 'false' otherwise.
# NOTE: If set to 'true' the run directory $run_dir is cleaned!
force_run_from_scratch=[[[MOD:GENERAL:FORCE_RUN_FROM_SCRATCH]]]

# Resolution
ifs_grid=[[[MOD:IFS:GRID]]]

# Restart frequency. Use any (reasonable) number and time unit you want.
# For runs without restart, leave this variable empty
rst_freq="[[[MOD:GENERAL:RST_FREQ]]]"

# Number of restart legs to be run in one go
run_num_legs=[[[MOD:GENERAL:RUN_NUM_LEGS]]]

# Coupling frequency [sec]
# Don't change the coupling frequency because UPDCLIE (where SST and SIC
# are updated) is called every 24 hours (hardcoded in ifs-36r4/src/ifs/utility/updtim.F90)
cpl_freq_amip_sec=86400

# Directories
start_dir=${PWD}
ctrl_file_dir=${start_dir}/ctrl
output_control_files_dir=${start_dir}/[[[MOD:GENERAL:OUTPUT_CONTROL_FILES_DIR]]]

# Architecture
build_arch=[[[PLT:ACTIVE:BUILD_ARCH]]]

# This file is used to store information about restarts
ece_info_file="ece.info"

# -----------------------------------------------------------------------------
# *** Read platform dependent configuration
# -----------------------------------------------------------------------------
source ./ecconf.cfg

configure

# -----------------------------------------------------------------------------
# *** Time step settings
# -----------------------------------------------------------------------------
case "${ifs_grid}" in

    T159L*) ifs_time_step_sec=3600 ;;
    T255L*) ifs_time_step_sec=2700 ;;
    T511L*) ifs_time_step_sec=900  ;;

    *)  error "Can't set time steps for unknown horizontal grid: ${ifs_grid}"
        ;;
esac

# -----------------------------------------------------------------------------
# *** IFS configuration
# -----------------------------------------------------------------------------

ifs_version=36r4

ifs_di_freq=$(( 24 * 3600 / ifs_time_step_sec ))
ifs_ddh_freq=$(( 120 * 3600 / ifs_time_step_sec ))

ifs_res_hor=$(echo ${ifs_grid} | sed 's:T\([0-9]\+\)L\([0-9]\+\):\1:')
ifs_res_ver=$(echo ${ifs_grid} | sed 's:T\([0-9]\+\)L\([0-9]\+\):\2:')

ifs_numproc=[[[MOD:IFS:NUMPROC]]]

ifs_exe_file=${ecearth_src_dir}/ifs-${ifs_version}/bin/ifsmaster-${build_arch}

ifs_lastout=false

ifs_cmip5=[[[MOD:IFS:CMIP5]]]
ifs_cmip5_rcp=[[[MOD:IFS:CMIP5_RCP]]]
ifs_cmip_fixyear=[[[MOD:IFS:CMIP_FIXYEAR]]]

ifs_cmip6=[[[MOD:IFS:CMIP6]]]
ifs_mac2sp=[[[MOD:IFS:CMIP6]]]
ifs_cmip6piaer=[[[MOD:IFS:CMIP6]]]
ifs_cmip6_scenario=[[[MOD:IFS:CMIP6_SCENARIO]]]

lcmip6_strataer_simp=FALSE
lcmip6_strataer_full=[[[MOD:IFS:CMIP6]]]
lcmip6_strataer_bckgd=FALSE

ifs_A4xCO2=[[[MOD:IFS:CMIP6_A4xCO2]]]
ifs_1PCTCO2=[[[MOD:IFS:CMIP6_1PCTCO2]]]

# Time-varying orbital forcing (Qiong Zhang, SU-2013-09)
# https://dev.ec-earth.org/projects/ecearth3/wiki/Orbital_forcing_in_EC-Earth_3
#
#   ifs_orb_switch=FALSE, no orbital calculations applied
#   ifs_orb_switch=TRUE, use orbital calculations according to ifs_orb_mode
#   ifs_orb_mode="fixed_year", or "variable_year", or "fixed_parameters"
#     fixed_year: calculate the orbital parameters at ifs_orb_iyear, e.g.,1850
#     variable_year: calculate orbital parameters annually start from ifs_orb_iyear
#     fixed_parameters: prescribe orbital parameters for given year
ifs_orb_switch=FALSE
ifs_orb_mode="variable_year"
ifs_orb_iyear=$(date -u -d "${run_start_date}" +%Y)

# IFS tuning parameters
ifs_tuning_parameter_file=${ctrl_file_dir}/ifs-tuning-parameters-${ifs_grid}.sh
if [ -f ${ifs_tuning_parameter_file} ]
then
    source ${ifs_tuning_parameter_file}
else
    error "Sorry, ${ifs_tuning_parameter_file} not found, exiting."
fi

# Select source of vegetation data:
# ifs       climatology from IFS
# era20c    vegetation from an off-line LPJ-Guess run forced with ERA20C
#           (currently available only for T255 and T159)
#
ifs_veg_source="era20c"

case ${ifs_veg_source} in
"ifs" )
    # Use Lambert-Beer to compute effective vegetation cover
    n_compute_eff_veg_fraction=2
    ;;
"era20c" )
    # LPJG vegetation is provided as effective cover
    # Don't use Lambert-Beer
    n_compute_eff_veg_fraction=0
    ;;
* )
    error "Vegetation from ${ifs_veg_source} not implemented"
    ;;
esac

# use DMI land ice physics and varying snow albedo
ifs_landice=false

# -----------------------------------------------------------------------------
# *** AMIP-reader configuration
# -----------------------------------------------------------------------------

amip_exe_file=${ecearth_src_dir}/amip-forcing/bin/amip-forcing.exe
amip_numproc=1

# -----------------------------------------------------------------------------
# *** OASIS configuration
# -----------------------------------------------------------------------------

# Decide whether the OASIS weight files for interpolation should be linked from
# the setup directory (true) or not (false). In the latter case, the weights
# are re-computed at the start of the run.
oas_link_weights=true

# -----------------------------------------------------------------------------
# *** Extra initial conditions saved during the run
# -----------------------------------------------------------------------------
if has_config save_ic
then
    source ./libsave_ic.sh
    declare -a save_ic_date save_ic_date1 save_ic_sec save_ic_day save_ic_ppt_file save_ic_nemo_ts
fi

# =============================================================================
# *** END of User configuration
# =============================================================================

# =============================================================================
# *** This is where the code begins ...
# =============================================================================

# -----------------------------------------------------------------------------
# *** Create the run dir if necessary and go there
#     Everything is done from here.
# -----------------------------------------------------------------------------
if [ ! -d ${run_dir} ]
then
    mkdir -p ${run_dir}
fi
cd ${run_dir}

# -----------------------------------------------------------------------------
# *** Determine the time span of this run and whether it's a restart leg
# -----------------------------------------------------------------------------

# Regularise the format of the start and end date of the simulation
run_start_date=$(date -uR -d "${run_start_date}")
run_end_date=$(date -uR -d "${run_end_date}")


# -----------------------------------------------------------------------------
# *** Set path to grib_set
# -----------------------------------------------------------------------------

grib_set=${GRIB_BIN_PATH}${GRIB_BIN_PATH:+/}grib_set

# Loop over the number of legs
for (( ; run_num_legs>0 ; run_num_legs-- ))
do

    # Check for restart information file and set the current leg start date
    #   Ignore restart information file if force_run_from_scratch is true
    if ${force_run_from_scratch} || ! [ -r ${ece_info_file} ]
    then
        leg_is_restart=false
        leg_start_date=${run_start_date}
        leg_number=1
    else
        leg_is_restart=true
        . ./${ece_info_file}
        leg_start_date=${leg_end_date}
        leg_number=$((leg_number+1))
    fi

    # Compute the end date of the current leg
    if [ -n "${rst_freq}" ]
    then
        leg_end_date=$(date -uR -d "${leg_start_date} + ${rst_freq}")
    else
        leg_end_date=${run_end_date}
    fi

    if [ $(date -d "${leg_end_date}" +%s) -ge $(date -d "${run_end_date}" +%s) ]
    then
        leg_end_date=${run_end_date}
        ifs_lastout=true
    fi

    # Some time variables needed later
    leg_length_sec=$(( $(date -d "${leg_end_date}" +%s) - $(date -d "${leg_start_date}" +%s) ))
    leg_start_sec=$(( $(date -d "${leg_start_date}" +%s) - $(date -d "${run_start_date}" +%s) ))
    leg_end_sec=$(( $(date -d "${leg_end_date}" +%s) - $(date -d "${run_start_date}" +%s) ))
    leg_start_date_yyyymmdd=$(date -u -d "${leg_start_date}" +%Y%m%d)
    leg_start_date_yyyy=$(date -u -d "${leg_start_date}" +%Y)
    leg_end_date_yyyy=$(date -u -d "${leg_end_date}" +%Y)

    # Check whether there's actually time left to simulate - exit otherwise
    if [ ${leg_length_sec} -le 0 ]
    then
        info "Leg start date equal to or after end of simulation."
        info "Nothing left to do. Exiting."
        exit 0
    fi

    # Initial conditions saved during the run
    do_save_ic=false
    has_config save_ic && save_ic_get_config
    # if you do not use an option with save_ic, you must define 'do_save_ic' and
    # 'save_ic_date_offset' here or in ../libsave_ic.sh/save_ic_get_config()
    ${do_save_ic} && save_ic_define_vars

    # -------------------------------------------------------------------------
    # *** Prepare the run directory for a run from scratch
    # -------------------------------------------------------------------------
    if ! $leg_is_restart
    then
        # ---------------------------------------------------------------------
        # *** Check if run dir is empty. If not, and if we are allowed to do so
        #     by ${force_run_from_scratch}, remove everything
        # ---------------------------------------------------------------------
        if $(ls * >& /dev/null)
        then
            if ${force_run_from_scratch}
            then
                rm -fr ${run_dir}/*
            else
                error "Run directory not empty and \$force_run_from_scratch not set."
            fi
        fi

        # ---------------------------------------------------------------------
        # *** Copy executables of model components
        # *** Additionally, create symlinks to the original place for reference
        # ---------------------------------------------------------------------
        cp    ${ifs_exe_file} .
        ln -s ${ifs_exe_file} $(basename ${ifs_exe_file}).lnk

        cp    ${amip_exe_file} .
        ln -s ${amip_exe_file} $(basename ${amip_exe_file}).lnk

        # ---------------------------------------------------------------------
        # *** Files needed for IFS (linked)
        # ---------------------------------------------------------------------

        # Initial data
        ln -s \
        ${ini_data_dir}/ifs/${ifs_grid}/${leg_start_date_yyyymmdd}/ICMGGECE3INIUA \
                                                            ICMGG${exp_name}INIUA
        ln -s \
        ${ini_data_dir}/ifs/${ifs_grid}/${leg_start_date_yyyymmdd}/ICMSHECE3INIT \
                                                            ICMSH${exp_name}INIT
        rm -f ICMGG${exp_name}INIT
        cp ${ini_data_dir}/ifs/${ifs_grid}/${leg_start_date_yyyymmdd}/ICMGGECE3INIT \
                                                            ICMGG${exp_name}INIT

        # add bare_soil_albedo to ICMGG*INIT
        tempfile=tmp.$$
        ${grib_set} -s dataDate=$(date -u -d "$run_start_date" +%Y%m%d) \
            ${ini_data_dir}/ifs/${ifs_grid}/climate/bare_soil_albedos.grb \
            ${tempfile}

        cat ${tempfile} >> ICMGG${exp_name}INIT
        rm -f ${tempfile}

        # add land ice mask if needed
        if ${ifs_landice}
        then
            tempfile=tmp.$$
            cdo divc,10 -setcode,82 -selcode,141 ICMGG${exp_name}INIT ${tempfile}
            ${grib_set} -s gridType=reduced_gg ${tempfile} ${tempfile}
            cat ${tempfile} >> ICMGG${exp_name}INIT
            rm -f ${tempfile}
        fi

        # Other stuff
        ln -s ${ini_data_dir}/ifs/rtables/* .

        # Output control (ppt files)
        if [ ! -f ${output_control_files_dir}/pptdddddd0600 ] &&  [ ! -f ${output_control_files_dir}/pptdddddd0300 ];then
           echo "Error from ece-esm.sh: Neither the file pptdddddd0600 or pptdddddd0300 exists in the directory:"
           echo " " ${output_control_files_dir}
           exit -1
        fi
        mkdir postins
        cp ${output_control_files_dir}/ppt* postins/
        if [ -f postins/pptdddddd0600 ];then
           ln -s pptdddddd0600 postins/pptdddddd0000
           ln -s pptdddddd0600 postins/pptdddddd1200
           ln -s pptdddddd0600 postins/pptdddddd1800
        fi
        if [ -f postins/pptdddddd0300 ];then
           ln -s pptdddddd0300 postins/pptdddddd0900
           ln -s pptdddddd0300 postins/pptdddddd1500
           ln -s pptdddddd0300 postins/pptdddddd2100
           if [ ! -f postins/pptdddddd0600 ];then
               ln -s pptdddddd0300 postins/pptdddddd0000
               ln -s pptdddddd0300 postins/pptdddddd0600
               ln -s pptdddddd0300 postins/pptdddddd1200
               ln -s pptdddddd0300 postins/pptdddddd1800
           fi
        fi
        /bin/ls -1 postins/* > dirlist

        # ---------------------------------------------------------------------
        # *** Files needed for OASIS (linked)
        # ---------------------------------------------------------------------

        oas_grid_dir=${ini_data_dir}/oasis/AMIP

        # Name table file
        ln -s ${ini_data_dir}/oasis/cf_name_table.txt

        # Grid definition files
        ln -s ${oas_grid_dir}/areas.nc
        ln -s ${oas_grid_dir}/grids.nc
        ln -s ${oas_grid_dir}/masks.nc

        # Weight files
        case ${ifs_res_hor} in
            159)  oas_agrd=080
                  ;;
            255)  oas_agrd=128
                  ;;
            511)  oas_agrd=256
                  ;;
            799)  oas_agrd=400
                  ;;
            *)    error "Unsupported horizontal resolution (IFS): ${ifs_res_hor}"
                  ;;
        esac

        if ${oas_link_weights}
        then
            for f in ${oas_grid_dir}/rmp_AMIP_to_L${oas_agrd}_GAUSWGT.nc
            do
                ln -s $f
            done
        fi

    else # i.e. $leg_is_restart == true

        # ---------------------------------------------------------------------
        # *** Remove all leftover output files from previous legs
        # ---------------------------------------------------------------------

        # IFS files
        rm -f ICM{SH,GG}${exp_name}+??????

    fi # ! $leg_is_restart

    # -------------------------------------------------------------------------
    # *** Initial conditions saved during the run
    # -------------------------------------------------------------------------
    ${do_save_ic} && save_ic_prepare_output

    # -------------------------------------------------------------------------
    # *** Create some control files
    # -------------------------------------------------------------------------

    # IFS frequency output for namelist
    if [ -f postins/pptdddddd0300 ]
    then
        ifs_output_freq=$(( 3 * 3600 / ifs_time_step_sec ))
    elif [ -f postins/pptdddddd0600 ]
    then
        ifs_output_freq=$(( 6 * 3600 / ifs_time_step_sec ))
    else
        error "IFS output frequency undefined."
    fi

    # IFS, AMIP-reader namelists and OASIS namcouple files
    . ${ctrl_file_dir}/namelist.ifs.sh  > fort.4
    . ${ctrl_file_dir}/namelist.amip.sh > namelist.amip
    . ${ctrl_file_dir}/namcouple.sh     > namcouple

    # -------------------------------------------------------------------------
    # *** Create ICMCL file with vegetation fields
    # -------------------------------------------------------------------------
    tempfile=tmp.$$

    case ${ifs_veg_source} in
    "ifs" )
        # Vegetation from IFS (climatology)

        icmclfile=${ini_data_dir}/ifs/${ifs_grid}/climate/ICMCL_ONLY_VEG_PD

        # Create data for december, the year before the leg starts
        ${grib_set} \
            -s dataDate=$(printf "%04d" $((leg_start_date_yyyy-1)))1215 \
            ${icmclfile}-12 ICMCL${exp_name}INIT

        # Create data for all month in the years of the leg
        for (( y=${leg_start_date_yyyy} ; y<=${leg_end_date_yyyy} ; y++ ))
        do
            yy=$(printf "%04d" $y)
            for m in {1..12}
            do
                mm=$(printf "%02d" $m)
                ${grib_set} -s dataDate=${yy}${mm}15 ${icmclfile}-${mm} ${tempfile}
                cat ${tempfile} >> ICMCL${exp_name}INIT
            done
        done

        # Create data for january, the year after the leg ends
        ${grib_set} \
            -s dataDate=$(printf "%04d" $((leg_end_date_yyyy+1)))0115 \
            ${icmclfile}-01 ${tempfile}
        cat ${tempfile} >> ICMCL${exp_name}INIT
        ;;
    "era20c" )
        # Vegetation from an off-line LPJG run forced with ERA20C (v16)

        rm -f ICMCL${exp_name}INIT

        # Create data for all years of the leg, including one year
        # before and one year after
        for (( yr=leg_start_date_yyyy-1 ; yr<=leg_end_date_yyyy+1 ; yr+=1 ))
        do
            if [ $ifs_cmip_fixyear -le 0 ]
            then
                cat ${ini_data_dir}/ifs/${ifs_grid}/icmcl_v16/icmcl_$yr.grb >> ICMCL${exp_name}INIT
            else
                # Fixed year forcing, requires cdo!
                # If cdo is not available at runtime you need to fix proper
                # icmcl files beforehand and use them here
                cdo setyear,$yr ${ini_data_dir}/ifs/${ifs_grid}/icmcl_v16/icmcl_${ifs_cmip_fixyear}.grb ${tempfile}
                cat ${tempfile} >> ICMCL${exp_name}INIT
            fi
        done
        ;;
    * )
        error "Vegetation from ${ifs_veg_source} not implemented"
        ;;
    esac

    # Clean up
    rm -f ${tempfile}

    # -------------------------------------------------------------------------
    # *** Start the run
    # -------------------------------------------------------------------------
    export DR_HOOK_IGNORE_SIGNALS='-1'

    export CPLNG='active'

    # Use the launch function from the platform configuration file
    t1=$(date +%s)
    launch \
        ${ifs_numproc}  ${ifs_exe_file} -v ecmwf -e ${exp_name} -- \
        ${amip_numproc} ${amip_exe_file}
    t2=$(date +%s)

    tr=$(date -d "0 -$t1 sec + $t2 sec" +%T)

    # -------------------------------------------------------------------------
    # *** Check for signs of success
    #     Note the tests provide no guarantee that things went fine! They are
    #     just based on the IFS and NEMO log files. More tests (e.g. checking
    #     restart files) could be implemented.
    # -------------------------------------------------------------------------

    # Checking for IFS success
    if [ -f ifs.stat ]
    then
        if [ "$(awk 'END{print $3}' ifs.stat)" == "CNT0" ]
        then
            info "Leg successfully completed according to IFS log file 'ifs.stat'."
        else
            error "Leg not completed according to IFS log file 'ifs.stat'."
        fi
    else
        error "IFS log file 'ifs.stat' not found after run."
    fi

    # -------------------------------------------------------------------------
    # *** Post-process initial conditions saved during the run if requested
    # -------------------------------------------------------------------------
    ${do_save_ic} && save_ic_postproc

    # -------------------------------------------------------------------------
    # *** Move IFS output files to archive directory
    # -------------------------------------------------------------------------
    outdir="output/ifs/$(printf %03d $((leg_number)))"
    mkdir -p ${outdir}

    prv_leg=$(printf %03d $((leg_number-1)))

    # This takes care of a special IFS feature: The output for the last time
    # step of each leg is written at the first time step of the new leg. The
    # following code makes sure that the output is appended to the appropriate
    # file. Since GRIB files are just streams, its done with a simple cat
    # command.
    for f in ICMSH${exp_name}+?????? ICMGG${exp_name}+??????
    do
        if [ -f output/ifs/${prv_leg}/${f} ]
        then
            cat ${f} >> output/ifs/${prv_leg}/${f}
            rm -f ${f}
        else
            mv ${f} ${outdir}
        fi
    done

    # -------------------------------------------------------------------------
    # *** Move IFS restart files to archive directory
    # -------------------------------------------------------------------------
    if $leg_is_restart
    then
        outdir="restart/ifs/$(printf %03d $((leg_number)))"
        mkdir -p ${outdir}

        # Figure out the time part of the restart files (cf. CTIME on rcf files)
        # NOTE: Assuming that restarts are at full days (time=0000) only!
        nd="$(printf %06d $((leg_start_sec/(24*3600))))0000"

        mv srf${nd}.???? ${outdir}
    fi

    # -------------------------------------------------------------------------
    # *** Copy rcf files to the archive directory (of the next leg!)
    # -------------------------------------------------------------------------
    outdir="restart/ifs/$(printf %03d $((leg_number+1)))"
    mkdir -p ${outdir}

    for f in rcf
    do
        test -f ${f} && cp ${f} ${outdir}
    done

    # -------------------------------------------------------------------------
    # *** Move log files to archive directory
    # -------------------------------------------------------------------------
    outdir="log/$(printf %03d $((leg_number)))"
    mkdir -p ${outdir}

    for f in \
        ifs.log ifs.stat fort.4 amip.log namelist.amip
    do
        test -f ${f} && mv ${f} ${outdir}
    done

    # -------------------------------------------------------------------------
    # *** Write the restart control file
    # -------------------------------------------------------------------------


    # Compute CPMIP performance
    sypd="$(cpmip_sypd $leg_length_sec $(($t2 - $t1)))"
    chpsy="$(cpmip_chpsy  $leg_length_sec $(($t2 - $t1)) $(($ifs_numproc + $amip_numproc)))"

    echo "#"                                             | tee -a ${ece_info_file}
    echo "# Finished leg at `date '+%F %T'` after ${tr} (hh:mm:ss)" \
                                                         | tee -a ${ece_info_file}
    echo "# CPMIP performance: $sypd SYPD   $chpsy CHPSY"| tee -a ${ece_info_file}
    echo "leg_number=${leg_number}"                      | tee -a ${ece_info_file}
    echo "leg_start_date=\"${leg_start_date}\""          | tee -a ${ece_info_file}
    echo "leg_end_date=\"${leg_end_date}\""              | tee -a ${ece_info_file}

    # Need to reset force_run_from_scratch in order to avoid destroying the next leg
    force_run_from_scratch=false

done # loop over legs

# -----------------------------------------------------------------------------
# *** Platform dependent finalising of the run
# -----------------------------------------------------------------------------
finalise

exit 0