ecearth3/runtime/classic/platform/ecmwf-cca-intel.cfg.tmpl

# Platform dependent configuration functions for the CRAY at ECMWF machine

function configure()
{
    # This function should configure all settings/modules needed to 
    # later prepare the EC-Earth run directory and set variables used 
    # in the run script

    # Configure paths for building/running EC-Earth
    ecearth_src_dir=[[[PLT:ACTIVE:ECEARTH_SRC_DIR]]]
    run_dir=[[[PLT:ACTIVE:RUN_DIR]]]
    ini_data_dir=[[[PLT:ACTIVE:INI_DATA_DIR]]]

    # File for standard output.
    # NOTE: This will be modified for restart jobs!
    stdout_file=${start_dir}/out/$exp_name.out

    # Resubmit this job for automatic restarts? [true/false]
    # Also, add options for the resubmit command here.
    resubmit_job=[[[PLT:ACTIVE:RESUBMIT_JOB]]]
    resubmit_opt="[[[PLT:ACTIVE:RESUBMIT_OPT]]]"

    # Configure GRIBEX paths
    export LOCAL_DEFINITION_TEMPLATES=[[[PLT:ACTIVE:GRIBEX_DEFINITION_PATH]]]
    export ECMWF_LOCAL_TABLE_PATH=[[[PLT:ACTIVE:GRIBEX_DEFINITION_PATH]]]

    # Configure grib api paths
    export GRIB_DEFINITION_PATH=[[[PLT:ACTIVE:ECEARTH_SRC_DIR]]]/util/grib_table_126:[[[PLT:ACTIVE:GRIBAPI_BASE_DIR]]]/[[[PLT:ACTIVE:GRIBAPI_DEFINITION_SUBDIR]]]
    export GRIB_SAMPLES_PATH=[[[PLT:ACTIVE:GRIBAPI_BASE_DIR]]]/[[[PLT:ACTIVE:GRIBAPI_SAMPLES_SUBDIR]]]
    export GRIB_BIN_PATH=[[[PLT:ACTIVE:GRIBAPI_BASE_DIR]]]/[[[PLT:ACTIVE:GRIBAPI_BIN_SUBDIR]]]

    export emos_tool=/usr/local/apps/libemos/000458/GNU/63/bin/emos_tool

    # Configure number of processors per node
    proc_per_node=[[[PLT:ACTIVE:PROC_PER_NODE]]]

    # Configure and load modules
    pre_load_modules_cmd="[[[PLT:ACTIVE:PRE_LOAD_MODULES_CMD]]]"
    module_list="[[[PLT:ACTIVE:MODULE_LIST]]]"

    # Load any required modules
    if [[ -n "$module_list" ]]
    then
        if [ -n "${pre_load_modules_cmd}" ]
        then
            eval $(echo "${pre_load_modules_cmd}")
        fi
        for m in ${module_list}
        do
            module load $m
        done
        module list
    fi

    # Add directories to the shared library search path
    if [ -n "[[[PLT:ACTIVE:ADD_TO_LD_LIBRARY_PATH]]]" ]
    then
        export LD_LIBRARY_PATH=${LD_LIBRARY_PATH:+${LD_LIBRARY_PATH}:}"[[[PLT:ACTIVE:ADD_TO_LD_LIBRARY_PATH]]]"
    fi

    #hostname
    ulimit -s unlimited
    ulimit -v unlimited
    #ulimit -n 2048
    #ulimit -a
}

function configure_python()
{
    # specific for python+eccodes setup - used for OSM pre/post-processing
    module rm grib_api
    module load eccodes
    module unload python
    module load python/2.7.15-01
    unset GRIB_DEFINITION_PATH
    unset GRIB_SAMPLES_PATH
    unset GRIB_BIN_PATH
}

function launch()
{
    use_forking=`echo "[[[PLT:ACTIVE:USE_FORKING]]]" | tr '[:upper:]' '[:lower:]'`
    if [ "${use_forking}" = "true" ]
    then
        launch_forking $*
    else
        launch_default $*
    fi
}

function launch_default()
{    
    # version using aprun
    cmd="aprun"

    # account for possible user tasks_per_node request
    IFS=':' read -ra tasks_per_node <<< "$EC_tasks_per_node"
    read hyperth <<< "$EC_hyperthreads"

    i=0
    while (( "$#" ))
    do
        nranks=$1
        executable=./$(basename $2)
        shift
        shift
        if [ "$executable" == "./master1s.exe" -o "$executable" == "./master1s_cpl.exe" ]
        then
          cmd+=" -d ${OMP_NUM_THREADS:-1} -cc depth  $executable"
        else
          cmd+=" -n $nranks -N ${tasks_per_node[$i]} -j ${hyperth} $executable"
        fi
        
        while (( "$#" )) && [ "$1" != "--" ]
        do
            cmd+=" $1"
            shift
        done
        shift || true
        
        (( "$#" )) && cmd+=" :"
        ((i+=1))
    done

    echo $cmd
    $cmd 
}

function launch_forking()
{
    # Version using launcher script to allow node sharing.
    LAUNCHER="launch.sh"

    nproc=0
    cat <<-EOF >$LAUNCHER # Indentation with tabs, not spaces!!
	#!/bin/ksh
	rank=\$EC_FARM_ID
	export OMP_NUM_THREADS=1
	if false # just to manage all the other test cases uniformly with an elif
	then
		:
	EOF

    # Create the launcher script
    while [ $# -gt 0 ]
    do
        nranks=$1
        executable=./$(basename $2)
        shift # nranks
        shift # executable

        ((nproc+=nranks))

        while [ $# -gt 0 ] && [ "$1" != "--" ]
        do
            executable="$executable $1"
            shift # argument
        done
        shift || true # separator -- or nothing

        cat <<-EOF >>$LAUNCHER # Indentation with tabs, not spaces!!
		elif [ \$rank -lt $nproc ]
		then
			exec $executable
		EOF
    done
    cat <<-EOF >>$LAUNCHER # Indentation with tabs, not spaces!!
	fi
	EOF

    # Create the command to execute
    IFS=':' read -ra total_tasks <<< "$EC_total_tasks"
    IFS=':' read -ra tasks_per_node <<< "$EC_tasks_per_node"
    read hyperth <<< "$EC_hyperthreads"

    i=0
    nproc_header=0
    cmd="aprun"
    for n in ${total_tasks[*]}
    do
        cmd+=" -n $n -N ${tasks_per_node[$i]} -j ${hyperth} ./${LAUNCHER}"
        ((i+=1))
        ((nproc_header+=n))
        [ "$nproc_header" -lt "$nproc" ] && cmd+=" :"
    done
    # Check that the total number of tasks in the job file matches the ecconf settings
    if [ "$nproc_header" -ne "$nproc" ]
    then
        echo "The job header total tasks does not match the ecconf settings"
        exit 2
    fi

    chmod +x $LAUNCHER
    export PMI_NO_FORK=1
    echo $cmd
    $cmd
}

function finalise()
{
    # This function should execute of any post run functionality, e.g. 
    # platform dependent cleaning or a resubmit

    if ${resubmit_job} && [ $(date -d "${leg_end_date}" +%s) -lt $(date -d "${run_end_date}" +%s) ]
    then
        info "Resubmitting job for leg $((leg_number+1))"

        # go to submit dir
        cd ${PBS_O_WORKDIR}  # same as ${start_dir}

        ##  # FOLLOWING WILL NOT WORK, BECAUSE QSUB OPTIONS ARE IGNORED
        ##  
        ##  but the qsub options below are ignored
        ##  has_config xios      && ectasks=1
        ##  has_config nemo      && ectasks=${ectasks}:${nem_numproc}
        ##  has_config rnfmapper && ectasks=${ectasks}:1
        ##  has_config ifs       && ectasks=${ectasks}:${ifs_numproc}
        ##  has_config tm5       && ectasks=${ectasks}:${tm5_numproc}
        ##   
        ##  ectasks=$(echo ${ectasks} | sed "s/^://")
        ##  ecthreads=$(echo ${ectasks} | sed "s/[^:]\+/1/g")
        ##   
        ##  qsub -N eceDev \
        ##      -q np \
        ##      -l EC_billing_account=spnltune \
        ##      -l EC_total_tasks=${ectasks} \
        ##      -l EC_threads_per_task=$ecthreads} \
        ##      -l EC_memory_per_task=2GB \
        ##      -l EC_hyperthreads=1 \
        ##      -j oe \
        ##      -o ${run_dir}/$(basename ${stdout_file}).$(printf %03d $((leg_number+1))) \
        ##       ./${PBS_JOBNAME}

        # So use sed instead 
        log=$(basename ${stdout_file}).$(printf %03d $((leg_number+1)))

        cp ./${PBS_JOBNAME} ./${PBS_JOBNAME}.$$
     
        sed "s:#PBS -o out/.*:#PBS -o out/${log}:" \
            <./${PBS_JOBNAME}.$$ \
            >./${PBS_JOBNAME}

        \rm -f ./${PBS_JOBNAME}.$$

        # Submit command
        set -x
        qsub ./${PBS_JOBNAME}
        set +x
    else
        info "Not resubmitting."
    fi

}