ecearth3/runtime/classic/platform/surfsara-cartesius.cfg.tmpl

# Platform dependent configuration functions for the 'neuron' machine
# (KNMI, NL)

#hostlist="${HOME}/bin/ExpandNodeList -r -p 1"

function configure()
{
    # This function should configure all settings/modules needed to 
    # later prepare the EC-Earth run directory and set variables used 
    # in the run script

    # Configure paths for building/running EC-Earth
    export ecearth_src_dir=[[[PLT:ACTIVE:ECEARTH_SRC_DIR]]]
    export run_dir=[[[PLT:ACTIVE:RUN_DIR]]]
    export ini_data_dir=[[[PLT:ACTIVE:INI_DATA_DIR]]]

    # File for standard output.
    # NOTE: This will be modified for restart jobs!
    stdout_file=${start_dir}/out/$(basename ${SLURM_JOB_NAME}).out

    # Resubmit this job for automatic restarts? [true/false]
    # Also, add options for the resubmit command here.
    resubmit_job=[[[PLT:ACTIVE:RESUBMIT_JOB]]]
    resubmit_opt="[[[PLT:ACTIVE:RESUBMIT_OPT]]]"

    # Configure grib api paths
    export GRIB_DEFINITION_PATH=[[[PLT:ACTIVE:ECEARTH_SRC_DIR]]]/util/grib_table_126:[[[PLT:ACTIVE:GRIBAPI_BASE_DIR]]]/[[[PLT:ACTIVE:GRIBAPI_DEFINITION_SUBDIR]]]
    export GRIB_SAMPLES_PATH=[[[PLT:ACTIVE:GRIBAPI_BASE_DIR]]]/[[[PLT:ACTIVE:GRIBAPI_SAMPLES_SUBDIR]]]
    export GRIB_BIN_PATH=[[[PLT:ACTIVE:GRIBAPI_BASE_DIR]]]/[[[PLT:ACTIVE:GRIBAPI_BIN_SUBDIR]]]

    # Configure number of processors per node
    proc_per_node=[[[PLT:ACTIVE:PROC_PER_NODE]]]

    # Configure and load modules
    pre_load_modules_cmd="[[[PLT:ACTIVE:PRE_LOAD_MODULES_CMD]]]"
    module_list="[[[PLT:ACTIVE:MODULE_LIST]]]"

    if [ -n "${module_list}" ]
    then
        set +u
        if [ -n "${pre_load_modules_cmd}" ]
        then
            ${pre_load_modules_cmd}
        fi
        for m in "${module_list}"
        do
            eval $(/usr/libexec/cmod sh add $m)
        done
        set -u
    fi

    # Add directories to the shared library search path
    if [ -n "[[[PLT:ACTIVE:ADD_TO_LD_LIBRARY_PATH]]]" ]
    then
        export LD_LIBRARY_PATH=${LD_LIBRARY_PATH:+${LD_LIBRARY_PATH}:}"[[[PLT:ACTIVE:ADD_TO_LD_LIBRARY_PATH]]]"
    fi

    #hostname
    ulimit -s unlimited
    ulimit -n 2048
    #ulimit -a
}

function launch()
{
     #version using MPMD MPI job script as described on SURFSARA/Cartesius website
     set -x
     cmd="srun -l --multi-prog mfile" #command given at the end of launch
    
#     MYARG="strace -ostrace-out.$SLURM_PROCID -k -eexit_group ./appl-tm5.x --multi-prog mfile"	
#     cmd="srun bash -c \"$MYARG\"" #command given at the end of launch
     
     rm -f mfile    		 #removing old mfile

     proc_begin="0" 	
     while (( "$#" ))
     do
     	nranks=$1
	if [ $2 = "strace" ]; then
		executable="$2 $3 $4 $5 ./$(basename $6)"
		shift
		shift
		shift
		shift
	else
		executable=./$(basename $2)
        fi
	
	shift
	shift
	
	proc_end="$(($proc_begin+$nranks-1))"  	
	
	mfile_line=""
	mfile_line+="$proc_begin-$proc_end $executable"
        
	proc_begin="$(($proc_end+1))"   
      
	while (( "$#" )) && [ "$1" != "--" ] 
	do
	     mfile_line+=" $1"
	     shift        
	done
	shift || true

	echo "$mfile_line" >>  mfile	

     done
#     echo 'strace -k -eexit_group -ostrace.out ./appl-tm5.x' >> mfile
#     echo 'pretty_print_strace_out.py --tree strace.out' >> mfile
#     strace -k -eexit_group -ostrace.out ./appl-tm5.x
#     pretty_print_strace_out.py --tree strace.out
     $cmd	
     set +x

#	set -x
#    # version using the hydra process manager
#    cmd="mpiexec.hydra -bootstrap slurm -machinefile mfile"
#
#    rm -f mfile
#
#    nodes="$(scontrol show hostname $SLURM_NODELIST)"
#    #"$(nodeset -e $SLURM_NODELIST)"
#
#    while (( "$#" ))
#    do
#        nranks=$1
#        executable=./$(basename $2)
#        shift
#        shift
#
#        cmd+=" -n $nranks $executable"
#
#        while (( "$#" )) && [ "$1" != "--" ]
#        do
#            cmd+=" $1"
#            shift
#        done
#        shift || true
#
#        for node in $nodes
#        do
#            (( n = proc_per_node<nranks?proc_per_node:nranks ))
#            #echo "$node:$n" >> mfile
#	    for ((i = 1; i<=n ; i++));do echo "$node" >> mfile; done
#            (( nranks -= n )) || break
#        done
#
#        nodes=$(echo $nodes | sed "s/.*$node[[:space:]]*//")
#        (( "$#" )) && cmd+=" :"
#    done
#
#    export OMP_NUM_THREADS=1
#    # MPIEXEC_START_PROTOCOL=pmi
#    # I_MPI_PMI_EXTENSIONS=on
#    # I_MPI_FABRICS=shm:ofa
#    # export I_MPI_DEBUG=3
#    $cmd
#	set +x
}

function finalise()
{
    # This function should execute of any post run functionality, e.g. 
    # platform dependent cleaning or a resubmit

    if ${resubmit_job} && [ $(date -d "${leg_end_date}" +%s) -lt $(date -d "${run_end_date}" +%s) ]
    then
        info "Resubmitting job for leg $((leg_number+1))"
        # Need to go to start_dir to find the run script
        cd ${start_dir}
        # Submit command
        # Note: This does not work if you specify a job name with sbatch -J jobname!
        sbatch -N ${SLURM_JOB_NUM_NODES}                                                 \
               -n $((ifs_numproc + nem_numproc + tm5_numproc))                           \
               --exclusive                                                               \
               --ntasks-per-node=${proc_per_node}                                        \
               -o ${run_dir}/$(basename ${stdout_file}).$(printf %03d $((leg_number+1))) \
               -e ${run_dir}/$(basename ${stdout_file}).$(printf %03d $((leg_number+1))) \
               -d ${SLURM_JOB_ID}                                                        \
               ${resubmit_opt}                                                           \
               ./${SLURM_JOB_NAME}
    fi

}