#!/bin/bash
#SBATCH --job-name=ifs_nemo_H
#SBATCH --partition=all --exclusive
#SBATCH --nodes=63
#SBATCH --ntasks-per-node=28
#SBATCH --output=out/[[[MOD:GENERAL:EXP_NAME]]].%j.out
########SBATCH --time=04:00:00
####### --cpu_bind=none
####### -L lustre_na*4

export INTEL_LICENSE_FILE=/opt/intel/licenses
COMPILO_ROOTDIR=/opt/intel/compilers_and_libraries_2017.2.174/linux
source $COMPILO_ROOTDIR/bin/compilervars_arch.sh intel64 linux
source $COMPILO_ROOTDIR/mkl/bin/mklvars.sh     intel64
source /opt/intel/impi/5.1.3.258/intel64/bin/mpivars.sh

set -e

echo "nodeset : $SLURM_NODELIST and current frequency:"
clush -w $SLURM_NODELIST cat /sys/devices/system/cpu/cpu[0-9]*/cpufreq/scaling_cur_freq 2>/dev/null | sort -u | awk '{print $2}' | sort -u

./ece-ifs+nemo.sh


# #PLS ------ HERE you can put tailored copy of ece-ifs+nemo.sh.tmpl --------------

# Some possible changes:


# if [ ! -d ${run_dir} ]
# then
#     mkdir -p ${run_dir}
#     lfs setstripe -c 12 ${run_dir}
# fi
# cd ${run_dir}



#     # Use the launch function from the platform configuration file
#     t1=$(date +%s)
#     #DC launch \
#     #DC     ${xio_numproc} ${xio_exe_file} -- \
#     #DC     ${nem_numproc} ${nem_exe_file} -- \
#     #DC     ${rnf_numproc} ${rnf_exe_file} -- \
#     #DC     ${ifs_numproc} ${ifs_exe_file} -v ecmwf -e ${exp_name}
#     xio_task0=0
#     xio_task1=$((xio_task0+xio_numproc-1))
#     nem_task0=$((xio_task1+1))
#     nem_task1=$((nem_task0+nem_numproc-1))
#     rnf_task0=$((nem_task1+1))
#     rnf_task1=$((rnf_task0+rnf_numproc-1))
#     ifs_task0=$((rnf_task1+1))
#     ifs_task1=$((ifs_task0+ifs_numproc-1))
#     echo "${xio_task0}-${xio_task1} ${xio_exe_file}"                          >conf.txt
#     echo "${nem_task0}-${nem_task1} ${nem_exe_file}"                         >>conf.txt
#     echo "${rnf_task0}-${rnf_task1} ${rnf_exe_file}"                         >>conf.txt
#     echo "${ifs_task0}-${ifs_task1} ${ifs_exe_file} -v ecmwf -e ${exp_name}" >>conf.txt

#     NBTASKS=$((xio_numproc+nem_numproc+rnf_numproc+ifs_numproc))
#     export OMP_NUM_THREADS=1
#     case $OMP_NUM_THREADS in
#        1) BINDING=map_cpu:$LIST_CORES ;;
#        2) MASK=`~/KNMI/mytools/build_mask.14cores.sh 28x2`
#           BINDING=mask_cpu:$MASK ;;
#        *) echo "$OMP_NUM_THREADS OpenMP not yet implemented." ; exit ;;
#     esac
#     launch_atos conf.txt $NODES_ECE_IFS_NEMO $NBTASKS $BINDING $OMP_NUM_THREADS
#     t2=$(date +%s)


#     # This takes care of a special IFS feature: The output for the last time
#     # step of each leg is written at the first time step of the new leg. The
#     # following code makes sure that the output is appended to the appropriate
#     # file. Since GRIB files are just streams, its done with a simple cat
#     # command.
#     for f in ICMSH${exp_name}+?????? ICMGG${exp_name}+??????
#     do
#         if [ -f output/ifs/${prv_leg}/${f} ]
#         then
#             cat ${f} >> output/ifs/${prv_leg}/${f}
#             rm -f ${f}
#         else
#             mv ${f} ${outdir}
#         fi
#     done
#     postprocess ${outdir} $LAST_NODE
#     cd ${run_dir}

# ...
# done # loop over legs
# # attente du dernier postprocessing
# wait
# PP_DATE_END=`date +"%H %M %S"`
# echo "on node $LAST_NODE post processing ends at : $PP_DATE_END"
#
# # -----------------------------------------------------------------------------
# # *** Platform dependent finalising of the run
# # -----------------------------------------------------------------------------
# finalise