!===============================================
! compiler information (GNU compiler [gfortran])
!===============================================

! Gfortran compiler: do not include a compiler.rc, since CRAY uses ftn
! wrapper. Just set flags here. Flags dedicated to hardware are normally
! already included in the ftn wrapper.

compiler.fc             :  ftn
compiler.f77            :  ${compiler.fc}
compiler.fc.openmp      :  ${compiler.fc}
mpi.compiler.fc         :  ${compiler.fc}
mpi.compiler.fc.openmp  :  ${compiler.fc}

! GNU Fortran (GCC) 4.8.2 20131016 (Cray Inc.)
compiler.getversion_flag : --version

compiler.flags.default.fflags       :  -ffree-line-length-none  -cpp -fimplicit-none
compiler.flags.default.ldflags      : 

compiler.flags.real8.fflags         :  -fdefault-real-8
compiler.flags.real8.ldflags        : 

compiler.flags.mpi.fflags           : 
compiler.flags.mpi.ldflags          : 

compiler.flags.openmp.fflags        :  -fopenmp
compiler.flags.openmp.ldflags       :  -fopenmp

compiler.flags.optim-none.fflags    :  -O0
compiler.flags.optim-none.ldflags   : 

! for bit reproducibility (-fno-fast-math instead?)
compiler.flags.optim-strict.fflags  :  -O2 -mavx -ftree-vectorize -ffast-math -funroll-loops
compiler.flags.optim-strict.ldflags :  

! -O3 = O2 plus -finline-functions, -funswitch-loops, -fpredictive-commoning, -fgcse-after-reload, -ftree-loop-vectorize, -ftree-loop-distribute-patterns, -ftree-slp-vectorize, -fvect-cost-model, -ftree-partial-pre and -fipa-cp-clone options.
! -Ofast = O3 plus -ffast-math -fno-protect-parens and -fstack-arrays
compiler.flags.optim-fast.fflags    :  -Ofast -mavx -funroll-loops
compiler.flags.optim-fast.ldflags   :  -Ofast

compiler.flags.optim-vfast.fflags   :  -O5
compiler.flags.optim-vfast.ldflags  :  

compiler.flags.check-all.fflags     : -Wall -fcheck=all -ffpe-trap=zero,invalid
compiler.flags.check-all.ldflags    : 

compiler.flags.debug.fflags         : -fmax-errors=0 -g -fdump-core -fbacktrace -O0
compiler.flags.debug.ldflags        : 

!===============================================
! libraries
!===============================================

! fnt is a wrapper, which already knows all libraries locations. Then we just need to specify the flags here.

! Z library (used for compression in HDF)
compiler.lib.z.fflags               : 
compiler.lib.z.libs                 :  -lz

! JPEG library (used for compression in HDF)
compiler.lib.jpeg.fflags            : 
compiler.lib.jpeg.libs              :  -ljpeg

! SZ library (used for compression in HDF)
SZIP_VERSION                        :  
SZIP_HOME                           :  
compiler.lib.sz.fflags              : 
compiler.lib.sz.libs                :

! HDF4 library:
HDF_VERSION                         : 
HDF_HOME                            : 
compiler.lib.hdf4.fflags            : 
compiler.lib.hdf4.libs              :

! HDF5 library:
HDF5_VERSION                        : 
HDF5_HOME                           : 
compiler.lib.hdf5.fflags            : 
compiler.lib.hdf5.libs              :  -lhdf5_hl -lhdf5

! HDF5 library with parallel features enabled:
HDF5_PAR_VERSION                    :  
HDF5_PAR_HOME                       :  
compiler.lib.hdf5_par.fflags        :  
compiler.lib.hdf5_par.libs          :  -lhdf5_hl -lhdf5

! NetCDF library:
NETCDF_VERSION                      :  
NETCDF_HOME                         :  
compiler.lib.netcdf.fflags          :  
compiler.lib.netcdf.libs            :  -lnetcdff -lnetcdf

! NetCDF4 library:
NETCDF4_VERSION                     :  
NETCDF4_HOME                        :  
compiler.lib.netcdf4.fflags         :  
compiler.lib.netcdf4.libs           :  -lnetcdff -lnetcdf

! NetCDF4 library with parallel features enabled:
NETCDF4_PAR_VERSION                 :  
NETCDF4_PAR_HOME                    :  
compiler.lib.netcdf4_par.fflags     :  
compiler.lib.netcdf4_par.libs       :  -lnetcdff -lnetcdf

! UDUNITS v1.x library (not available)
UDUNITS_HOME                        :  
compiler.lib.udunits1.fflags        :  -I${UDUNITS_HOME}/include
compiler.lib.udunits1.libs          :  -L${UDUNITS_HOME}/lib -ludunits

! UDUNITS v2.x library (available through module, nothing to specify here)
compiler.lib.udunits2.fflags        :  
compiler.lib.udunits2.libs          :  


! OASIS3 library ( ${ECEARTH} is defined in your environment ) 
! Bother with this only if we are within EC-Earth
#if "oasis3" in "${my.tm5.define}":
OASIS3_ARCH                     : aix
OASIS3_MPIV                     : MPI1
OASIS3_HOME                     : ${ECEARTH}/oasis3/prism_2-5/prism/${OASIS3_ARCH}
compiler.lib.oasis3.fflags      :  -I${OASIS3_HOME}/build/lib/psmile.${OASIS3_MPIV}
compiler.lib.oasis3.libs        :  -L${OASIS3_HOME}/lib -lpsmile.${OASIS3_MPIV} -lmpp_io
#endif

! MPI library
compiler.lib.mpi.fflags  : 
compiler.lib.mpi.libs    : 

! GRIBEX library
compiler.lib.gribex.fflags : 
compiler.lib.gribex.libs   :

! GRIB-API library
compiler.lib.grib_api.fflags : ${GRIB_API_INCLUDE}
compiler.lib.grib_api.libs   : ${GRIB_API_LIB}

! LAPACK library:
compiler.lib.lapack.fflags : 
compiler.lib.lapack.libs   : 


!===============================================================
! SETTINGS FOR BATCH SCHEDULER (qsub,bsub,loadleveler,slurm,pbs)
!===============================================================
queue : pbs

! passed directly to qsub command
queue.pbs.submit.options :

! keep default for now. May become an option if needed
hyperthread : 2

! number of tasks (trap non-MPI cases)
#if "${par.mpi}" in ["T","True"] :
my.queue.ntask : ${par.ntask}
#else
my.queue.ntask : 1
#endif

! number of threads (trap non-OpenMP cases)
#if "${par.openmp}" in ["T","True"] :
my.queue.nthread : ${par.nthread}
#else
my.queue.nthread : 1
#endif

!------------------------------------------------------------------
! PBS queue options: if set to <none>, it is skipped, to <auto> it is set in
!                    the python script. If empty it is used as 'option without
!                    argument'. 
!                    
!                    "l" option has special treatment: it is seen as a space-
!                    separated list of options itself, and is skipped if empty.
!                    
!                    See bin/submit_tm5_tools.py/QueueOptions_PBS for details.
!-----------------------------------------------
! ** queue options with same value for ALL steps
!-----------------------------------------------
! space seperated list:
queue.pbs.options.all : j m l
! shell (DO NOT USE IT! KEPT HERE TO REMEMBER)
queue.pbs.option.all.S : /opt/pbs/default/bin/pbs_python
! join output/error log: oe (joined in out) | eo (joined in err) | n (not joined)
queue.pbs.option.all.j : oe
! when to send emails : a (abort) | b (begin) | e (end) | n (never)
queue.pbs.option.all.m : n
! pass environment (ECMWF doc says to be careful)
queue.pbs.option.all.V : 

! account (to overwrite default one attached to $USER)
#if ("${my.queue.account}" == ""):
queue.pbs.option.all.l : 
#else
queue.pbs.option.all.l : EC_billing_account=${my.queue.account}
#endif

!------------------------------------------------
! ** queue options with different values per step
!------------------------------------------------
! list options per step:
queue.pbs.options.init : N q o l
queue.pbs.options.run  : N q o l
queue.pbs.options.done : N q o

! ~~~~~ step init options

queue.pbs.option.init.q : ns
queue.pbs.option.init.N : ${job.name}_init
queue.pbs.option.init.o : <auto>
queue.pbs.option.init.l : EC_ecfs=1

! ~~~~~ step done options

queue.pbs.option.done.q : ns
queue.pbs.option.done.N : ${job.name}_done
queue.pbs.option.done.o : <auto>
queue.pbs.option.done.W : block=true

! ~~~~~ step run options

queue.pbs.option.run.N : ${job.name}_run
queue.pbs.option.run.o : <auto>

! check if run step's gonna be fractional (nf) or full (nq) node occupancy,
! which depends on: hyperthreading (1 or 2), ntasks and nthreads
#if (${my.queue.ntask}*${my.queue.nthread} == 1 ) :
queue.pbs.option.run.q : ns
module.cmd:
#elif (${my.queue.ntask}*${my.queue.nthread} > ${hyperthread}*12 ) :
queue.pbs.option.run.q : np
module.cmd:
#else
queue.pbs.option.run.q : nf
! NOTE: the module command, even with the qualified path, is not found in the
! python script. It is here just to put mpiexec on the $PATH - so we revert to
! using the fully qualified path for mpiexec hereafter. 
module.cmd: 
!module.cmd: /opt/modules/3.2.6.7/bin/modulecmd bash load cray-snplauncher
#endif

#if (${my.queue.nthread} > 1):
queue.pbs.option.run.l : EC_total_tasks=${my.queue.ntask} EC_threads_per_task=${my.queue.nthread} EC_hyperthreads=${hyperthread}
#else
queue.pbs.option.run.l : EC_total_tasks=${my.queue.ntask} EC_hyperthreads=${hyperthread}
#endif

queue.pbs.option.run.W : block=true

! TODO if needed: Resources (thru -l EC_memory_per_task)
!queue.pbs.option.run.resources           : ${my.resource.memory}

!===============================================
! make
!===============================================
! the setup script will insert the 'build.jobs' specified in the expert.rc
! or passed as argument to the setup script:
maker               :  gmake -j %{build.jobs}

! Submit compilation with a job script? 
! - if F (default), pycasso calls the ${maker} 
! - if T, then compilation is submitted according to "submit.to" key (default
!         of which is set in expert.rc, and can be overwritten at the CLI): either "queue", or foreground.
!
! "my.queue.make" is set in the main rc file (so user can switch it on/off from there).
! 
build.make.submit        : ${my.queue.make}
! sub options: assume that %{build.jobs} is less than 24 (default is set in expert, probably to 8)
queue.pbs.options.build  : N q o l
queue.pbs.option.build.q : nf
queue.pbs.option.build.N : build-tm
queue.pbs.option.build.o : <auto>
queue.pbs.option.build.l : EC_total_tasks=${build.jobs} EC_hyperthreads=1

!==========================================================================
! MPI runner (LUCKILY, THE ENVIRONMENT VARIABLES ARE **NOT** EXPANDED UNTIL
! SCRIPTS ARE SUBMITTED). Depends on:
!   (1) pure openMP, pure MPI or hybrid runs
!   (2) node occupancy: fractional or full (see above)
!==========================================================================
#if "${queue.pbs.option.run.q}" == "np":
mpirun.command      :  aprun

#if (${my.queue.nthread} > 1):
! Hybrid (TODO: does it covers pure openMP?)
mpirun.args         :  -N $EC_tasks_per_node -n $EC_total_tasks -d $OMP_NUM_THREADS -j $EC_hyperthreads
#else
!pure MPI
mpirun.args         :  -N $EC_tasks_per_node -n $EC_total_tasks -j $EC_hyperthreads
#endif

#else
mpirun.command      :  /opt/cray/snplauncher/6.3.1/bin/mpiexec
! pure MPI only. TODO: hybrid, and pure openMP
mpirun.args         :  -n $EC_total_tasks
#endif

! name of command and host files (leave empty to not write):
mpirun.cmdfile      :  
mpirun.hostfile     :  

!===============================================
! debugger (FIXME)
!===============================================
! debugger type: totalview | idb | kdbg
debugger : totalview

! command for debugger:
debugger.command : totalview -searchPath=${build.sourcedir}
!debugger.command : totalview

!===============================================
! model data
!===============================================
! the user scratch directory:
my.scratch          :  ${SCRATCH}

! base path to various data files (static):
my.data.dir         :  /perm/ms/nl/nm6/TM5_INPUT

! permanent archives to search for meteo files:
#if "${my.meteo.format}" == "tm5-nc"
my.meteo.search     :  ecfs:/nlh/TM/meteo-nc
#else
my.meteo.search     :  ecfs:/nlh/TM/meteo
#endif

! extra install tasks:
my.install.tasks        :