ecearth3/sources/tm5mp/rc/machine-ecmwf-cca-ifort.rc

!===============================================
! compiler information (Intel compiler [ifort])
!===============================================

! Ifort compiler: do not include a compiler.rc, since CRAY uses ftn
! wrapper. Just set flags here. Flags dedicated to hardware are normally
! already included in the ftn wrapper.

compiler.fc             :  ftn
compiler.f77            :  ${compiler.fc}
compiler.fc.openmp      :  ${compiler.fc}
mpi.compiler.fc         :  ${compiler.fc}
mpi.compiler.fc.openmp  :  ${compiler.fc}

! ifort (IFORT) 14.0.1 20131008 or 15.0.1 20141023
compiler.getversion_flag : --version

compiler.flags.default.fflags       :  -warn declarations -traceback
compiler.flags.default.ldflags      : 

compiler.flags.real8.fflags         :  -real-size 64
compiler.flags.real8.ldflags        : 

compiler.flags.mpi.fflags           : 
compiler.flags.mpi.ldflags          : 

compiler.flags.openmp.fflags        :  -fopenmp
compiler.flags.openmp.ldflags       : 

compiler.flags.optim-none.fflags    :  -O0
compiler.flags.optim-none.ldflags   : 

! for bit reproducibility (note: '-xCORE-AVX-I' is on by default, since hardware related)
compiler.flags.optim-strict.fflags  :  -O2 -fp-model strict -fp-model source
compiler.flags.optim-strict.ldflags :  

compiler.flags.optim-fast.fflags    :  -fast
compiler.flags.optim-fast.ldflags   :  -fast

compiler.flags.optim-vfast.fflags   : 
compiler.flags.optim-vfast.ldflags  : 

! runtime checks
compiler.flags.check-all.fflags     : -check bounds,uninit -ftrapuv -fpe0  -fp-model strict -fp-model source
!compiler.flags.check-all.fflags     : -check bounds -ftrapuv -fpe0
compiler.flags.check-all.ldflags    : 

! copied from pycasso-compiler-ifort-12.1.rc
compiler.flags.debug.fflags         : -g -debug all  -fp-model strict -fp-model source 
compiler.flags.debug.ldflags        : 

!===============================================
! libraries
!===============================================

! fnt is a wrapper, which already knows all libraries locations. Then we just need to specify the flags here.

! Z library (used for compression in HDF)
compiler.lib.z.fflags               : 
compiler.lib.z.libs                 :  -lz

! JPEG library (used for compression in HDF)
compiler.lib.jpeg.fflags            : 
compiler.lib.jpeg.libs              :  -ljpeg

! SZ library (used for compression in HDF)
SZIP_VERSION                        :  
SZIP_HOME                           :  
compiler.lib.sz.fflags              : 
compiler.lib.sz.libs                :

! HDF4 library (without netcdf interface)
HDF_VERSION                         : 
HDF_HOME                            : ${HDF_DIR}
compiler.lib.hdf4.fflags            : -I${HDF_HOME}/include
compiler.lib.hdf4.libs              : -L${HDF_HOME}/lib -lmfhdf -ldf

! HDF5 library:
HDF5_VERSION                        : 
HDF5_HOME                           : 
compiler.lib.hdf5.fflags            : 
compiler.lib.hdf5.libs              :  -lhdf5_hl -lhdf5

! HDF5 library with parallel features enabled:
HDF5_PAR_VERSION                    :  
HDF5_PAR_HOME                       :  
compiler.lib.hdf5_par.fflags        :  
compiler.lib.hdf5_par.libs          :  -lhdf5_hl -lhdf5

! NetCDF library:
NETCDF_VERSION                      :  
NETCDF_HOME                         :  
compiler.lib.netcdf.fflags          :  
compiler.lib.netcdf.libs            :  -lnetcdff -lnetcdf

! NetCDF4 library:
NETCDF4_VERSION                     :  
NETCDF4_HOME                        :  
compiler.lib.netcdf4.fflags         :  
compiler.lib.netcdf4.libs           :  -lnetcdff -lnetcdf

! NetCDF4 library with parallel features enabled:
NETCDF4_PAR_VERSION                 :  
NETCDF4_PAR_HOME                    :  
compiler.lib.netcdf4_par.fflags     :  
compiler.lib.netcdf4_par.libs       :  -lnetcdff -lnetcdf

! UDUNITS v1.x library (not available)
UDUNITS_HOME                        :  
compiler.lib.udunits1.fflags        :  -I${UDUNITS_HOME}/include
compiler.lib.udunits1.libs          :  -L${UDUNITS_HOME}/lib -ludunits

! UDUNITS v2.x library (available through module, nothing to specify here)
compiler.lib.udunits2.fflags        :  
compiler.lib.udunits2.libs          :  


! OASIS3 library ( ${ECEARTH} is defined in your environment ) 
! Bother with this only if we are within EC-Earth
#if "oasis3" in "${my.tm5.define}":
OASIS3_ARCH                     : aix
OASIS3_MPIV                     : MPI1
OASIS3_HOME                     : ${ECEARTH}/oasis3/prism_2-5/prism/${OASIS3_ARCH}
compiler.lib.oasis3.fflags      :  -I${OASIS3_HOME}/build/lib/psmile.${OASIS3_MPIV}
compiler.lib.oasis3.libs        :  -L${OASIS3_HOME}/lib -lpsmile.${OASIS3_MPIV} -lmpp_io
#endif

! MPI library
compiler.lib.mpi.fflags  : 
compiler.lib.mpi.libs    : 

! GRIBEX library
compiler.lib.gribex.fflags : 
compiler.lib.gribex.libs   :

! GRIB-API library
compiler.lib.grib_api.fflags : ${GRIB_API_INCLUDE}
compiler.lib.grib_api.libs   : ${GRIB_API_LIB}

! LAPACK library:
LAPACK_LIB_DIR : /opt/intel/composer_xe_2015.1.133/mkl/lib/intel64
compiler.lib.lapack.fflags : 
compiler.lib.lapack.libs   : -L${LAPACK_LIB_DIR} -lmkl_intel_lp64 -lmkl_core -lmkl_sequential


!===============================================================
! SETTINGS FOR BATCH SCHEDULER (qsub,bsub,loadleveler,slurm,pbs)
!===============================================================
queue : pbs

! passed directly to qsub command
queue.pbs.submit.options :

! If set to 2, we have access to 72 (logical) cores per node, if set to 1,
! only to the 36 physical cores. In both cases, 120 GB per node are available.
hyperthread : 2

! number of tasks (trap non-MPI cases)
#if "${par.mpi}" in ["T","True"] :
my.queue.ntask : ${par.ntask}
#else
my.queue.ntask : 1
#endif

! number of threads (trap non-OpenMP cases)
#if "${par.openmp}" in ["T","True"] :
my.queue.nthread : ${par.nthread}
#else
my.queue.nthread : 1
#endif

!------------------------------------------------------------------
! PBS queue options: if set to <none>, it is skipped, to <auto> it is set in
!                    the python script. If empty it is used as 'option without
!                    argument'. 
!                    
!                    "l" option has special treatment: it is seen as a space-
!                    separated list of options itself, and is skipped if empty.
!                    
!                    See bin/submit_tm5_tools.py/QueueOptions_PBS for details.
!-----------------------------------------------
! ** queue options with same value for ALL steps
!-----------------------------------------------
! space seperated list:
queue.pbs.options.all : j m l
! shell (DO NOT USE IT! KEPT HERE TO REMEMBER)
queue.pbs.option.all.S : /opt/pbs/default/bin/pbs_python
! join output/error log: oe (joined in out) | eo (joined in err) | n (not joined)
queue.pbs.option.all.j : oe
! when to send emails : a (abort) | b (begin) | e (end) | n (never)
queue.pbs.option.all.m : n
! pass environment (ECMWF doc says to be careful)
queue.pbs.option.all.V : 

! account (to overwrite default one attached to $USER)
#if ("${my.queue.account}" == ""):
queue.pbs.option.all.l : 
#else
queue.pbs.option.all.l : EC_billing_account=${my.queue.account}
#endif

!------------------------------------------------
! ** queue options with different values per step
!
! To set a walltime request (which can lower the waiting in the queue), add walltime=<hh:mm:ss> to the list *run.l:
! 
!------------------------------------------------
! list options per step:
queue.pbs.options.init : N q o l
queue.pbs.options.run  : N q o l
queue.pbs.options.done : N q o

! ~~~~~ step init options

queue.pbs.option.init.q : ns
queue.pbs.option.init.N : ${job.name}_init
queue.pbs.option.init.o : <auto>
queue.pbs.option.init.l : EC_ecfs=1

! ~~~~~ step done options

queue.pbs.option.done.q : ns
queue.pbs.option.done.N : ${job.name}_done
queue.pbs.option.done.o : <auto>
queue.pbs.option.done.W : block=true

! ~~~~~ step run options

queue.pbs.option.run.N : ${job.name}_run
queue.pbs.option.run.o : <auto>

! check if run step's gonna be fractional (nf) or full (nq) node occupancy,
! which depends on: hyperthreading (1 or 2), ntasks and nthreads
#if (${my.queue.ntask}*${my.queue.nthread} == 1 ) :
queue.pbs.option.run.q : ns
module.cmd:
#elif (${my.queue.ntask}*${my.queue.nthread} > ${hyperthread}*18 ) :
queue.pbs.option.run.q : np
module.cmd:
#else
queue.pbs.option.run.q : nf
! NOTE: the module command, even with the qualified path, is not found in the
! python script. It is here just to put mpiexec on the $PATH - so we revert to
! using the fully qualified path for mpiexec hereafter. 
module.cmd: 
!module.cmd: /opt/modules/3.2.6.7/bin/modulecmd bash load cray-snplauncher
#endif

#if (${my.queue.nthread} > 1):
queue.pbs.option.run.l : EC_total_tasks=${my.queue.ntask} EC_threads_per_task=${my.queue.nthread} EC_hyperthreads=${hyperthread}
#else
queue.pbs.option.run.l : EC_total_tasks=${my.queue.ntask} EC_hyperthreads=${hyperthread}
#endif

queue.pbs.option.run.W : block=true

! TODO if needed: Resources (thru -l EC_memory_per_task)
!queue.pbs.option.run.resources           : ${my.resource.memory}

!===============================================
! make
!===============================================
! the setup script will insert the 'build.jobs' specified in the expert.rc
! or passed as argument to the setup script:
maker               :  gmake -j %{build.jobs}

! Submit compilation with a job script? 
! - if F (default), pycasso calls the ${maker} 
! - if T, then compilation is submitted according to "submit.to" key (default
!         of which is set in expert.rc, and can be overwritten at the CLI): either "queue", or foreground.
!
! "my.queue.make" is set in the main rc file (so user can switch it on/off from there).
! 
build.make.submit        : ${my.queue.make}
! sub options: assume that %{build.jobs} is less than 24 (default is set in expert, probably to 8)
queue.pbs.options.build  : N q o l
queue.pbs.option.build.q : nf
queue.pbs.option.build.N : build-tm
queue.pbs.option.build.o : <auto>
queue.pbs.option.build.l : EC_total_tasks=${build.jobs} EC_hyperthreads=1

!==========================================================================
! MPI runner (LUCKILY, THE ENVIRONMENT VARIABLES ARE **NOT** EXPANDED UNTIL
! SCRIPTS ARE SUBMITTED). Depends on:
!   (1) pure openMP, pure MPI or hybrid runs
!   (2) node occupancy: fractional or full (see above)
!==========================================================================
#if "${queue.pbs.option.run.q}" == "np":
mpirun.command      :  aprun

#if (${my.queue.nthread} > 1):
! Hybrid (TODO: does it covers pure openMP?)
mpirun.args         :  -N $EC_tasks_per_node -n $EC_total_tasks -d $OMP_NUM_THREADS -j $EC_hyperthreads
#else
!pure MPI
mpirun.args         :  -N $EC_tasks_per_node -n $EC_total_tasks -j $EC_hyperthreads
#endif

#else
mpirun.command      :  /opt/cray/snplauncher/6.3.1/bin/mpiexec
! pure MPI only. TODO: hybrid, and pure openMP
mpirun.args         :  -n $EC_total_tasks
#endif

! name of command and host files (leave empty to not write):
mpirun.cmdfile      :  
mpirun.hostfile     :  

!===============================================
! debugger (FIXME)
!===============================================
! debugger type: totalview | idb | kdbg
debugger : totalview

! command for debugger:
debugger.command : totalview -searchPath=${build.sourcedir}
!debugger.command : totalview

!===============================================
! model data
!===============================================
! the user scratch directory:
my.scratch          :  ${SCRATCH}

! base path to various data files (static):
my.data.dir         :  /perm/ms/nl/nm6/TM5_INPUT

! permanent archives to search for meteo files:
my.meteo.search     :  ecfs:/nlh/TM/meteo-nc

! extra install tasks:
my.install.tasks        :