! ! compiler information ! ! template settings for IBM xlf compiler: #include base/${my.branch}/rc/pycasso-compiler-xlf-12.1.rc ! Suppress warnings on Cineca/SP6: my.default.fflags : -qsuppress=1501-245 ! ! libraries ! ! Z library (used for compression in HDF4) compiler.lib.z.fflags : compiler.lib.z.libs : -L${ZLIB_LIB} -lz ! JPEG library (used for compression in HDF4) compiler.lib.jpeg.fflags : compiler.lib.jpeg.libs : -L${LIBJPEG_LIB} -ljpeg ! SZ library (used for compression in HDF4) compiler.lib.sz.fflags : compiler.lib.sz.libs : -L${SZLIB_HOME}/lib -lsz ! HDF4 library: compiler.lib.hdf4.fflags : -I${HDF4_INC} compiler.lib.hdf4.libs : -L${HDF4_LIB} -lmfhdf -ldf ! NetCDF libraries: NETCDF_VERSION : 4.0.1 ! NETCDF_HOME : /cineca/prod/libraries/netcdf/${NETCDF_VERSION}/xl--10.1 compiler.lib.netcdf.fflags : -I${NETCDF_HOME}/include compiler.lib.netcdf.libs : -L${NETCDF_HOME}/lib -lnetcdf ! NETCDF4_HOME : /cineca/prod/libraries/netcdf/${NETCDF_VERSION}_ser/xl--10.1 compiler.lib.netcdf4.fflags : -I${NETCDF4_HOME}/include compiler.lib.netcdf4.libs : -L${NETCDF4_HOME}/lib -lnetcdf ! NETCDF4_PAR_HOME : /cineca/prod/libraries/netcdf/${NETCDF_VERSION}_gpfs/xl--10.1 compiler.lib.netcdf4_par.fflags : -I${NETCDF4_PAR_HOME}/include compiler.lib.netcdf4_par.libs : -L${NETCDF4_PAR_HOME}/lib -lnetcdf ! HDF5 libraries: HDF5_VERSION : 1.8.4 ! HDF5_HOME : /cineca/prod/libraries/hdf5/${HDF5_VERSION}_ser/xl--10.1 compiler.lib.hdf5.fflags : compiler.lib.hdf5.libs : -L${HDF5_HOME}/lib -lhdf5_hl -lhdf5 ! HDF5_PAR_HOME : /cineca/prod/libraries/hdf5/${HDF5_VERSION}_par/xl--10.1 compiler.lib.hdf5_par.fflags : compiler.lib.hdf5_par.libs : -L${HDF5_PAR_HOME}/lib -lhdf5_hl -lhdf5 ! MPI library: automatically included using 'mp'xlf wrapper compiler.lib.mpi.fflags : compiler.lib.mpi.libs : ! MPI library: automatically included using xlf'_r' wrapper compiler.lib.openmp.fflags : compiler.lib.openmp.libs : !! GRIB library: !compiler.lib.grib.fflags : !compiler.lib.grib.libs : ${EMOS_LIB} !! Lapack library: !compiler.lib.lapack.fflags : !compiler.lib.lapack.libs : ${LAPACK_LIB} ! TotalView memory debugging: (DOES NOT WORK ON CINECA YET) !TV_VERSION : 8.7.0-4 !TV_ARCH : rs6000 !TV_HOME : /cineca/prod/tools/totalview/${TV_VERSION}/binary/toolworks/totalview.${TV_VERSION}/${TV_ARCH} !TV_HOME_MR : ${CCU_DATA}/opt/totalview.${TV_VERSION}/${TV_ARCH} !compiler.lib.tv.fflags : !compiler.lib.tv.libs : -L${TV_HOME_MR}/lib -L${TV_HOME}/lib ${TV_HOME}/lib/aix_malloctype64_5.o ! ! settings for LoadLeveler queue ! ! ! loadleveler classes ! ! o llclass ! o https://hpc.cineca.it/docs/startup-guides/SpClasses ! ! +-----------+---------+------------+-----+---------+---------------------+ ! | | max | max | max | default | possible | ! |queue name | Tasks | wall time | mem (Gby) | wait time | ! +-----------+---------+------------+-----+---------+---------------------+ ! |debug 32 10 min 7 0.5 minutes | ! | | ! |serial 4 6 days 96 2 days | ! |small 64 6 hours 56 0.5 hours | ! |parallel 9728 6 hours 112 1 hours | ! |longpar 1024 1 day 56 1 days | ! | | ! |monster 9728 6 days 112 1 only special | ! | | ! |longdebug | ! |archive 1 4 hours - - for "cart" only | ! |private only internal use | ! +-----------+---------+------------+-----+---------+---------------------+ ! ! If not defined, the class is set automically given the requested ! number of cpu's and the wall_clock_limit . ! However, for TM5 we explicitly need to the define class 'archive' ! for the 'init' and 'done' steps of the jobs, therefore we need ! to define a 'class' for the 'run' step explicitly too since it ! remains 'archive' otherwise. Now set to 'serial', but the submit ! command will probalby re-set it to something else if the number ! of cpu's or the wall_clock_limit do not fit for this class. ! ! Architecture: ! 168 nodes ; per node: ! - 128 Gb memory (4 Gb/core ?) ! - 32 cores ; ! - ST (Single Thread) mode : 1 'cpu'/core ! - SMT (Symmetric Multi-Threading) mode : 2 cpu/core ! ! In theory 2 Gb memory per cpu, but errors about limit of 650 Mb . ! ! MPI/OpenMP following the "User Guide 2010" : ! o serial job: ! # @ job_type = serial ! o MPI job: 32 tasks, each task on 1 core ! # @ job_type = parallel ! # @ total_tasks = 32 ! # @ task_affinity = core(1) ! o OpenMP job: 1 task on 32 cpu's, 32 threads ! # @ job_type = serial ! # @ task_affinity = cpu(32) ! # @ parallel_threads = 32 ! o MPI/OpenMP job: 8 MPI tasks, each task running on ! 4 cores in ST mode providing 4 OpenMP threads/task : ! # @ job_type = parallel ! # @ total_tasks = 8 ! # @ task_affinity = core(4) ! # @ parallel_threads = 4 ! o MPI/OpenMP job: 8 MPI tasks, each task running on ! 4 cores in SMT mode (8 cpu's) providing 8 OpenMP threads/task : ! # @ job_type = parallel ! # @ total_tasks = 8 ! # @ task_affinity = cpu(8) ! # @ parallel_threads = 8 ! ! queue system (bsub,loadleveler) queue : loadleveler ! passed directly: queue.ll.submit.options : ! number of processors per node on this machine: queue.ll.npe_per_node : 32 ! ** queue options with same value for all steps: ! space seperated list: queue.ll.options.default : job_name notification initialdir environment ! job name: queue.ll.option.default.job_name : ${job.name} ! when to send emails : always | error | start | never | complete* !queue.ll.option.default.notification : never queue.ll.option.default.notification : error ! run directory: queue.ll.option.default.initialdir : ${rundir} ! current path: queue.ll.option.default.environment : PATH = ${PATH} ; DISPLAY = ${DISPLAY} ! ** queue options with different values per step: ! same for all: my.ll.options.steps : output error wall_clock_limit class job_type task_affinity parallel_threads ! list options per step: queue.ll.options.init : step_name ${my.ll.options.steps} queue queue.ll.options.run : step_name dependency ${my.ll.options.steps} total_tasks resources queue queue.ll.options.done : step_name dependency ${my.ll.options.steps} queue ! ~ step init options queue.ll.option.init.step_name : init queue.ll.option.init.class : archive queue.ll.option.init.output : queue.ll.option.init.error : queue.ll.option.init.job_type : serial queue.ll.option.init.task_affinity : cpu(1) ! serial runs always in SMT mode queue.ll.option.init.parallel_threads : 1 queue.ll.option.init.wall_clock_limit : 0:30:00 queue.ll.option.init.queue : ! ~ step run options queue.ll.option.run.step_name : run #if 'init' in "${job.steps}".split() : queue.ll.option.run.dependency : (init == 0) #else queue.ll.option.run.dependency : #endif queue.ll.option.run.output : queue.ll.option.run.error : ! ! parallel (MPI) run ? #if "${par.mpi}" in ["T","True"] : queue.ll.option.run.class : serial ! up to 4 tasks is still called 'serial' ... queue.ll.option.run.job_type : parallel queue.ll.option.run.total_tasks : ${par.ntask} #else queue.ll.option.run.class : serial queue.ll.option.run.job_type : serial queue.ll.option.run.total_tasks : #endif ! ! SMT or ST mode ? queue.ll.option.run.task_affinity : cpu(${par.nthread}) ! SMT mode !queue.ll.option.run.task_affinity : core(${par.nthread}) ! ST mode ! ! number of OpenMP threads: queue.ll.option.run.parallel_threads : ${par.nthread} ! queue.ll.option.run.wall_clock_limit : 6:00:00 queue.ll.option.run.resources : ConsumableMemory(1Gb) queue.ll.option.run.queue : ! ~ step done options queue.ll.option.done.step_name : done #if 'run' in "${job.steps}".split() : queue.ll.option.done.dependency : (run == 0) #else queue.ll.option.done.dependency : #endif queue.ll.option.done.output : queue.ll.option.done.error : queue.ll.option.done.class : archive queue.ll.option.done.job_type : serial queue.ll.option.done.task_affinity : cpu(1) ! serial runs always in SMT mode queue.ll.option.done.parallel_threads : 1 queue.ll.option.done.wall_clock_limit : 0:30:00 queue.ll.option.done.queue : ! ! maker ! ! make command; ! the setup script will insert the 'build.jobs' specified in the expert.rc ! or passed as argument to the setup script: ! maker : gmake -j %{build.jobs} ! ! MPI runner ! ! Parallel Operating Environment (POE) ! Common arguments: ! -procs procs # number of processors ! -cmdfile ! -hostfile ! -labelio {yes | no} # label standard output lines with pe id ! mpirun.command : poe mpirun.args : -labelio yes -procs ${par.ntask} -cmdfile ${mpirun.cmdfile} ! name of command and host files (empty for not written): ! (hostfile is not necessary on cineca) mpirun.cmdfile : ${my.basename}.cmdfile mpirun.hostfile : ! ! debugger ! ! debugger type: totalview | idb | kdbg debugger : totalview ! command for debugger: debugger.command : totalview -searchPath=${build.sourcedir} ! ! model data ! ! per-user scratch directory: my.scratch : ${CINECA_SCRATCH} ! base path to various data files: !my.data.dir : ${TM5_DATA}/TM ! when running in queue, only scratch dir is available: my.data.dir : ${CINECA_SCRATCH}/TM ! local temporary meteo archive, shared by all users: my.meteo.dir : ${my.scratch}/tmm-buf/${my.meteo.class} ! permanent archives to search for meteo files: !my.meteo.search : ${TM5_DATA}/TM/meteo my.meteo.search : cart:TM-meteo ! extra install tasks: my.install.tasks : TM_input ! specify how to install TM_input on runtime scratch: TM_input.install.dir : ${my.data.dir} TM_input.install.arch : ${TM5_DATA}/TM TM_input.install.rsync : input