Fix restart & IIMPI

3.3.3.2/run/EC00.sh

@@ -3,7 +3,7 @@
 #
 # Job options 
 #
-#SBATCH --job-name=ECE_00
+#SBATCH --job-name=EC00
 #SBATCH --time=12:00:00
 #SBATCH --account=ecearth
 #
@@ -75,7 +75,7 @@ export exp_name=EC00
 
 # Simulation start and end date. Use any (reasonable) syntax you want.
 run_start_date="1850-01-01"
-run_end_date="${run_start_date} + 6 years"
+run_end_date="${run_start_date} + 10 years"
 
 # Set $force_run_from_scratch to 'true' if you want to force this run to start
 # from scratch, possibly ignoring any restart files present in the run

3.3.3.2/run/ecconf.cfg

@@ -28,6 +28,12 @@ function configure()
     resubmit_job=true
     resubmit_opt=""
 
+    module load craype-x86-milan
+    module load PrgEnv-intel/8.3.3
+    MODULEPATH=$MODULEPATH:/gpfs/projects/acad/ecearth/softs/easybuild/modules/all
+    module load netCDF-Fortran/4.6.0-iompi-2022.05
+    module load imkl/2022.1.0 OpenJPEG/2.5.0-GCCcore-11.3.0 grib_api/1.24.0-iompi-2022.05
+
     # Configure grib api paths
     export GRIB_DEFINITION_PATH=${HOME}/models/ecearth_3.3.3.2/sources/util/grib_table_126:${EBROOTGRIB_API}/share/grib_api/definitions
     export GRIB_SAMPLES_PATH=${EBROOTGRIB_API}/share/grib_api/ifs_samples/grib1
@@ -36,12 +42,6 @@ function configure()
     # Configure number of processors per node
     proc_per_node=128
 
-    module load craype-x86-milan
-    module load PrgEnv-intel/8.3.3
-    MODULEPATH=$MODULEPATH:/gpfs/projects/acad/ecearth/softs/easybuild/modules/all
-    module load netCDF-Fortran/4.6.0-iompi-2022.05
-    module load imkl/2022.1.0 OpenJPEG/2.5.0-GCCcore-11.3.0 grib_api/1.24.0-iompi-2022.05
-
     # Use machinefiles or not
     [[ `echo "$use_machinefile" | tr '[:upper:]' '[:lower:]'` == true ]] && use_machinefile=true || use_machinefile=false
 
@@ -294,6 +294,6 @@ function finalise()
                -e ${run_dir}/$(basename ${stdout_file}).$(printf %03d $((leg_number+1))) \
                -d ${SLURM_JOB_ID}                                                        \
                ${resubmit_opt}                                                           \
-               ./${SLURM_JOB_NAME}
+               ./${SLURM_JOB_NAME}.sh
     fi
 }

eb/easyconfigs/HDF5-1.12.2-iimpi-2022.06.eb

@@ -0,0 +1,22 @@
+name = 'HDF5'
+# Note: Odd minor releases are only RCs and should not be used.
+version = '1.12.2'
+
+homepage = 'https://portal.hdfgroup.org/display/support'
+description = """HDF5 is a data model, library, and file format for storing and managing data.
+ It supports an unlimited variety of datatypes, and is designed for flexible
+ and efficient I/O and for high volume and complex data."""
+
+toolchain = {'name': 'iimpi', 'version': '2022.06'}
+toolchainopts = {'pic': True, 'usempi': True}
+
+source_urls = ['https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-%(version_major_minor)s/hdf5-%(version)s/src']
+sources = [SOURCELOWER_TAR_GZ]
+checksums = ['2a89af03d56ce7502dcae18232c241281ad1773561ec00c0f0e8ee2463910f14']
+
+dependencies = [
+    ('zlib', '1.2.12'),
+    ('Szip', '2.1.1'),
+]
+
+moduleclass = 'data'

eb/easyconfigs/grib_api-1.24.0-iimpi-2022.06.eb

@@ -0,0 +1,25 @@
+easyblock = 'ConfigureMake'
+
+name = 'grib_api'
+version = '1.24.0'
+
+homepage = 'https://software.ecmwf.int/wiki/display/GRIB/Home'
+description = """The ECMWF GRIB API is an application program interface accessible from C, FORTRAN and Python
+ programs developed for encoding and decoding WMO FM-92 GRIB edition 1 and edition 2 messages. A useful set of
+ command line tools is also provided to give quick access to GRIB messages."""
+
+toolchain = {'name': 'iimpi', 'version': '2022.06'}
+
+source_urls = ['https://software.ecmwf.int/wiki/download/attachments/3473437/']
+sources = ['grib_api-%(version)s-Source.tar.gz']
+checksums = ['6b0d443cb0802c5de652e5816c5b88734cb3ead454eb932c5ec12ef8d4f77bcd']
+
+dependencies = [
+    ('JasPer', '1.900.1'),  # 2.x doesn't work
+]
+
+configopts = '--with-jasper=$EBROOTJASPER'
+
+parallel = 1
+
+moduleclass = 'data'

eb/easyconfigs/iimpi-2022.06.eb

@@ -0,0 +1,18 @@
+# This is an easyconfig file for EasyBuild, see http://easybuilders.github.io/easybuild
+easyblock = 'Toolchain'
+
+name = 'iimpi'
+version = '2022.06'
+
+homepage = 'https://software.intel.com/parallel-studio-xe'
+description = """Intel C/C++ and Fortran compilers, alongside Intel MPI."""
+
+toolchain = SYSTEM
+
+local_comp_ver = '2022.1.0'
+dependencies = [
+    ('intel-compilers', local_comp_ver),
+    ('impi', '2021.7.0', '', ('intel-compilers', local_comp_ver)),
+]
+
+moduleclass = 'toolchain'

eb/easyconfigs/impi-2021.7.0-intel-compilers-2022.1.0.eb

@@ -0,0 +1,31 @@
+name = 'impi'
+version = '2021.7.0'
+
+homepage = 'https://software.intel.com/content/www/us/en/develop/tools/mpi-library.html'
+description = "Intel MPI Library, compatible with MPICH ABI"
+
+toolchain = {'name': 'intel-compilers', 'version': '2022.1.0'}
+
+# see https://software.intel.com/content/www/us/en/develop/articles/oneapi-standalone-components.html
+source_urls = ['https://registrationcenter-download.intel.com/akdlm/irc_nas/18926/']
+sources = ['l_mpi_oneapi_p_%(version)s.8711_offline.sh']
+checksums = ['4eb1e1487b67b98857bc9b7b37bcac4998e0aa6d1b892b2c87b003bf84fb38e9']
+
+dependencies = [('UCX', '1.12.1')]
+
+# set up all the mpi commands to default to intel compilers
+# set_mpi_wrappers_all = True
+
+modextravars = {
+    # to enable SLURM integration with srun (site-specific)
+    # 'I_MPI_PMI_LIBRARY': 'libpmi2.so',
+
+    # set this if mpirun gives you a floating point exception (SIGFPE), see
+    # https://software.intel.com/en-us/forums/intel-clusters-and-hpc-technology/topic/852307
+    # 'I_MPI_HYDRA_TOPOLIB': 'ipl',
+}
+
+# may be needed if you enable I_MPI_PMI_LIBRARY above
+# osdependencies = [('slurm-libpmi')]
+
+moduleclass = 'mpi'

eb/easyconfigs/netCDF-4.9.0-iimpi-2022.06.eb

@@ -0,0 +1,53 @@
+name = 'netCDF'
+version = '4.9.0'
+
+homepage = 'https://www.unidata.ucar.edu/software/netcdf/'
+description = """NetCDF (network Common Data Form) is a set of software libraries
+ and machine-independent data formats that support the creation, access, and sharing of array-oriented
+ scientific data."""
+
+toolchain = {'name': 'iimpi', 'version': '2022.06'}
+toolchainopts = {'pic': True, 'usempi': True}
+
+source_urls = ['https://github.com/Unidata/netcdf-c/archive/']
+sources = ['v%(version)s.tar.gz']
+patches = [
+    'netCDF-%(version)s_fix-lib-name.patch',
+    'netCDF-%(version)s_fix-linking-errors.patch',
+    'netCDF-%(version)s_skip-nasa-test.patch',
+]
+checksums = [
+    '9f4cb864f3ab54adb75409984c6202323d2fc66c003e5308f3cdf224ed41c0a6',  # v4.9.0.tar.gz
+    'ed45ae6c49cf8dcddaadef4c5cf403049bf3f761187413d7b03754d319345d6a',  # netCDF-4.9.0_fix-lib-name.patch
+    '2b9579c149579d137eb16ae5d790787774033dcec49c0d4993d9859f224e7b15',  # netCDF-4.9.0_fix-linking-errors.patch
+    '19d99e03c048b037dc01f03f5b8ddc910ebaceb076d0f050540d348f26dfcd2a',  # netCDF-4.9.0_skip-nasa-test.patch
+]
+
+builddependencies = [
+    ('Autotools', '20220317'),
+    ('CMake', '3.23.1'),
+    ('Doxygen', '1.9.4'),
+]
+
+dependencies = [
+    ('HDF5', '1.12.2'),
+    ('cURL', '7.83.0'),
+    ('Szip', '2.1.1'),
+    ('zstd', '1.5.2'),
+    ('bzip2', '1.0.8'),
+    ('libxml2', '2.9.13'),
+]
+
+# make sure both static and shared libs are built
+# and disable "remote" tests that access a unreliable external test server over internet
+configopts = [
+    "-DENABLE_DAP_REMOTE_TESTS=OFF -DBUILD_SHARED_LIBS=OFF",
+    "-DENABLE_DAP_REMOTE_TESTS=OFF -DBUILD_SHARED_LIBS=ON",
+]
+
+# some tests try to start 16 MPI ranks, so we need to allow oversubscription to avoid failing tests
+pretestopts = "OMPI_MCA_rmaps_base_oversubscribe=1 "
+
+runtest = 'test'
+
+moduleclass = 'data'

eb/easyconfigs/netCDF-Fortran-4.6.0-iimpi-2022.06.eb

@@ -0,0 +1,28 @@
+name = 'netCDF-Fortran'
+version = '4.6.0'
+
+homepage = 'https://www.unidata.ucar.edu/software/netcdf/'
+description = """NetCDF (network Common Data Form) is a set of software libraries
+ and machine-independent data formats that support the creation, access, and sharing of array-oriented
+ scientific data."""
+
+toolchain = {'name': 'iimpi', 'version': '2022.06'}
+toolchainopts = {'pic': True, 'usempi': True}
+
+source_urls = ['https://github.com/Unidata/netcdf-fortran/archive/']
+sources = ['v%(version)s.tar.gz']
+checksums = ['8194aa70e400c0adfc456127c1d97af2c6489207171d13b10cd754a16da8b0ca']
+
+builddependencies = [
+    ('M4', '1.4.19'),
+]
+
+dependencies = [
+    ('netCDF', '4.9.0'),
+    ('bzip2', '1.0.8'),
+]
+
+# (too) parallel build fails, but single-core build is fairly quick anyway (~1min)
+parallel = 1
+
+moduleclass = 'data'

softs.md

@@ -37,6 +37,8 @@ export EASYBUILD_CONFIGFILES
 
 ## Compile
 
+### IOMPI
+
 ```bash
 sg ecearth -c "eb OpenMPI-4.1.1-intel-compilers-2022.1.0.eb --optarch="march=core-avx2" --hide-deps=M4,Bison,flex,zlib,pkg-config,binutils,OpenSSL,help2man,ncurses,libpng,bzip2,cURL,NASM,pkgconf,ncurses,jbigkit,gettext,XZ,libarchive,libjpeg-turbo,libdeflate,gzip,lz4,zstd,LibTIFF,Szip,libiconv,util-linux,libtool,expat,groff,libxml2,JasPer,Doxygen,googletest,libaec,libevent,libreadline,nlohmann_json,DB,UDUNITS,Tcl,SQLite,Autoconf,Automake,Autotools,numactl,xorg-macros,libpciaccess,PROJ,hwloc,UCC,ecCodes,Perl,JasPer,intltool,UnZip,gperf,Brotli,git,freetype,GMP,libffi,fontconfig,Rust,Meson,X11,ANTLR,libtirpc,GSL,PCRE,libdap,ESMF"
 
@@ -49,6 +51,18 @@ sg ecearth -c "eb netCDF-4.9.0-iompi-2022.05.eb --optarch="march=core-avx2" --hi
 sg ecearth -c "eb netCDF-Fortran-4.6.0-iompi-2022.05.eb --optarch="march=core-avx2" --hide-deps=M4,Bison,flex,zlib,pkg-config,binutils,OpenSSL,help2man,ncurses,libpng,bzip2,cURL,NASM,pkgconf,ncurses,jbigkit,gettext,XZ,libarchive,libjpeg-turbo,libdeflate,gzip,lz4,zstd,LibTIFF,Szip,libiconv,util-linux,libtool,expat,groff,libxml2,JasPer,Doxygen,googletest,libaec,libevent,libreadline,nlohmann_json,DB,UDUNITS,Tcl,SQLite,Autoconf,Automake,Autotools,numactl,xorg-macros,libpciaccess,PROJ,hwloc,UCC,ecCodes,Perl,JasPer,intltool,UnZip,gperf,Brotli,git,freetype,GMP,libffi,fontconfig,Rust,Meson,X11,ANTLR,libtirpc,GSL,PCRE,libdap,ESMF"
 ```
 
+### IIMPI
+
+```bash
+sg ecearth -c "eb impi-2021.7.0-intel-compilers-2022.1.0.eb --optarch="march=core-avx2" --accept-eula-for=Intel-oneAPI --hide-deps=M4,Bison,flex,zlib,pkg-config,binutils,OpenSSL,help2man,ncurses,libpng,bzip2,cURL,NASM,pkgconf,ncurses,jbigkit,gettext,XZ,libarchive,libjpeg-turbo,libdeflate,gzip,lz4,zstd,LibTIFF,Szip,libiconv,util-linux,libtool,expat,groff,libxml2,JasPer,Doxygen,googletest,libaec,libevent,libreadline,nlohmann_json,DB,UDUNITS,Tcl,SQLite,Autoconf,Automake,Autotools,numactl,xorg-macros,libpciaccess,PROJ,hwloc,UCC,ecCodes,Perl,JasPer,intltool,UnZip,gperf,Brotli,git,freetype,GMP,libffi,fontconfig,Rust,Meson,X11,ANTLR,libtirpc,GSL,PCRE,libdap,ESMF"
+
+sg ecearth -c "eb iimpi-2022.06.eb --optarch="march=core-avx2" --hide-deps=M4,Bison,flex,zlib,pkg-config,binutils,OpenSSL,help2man,ncurses,libpng,bzip2,cURL,NASM,pkgconf,ncurses,jbigkit,gettext,XZ,libarchive,libjpeg-turbo,libdeflate,gzip,lz4,zstd,LibTIFF,Szip,libiconv,util-linux,libtool,expat,groff,libxml2,JasPer,Doxygen,googletest,libaec,libevent,libreadline,nlohmann_json,DB,UDUNITS,Tcl,SQLite,Autoconf,Automake,Autotools,numactl,xorg-macros,libpciaccess,PROJ,hwloc,UCC,ecCodes,Perl,JasPer,intltool,UnZip,gperf,Brotli,git,freetype,GMP,libffi,fontconfig,Rust,Meson,X11,ANTLR,libtirpc,GSL,PCRE,libdap,ESMF"
+
+sg ecearth -c "eb HDF5-1.12.2-iimpi-2022.06.eb --optarch="march=core-avx2" --hide-deps=M4,Bison,flex,zlib,pkg-config,binutils,OpenSSL,help2man,ncurses,libpng,bzip2,cURL,NASM,pkgconf,ncurses,jbigkit,gettext,XZ,libarchive,libjpeg-turbo,libdeflate,gzip,lz4,zstd,LibTIFF,Szip,libiconv,util-linux,libtool,expat,groff,libxml2,JasPer,Doxygen,googletest,libaec,libevent,libreadline,nlohmann_json,DB,UDUNITS,Tcl,SQLite,Autoconf,Automake,Autotools,numactl,xorg-macros,libpciaccess,PROJ,hwloc,UCC,ecCodes,Perl,JasPer,intltool,UnZip,gperf,Brotli,git,freetype,GMP,libffi,fontconfig,Rust,Meson,X11,ANTLR,libtirpc,GSL,PCRE,libdap,ESMF"
+
+sg ecearth -c "eb netCDF-4.9.0-iimpi-2022.06.eb --optarch="march=core-avx2" --hide-deps=M4,Bison,flex,zlib,pkg-config,binutils,OpenSSL,help2man,ncurses,libpng,bzip2,cURL,NASM,pkgconf,ncurses,jbigkit,gettext,XZ,libarchive,libjpeg-turbo,libdeflate,gzip,lz4,zstd,LibTIFF,Szip,libiconv,util-linux,libtool,expat,groff,libxml2,JasPer,Doxygen,googletest,libaec,libevent,libreadline,nlohmann_json,DB,UDUNITS,Tcl,SQLite,Autoconf,Automake,Autotools,numactl,xorg-macros,libpciaccess,PROJ,hwloc,UCC,ecCodes,Perl,JasPer,intltool,UnZip,gperf,Brotli,git,freetype,GMP,libffi,fontconfig,Rust,Meson,X11,ANTLR,libtirpc,GSL,PCRE,libdap,ESMF"
+```
+
 ## Contributors
 
 - ELIC members [@pbarriat](https://gogs.elic.ucl.ac.be/pbarriat)