Uprade with rebuild info

README.md

@@ -8,12 +8,15 @@ The goal of this project is to adapt this model for our needs in ELIC: install,
 First, load the required env modules
 
 ```bash
-module load craype-x86-milan
-module load PrgEnv-intel/8.3.3
-module load Perl/.5.34.1-GCCcore-11.3.0
+module purge
+module load EasyBuild/2023a
+module load Perl-bundle-CPAN/5.36.1-GCCcore-12.3.0
 MODULEPATH=$MODULEPATH:/gpfs/projects/acad/ecearth/softs/easybuild/modules/all
-module load netCDF-Fortran/4.6.0-iompi-2022.05
-module load imkl/2022.1.0 OpenJPEG/2.5.0-GCCcore-11.3.0 grib_api/1.24.0-iompi-2022.05
+module load netCDF-Fortran/4.6.1-iompi-2023a
+module load imkl/2023.1.0 
+module load grib_api/1.24.0-iompi-2023a 
+module load CDO/1.9.10-iompi-2023a
+module load OpenJPEG/2.5.0-GCCcore-12.3.0
 ```
 
 Then, clone this EC-Earth fork repository (3.3.3.2)
@@ -79,7 +82,7 @@ I prepared all the input files in `/gpfs/scratch/acad/ecearth/data/bsc32/` direc
 |  CC   | Release | RES |  XIOS  | NEMO | IFS | #NODES |  WTIME per YEAR        |
 | ----- | --- | ---- | ------ | -------------- | ------ | ------- | -------------------- |
 | `EC00.sh` | iompi/2022.05 | T255L91_ORCA1L75 |  1  |   86  |   320    |     3      | **121min**       |
-| `EC01.sh` | iompi/2022.05 | T255L91_ORCA1L75 |  3  |  128  |   288   |    4       | **??min**       |
+| `EC01.sh` | iompi/2022.05 | T255L91_ORCA1L75 |  1  |   98  |   392   |    4       | **??min**       |
 | `EC02.sh` | iompi/2022.05 | T255L91_ORCA1L75 |  1  |  128  |   512   |    6       | **83min**       |
 
 - 1 year (1850), restart every year

REBUILD.md

@@ -0,0 +1,97 @@
+# ReBuild EC-EARTH on LUCIA
+
+EC-Earth is a global coupled climate model. It is developed by a consortium of European research institutions.
+The goal of this project is to adapt this model for our needs in ELIC: install, run management, etc.
+
+## User requirements
+
+First, load the required env modules
+
+```bash
+module purge
+module load EasyBuild/2023a
+module load Perl-bundle-CPAN/5.36.1-GCCcore-12.3.0
+MODULEPATH=$MODULEPATH:/gpfs/projects/acad/ecearth/softs/easybuild/modules/all
+module load netCDF-Fortran/4.6.1-iompi-2023a
+module load imkl/2023.1.0 
+module load grib_api/1.24.0-iompi-2023a 
+module load CDO/1.9.10-iompi-2023a
+module load OpenJPEG/2.5.0-GCCcore-12.3.0
+```
+
+Then, go to your EC-Earth repository (3.3.3.2)
+
+```bash
+cd ~/models/ecearth_3.3.3.2
+```
+
+## ReCompile
+
+* Clean and recompile **oasis**:
+
+```bash
+cd ~/models/ecearth_3.3.3.2/sources/oasis3-mct/util/make_dir
+make BUILD_ARCH=ecconf -f TopMakefileOasis3 clean
+make BUILD_ARCH=ecconf -f TopMakefileOasis3 realclean
+make BUILD_ARCH=ecconf -f TopMakefileOasis3
+```
+
+* Clean and recompile **IFS**:
+
+```bash
+cd ~/models/ecearth_3.3.3.2/sources/ifs-36r4
+make BUILD_ARCH=ecconf clean
+make BUILD_ARCH=ecconf realclean
+./makeifs -j 4
+```
+
+* Recompile **xios** from scratch:
+
+> !!! change to do in `extern/blitz/blitz/funcs.h +575`
+> replace:
+> `return BZ_STD_SCOPE(isnan)(a);`
+> with:
+> `return isnan(a);`
+
+```bash
+cd ~/models/ecearth_3.3.3.2/sources/xios-2.5
+./make_xios --full --prod --arch ecconf --use_oasis oasis3_mct --netcdf_lib netcdf4_par -j 4
+```
+
+* Clean and recompile **NEMO**:
+
+```bash
+cd ~/models/ecearth_3.3.3.2/sources/nemo-3.6/CONFIG
+rm -rf ~/models/ecearth_3.3.3.2/sources/nemo-3.6/CONFIG/ORCA1L75_LIM3/BLD/
+./makenemo -n ORCA1L75_LIM3 -m ecconf -j4
+```
+
+* Clean and recompile **runoff-mapper**:
+
+```bash
+cd ~/models/ecearth_3.3.3.2/sources/runoff-mapper/src
+make clean
+rm -f ~/models/ecearth_3.3.3.2/sources/runoff-mapper/bin/runoff-mapper.exe
+make
+```
+
+## Run
+
+Check the job script `EC00.sh` and compare the changes with your script:
+
+[New EC00.sh](https://gogs.elic.ucl.ac.be/pbarriat/ecearth3/src/master/runtime/classic/EC00.sh)
+
+```bash
+cd
+wget https://gogs.elic.ucl.ac.be/pbarriat/ecearth3/raw/master/runtime/classic/EC00.sh
+cd ~/models/ecearth_3.3.3.2/runtime/classic
+vimdiff my_script.sh ~/EC00.sh
+```
+
+### Data
+
+I prepared all the input files in `/gpfs/scratch/acad/ecearth/data/bsc32/` directory.
+
+## Contributors
+
+- ELIC members [@pbarriat](https://gogs.elic.ucl.ac.be/pbarriat)

runtime/classic/EC00.sh

@@ -4,18 +4,18 @@
 # Job options 
 #
 #SBATCH --job-name=EC00
-#SBATCH --time=12:00:00
+#SBATCH --time=01:59:00
 #SBATCH --account=ecearth
 #
 #SBATCH --nodes=4
 #SBATCH --exclusive
-#SBATCH --ntasks-per-node=102
-#SBATCH --partition=batch
+#SBATCH --ntasks-per-node=123
+#SBATCH --partition=debug
 #
 set -ueo pipefail
 #
 LOCAL_NODES=4
-LOCAL_TASKS=408
+LOCAL_TASKS=492
 #
 stdout_file=${SLURM_SUBMIT_DIR-$PWD}/${SLURM_JOB_NAME-"local"}_${SLURM_JOB_ID-"id"}.log
 exec > ${stdout_file}
@@ -32,7 +32,7 @@ set -ue
 # Cluster variables
 #
 NB_CORES_PER_NODES=128
-MAX_CORES_PER_NODES=100
+MAX_CORES_PER_NODES=123
 LIST_CORES_SOCKET=`seq -s',' 0 $((NB_CORES_PER_NODES-1))`
 #
 # Directories
@@ -91,10 +91,10 @@ nem_grid=ORCA1L75
 
 # Restart frequency. Use any (reasonable) number and time unit you want.
 # For runs without restart, leave this variable empty
-rst_freq="1 year"
+rst_freq="6 months"
 
 # Number of restart legs to be run in one go
-run_num_legs=2
+run_num_legs=1
 
 # Coupling frequencies
 has_config ifs tm5  && cpl_freq_atm_ctm_hrs=6
@@ -121,7 +121,49 @@ ece_info_file="ece.info"
 # -----------------------------------------------------------------------------
 . ${start_dir}/ecconf.cfg
 
-configure
+# This function should configure all settings/modules needed to
+# later prepare the EC-Earth run directory and set variables used
+# in the run script
+
+# SCRATCH is not defined in MN3, define it here
+# and also make sure it is defined when compiling
+export SCRATCH=/gpfs/scratch/acad/ecearth/${USER}
+
+# Configure paths for building/running EC-Earth
+ecearth_src_dir=${HOME}/models/ecearth_3.3.3.2/sources
+run_dir=/gpfs/scratch/acad/ecearth/${USER}/ecearth/run/${exp_name}
+ini_data_dir=/gpfs/scratch/acad/ecearth/data/bsc32/v3.3.3.2/inidata
+archive_dir=/gpfs/scratch/acad/ecearth/${USER}/ecearth/archive/${exp_name}
+
+# File for standard output.
+# NOTE: This will be modified for restart jobs!
+stdout_file=${SLURM_SUBMIT_DIR-$PWD}/${SLURM_JOB_NAME-"local"}_${SLURM_JOB_ID-"id"}.log
+
+# Resubmit this job for automatic restarts? [true/false]
+# Also, add options for the resubmit command here.
+resubmit_job=true
+resubmit_opt=""
+
+module purge
+module load EasyBuild/2023a
+MODULEPATH=$MODULEPATH:/gpfs/projects/acad/ecearth/softs/easybuild/modules/all
+module load netCDF-Fortran/4.6.1-iompi-2023a
+module load imkl/2023.1.0 
+module load grib_api/1.24.0-iompi-2023a
+module load CDO/1.9.10-iompi-2023a
+
+# Configure grib api paths
+export GRIB_DEFINITION_PATH=${HOME}/models/ecearth_3.3.3.2/sources/util/grib_table_126:${EBROOTGRIB_API}/share/grib_api/definitions
+export GRIB_SAMPLES_PATH=${EBROOTGRIB_API}/share/grib_api/ifs_samples/grib1
+export GRIB_BIN_PATH=${EBROOTGRIB_API}/bin
+
+# Configure number of processors per node
+proc_per_node=128
+
+# Use machinefiles or not
+[[ `echo "$use_machinefile" | tr '[:upper:]' '[:lower:]'` == true ]] && use_machinefile=true || use_machinefile=false
+
+ulimit -s unlimited
 
 # -----------------------------------------------------------------------------
 # *** Time step settings
@@ -182,7 +224,7 @@ ifs_ddh_freq=$(( 120 * 3600 / ifs_time_step_sec ))
 export ifs_res_hor=$(echo ${ifs_grid} | sed 's:T\([0-9]\+\)L\([0-9]\+\):\1:')
 ifs_res_ver=$(echo ${ifs_grid} | sed 's:T\([0-9]\+\)L\([0-9]\+\):\2:')
 
-ifs_numproc=320
+ifs_numproc=392
 
 ifs_exe_file=${ecearth_src_dir}/ifs-${ifs_version}/bin/ifsmaster-${build_arch}
 
@@ -193,8 +235,8 @@ ifs_cmip5=TRUE
 # SWITCH FOR RCP AND HISTORICAL RUNS FOR CMIP5 (0=HISTO 1=RCP 3-PD, 2=RCP 4.5, 3=RCP 6.0, 4=RCP 8.5)
 ifs_cmip5_rcp=0
 # 1PCTCO2 and A4XCO2 should operate together with ifs_cmip_fixyear years, ie., the baseline GHGs level will use that for year ifs_cmip_fixyear
-export ifs_cmip_fixyear=0
-export ifs_cmip_fixyear_ch4=0
+export ifs_cmip_fixyear=1850
+export ifs_cmip_fixyear_ch4=1850
 
 # Repeat trap from ifs/suecrad.F90 for early catch
 if ! has_config tm5:ch4fb && (( $ifs_cmip_fixyear != $ifs_cmip_fixyear_ch4 ))
@@ -387,7 +429,7 @@ fi
 
 nem_exe_file=${ecearth_src_dir}/nemo-3.6/CONFIG/${nem_config}/BLD/bin/nemo.exe
 
-nem_numproc=86
+nem_numproc=98
 
 # Thermal conductivity of snow, see comment in ctrl/namelist.lim3.ref.sh
 case "${ifs_grid}" in
@@ -677,6 +719,7 @@ do
     leg_start_sec=$(( $(date -u -d "${leg_start_date}" +%s) - $(date -u -d "${run_start_date}" +%s) ))
     leg_end_sec=$(( $(date -u -d "${leg_end_date}" +%s) - $(date -u -d "${run_start_date}" +%s) ))
     leg_start_date_yyyymmdd=$(date -u -d "${leg_start_date}" +%Y%m%d)
+    leg_start_date_yyyymm=$(date -u -d "${leg_start_date}" +%Y%m)
     leg_start_date_yyyy=$(date -u -d "${leg_start_date}" +%Y)
     leg_end_date_yyyy=$(date -u -d "${leg_end_date}" +%Y)
 
@@ -710,7 +753,7 @@ do
             then
                 rm -fr ${run_dir}/*
             else
-                error "Run directory not empty and \$force_run_from_scratch not set."
+                error "Run directory ${run_dir} not empty and \$force_run_from_scratch not set."
             fi
         fi
 
@@ -1952,6 +1995,23 @@ done # loop over legs
 # -----------------------------------------------------------------------------
 # *** Platform dependent finalising of the run
 # -----------------------------------------------------------------------------
-finalise
+#finalise
+
+if ${resubmit_job} && [ $(date -d "${leg_end_date}" +%s) -lt $(date -d "${run_end_date}" +%s) ]
+then
+    info "Resubmitting job for leg $((leg_number+1))"
+    # Need to go to start_dir to find the run script
+    cd ${start_dir}
+    # Submit command
+    echo "sbatch -N ${SLURM_JOB_NUM_NODES-"1"} -o ${run_dir}/$(basename ${stdout_file}).$(printf %03d $((leg_number+1))) \
+              -e ${run_dir}/$(basename ${stdout_file}).$(printf %03d $((leg_number+1))) -d ${SLURM_JOB_ID-"id"} \
+              ./${SLURM_JOB_NAME-"run"}.sh"
+        # Note: This does not work if you specify a job name with sbatch -J jobname!
+        sbatch -N ${SLURM_JOB_NUM_NODES-"1"}                                             \
+               -o ${run_dir}/$(basename ${stdout_file}).$(printf %03d $((leg_number+1))) \
+               -e ${run_dir}/$(basename ${stdout_file}).$(printf %03d $((leg_number+1))) \
+               -d ${SLURM_JOB_ID-"id"}                                                   \
+               ./${SLURM_JOB_NAME-"run"}.sh
+fi
 
 exit 0

sources/ifs-36r4/Makefile.d/Makefile.config.ecconf

@@ -54,7 +54,7 @@ FFLAGS                  := -r8 -O3 -march=core-avx2 -ip -fp-model precise -exten
 FPPDEFS                 := $(addprefix -D,linux LINUX LITTLE LITTLE_ENDIAN POINTER_64 BLAS)
 
 CC                      := mpicc
-CFLAGS                  := -fp-model precise -march=core-avx2
+CFLAGS                  := -std=gnu90 -fp-model precise -march=core-avx2
 CPPDEFS                 := $(addprefix -D,linux LINUX LITTLE LITTLE_ENDIAN POINTER_64 BLAS)
 
 LD                      := $(F90)