ecearth3/runtime/autosubmit/compilation.sh

#!/usr/bin/env bash
# remote_setup.sh:
# Extracts the sources from the Autosubmit project bundle and compiles EC-Earth

set -xuve

#
# Architecture
#

HPCARCH=%HPCARCH%
CURRENT_ROOTDIR=%CURRENT_ROOTDIR%
PROJNAME=%PROJECT_DESTINATION%
PROJDIR=${CURRENT_ROOTDIR}/${PROJNAME}
TEMPLATE_NAME=%TEMPLATE_NAME%
NEMO_resolution=%NEMO_resolution%
ICE=%ICE%
NEMO_CONFIG=${NEMO_resolution}_${ICE}

MAKE_NUMPROC=%NUMPROC%

BSC_OUTCLASS=%BSC_OUTCLASS%

# define which runtime we are using
use_autosubmit=true 
[[ use_autosubmit && ${PROJNAME} == 'ecearth3' ]] && use_autosubmit_simple=true || use_autosubmit_simple=false
#when using the simplified AS runtime we use the same ECEARTH_SRC_DIR definition
#as in the full auto-ecearth AS runtime, to avoid having to change it in config-build and config-run
#TODO set SRCDIR from ECEARTH_SRC_DIR, requires writing this script as an ec-conf template
if $use_autosubmit_simple ; then
  SRCDIR=${CURRENT_ROOTDIR}/auto-ecearth3/sources/sources
else
  SRCDIR=${CURRENT_ROOTDIR}/${PROJNAME}/sources/sources
fi

# Variables that could be used from config-build.xml should be exported
export exp_name=%EXPID%

# these variables are to be used for development, allowing to retain any local
# changes to the sources and compile the model more quickly
# EXTRACT : extract files from uploaded .tar.gz file and copy runtime files (default:true)
# CLEAN   : do 'make clean' for all model components (default:true)
[ "%MODEL_EXTRACT%" = FALSE ] && MODEL_EXTRACT=false || MODEL_EXTRACT=true
[ "%MODEL_CLEAN%" = FALSE ] && MODEL_CLEAN=false || MODEL_CLEAN=true
export MODEL_CLEAN

#
# get model config from AS variables (for compilation only)
#
#ifs
[[ "$TEMPLATE_NAME" = ecearth3* || "$TEMPLATE_NAME" = ifs3* ]] && ifs="ifs" || ifs=""
[[ "$TEMPLATE_NAME" = ifs3* ]] && amip="amip" ||  amip=""
#nemo
if [[ "$TEMPLATE_NAME" = ecearth3* || "$TEMPLATE_NAME" = nemo3* ]]; then nemo="nemo"; lim3="lim3"; xios="xios:detached"; else nemo=""; lim3=""; xios=""; fi
[[ "$TEMPLATE_NAME" = ecearth3* ]] && rnfmapper="rnfmapper" || rnfmapper=""
[[ "%NEMO_remove_land%" = TRUE && ! -z $nemo ]] && elpin=":elpin" || elpin=""
[ "%PISCES%" = TRUE ] && pisces="pisces" || pisces=""
#others
[[ "%LPJG%" = TRUE ]] && lpjg=%LPJG_CONFIG% || lpjg=""
config="${ifs} ${amip} ${nemo}${elpin} ${pisces} ${lim3} ${rnfmapper} ${xios} oasis ${lpjg}"

# or you can set here the components you want to build
#config="ifs amip nemo rnfmapper xios oasis lpjg" # for all components (esm runtime)
#config="oasis lpjg" # for LPJG-offline

#
# Loading of modules
#

set +xuve
if [ ${HPCARCH} == 'ecmwf-xc40' ]
then
    export architecture=ecmwf-cca-intel-mpi
    # TODO use preloadcmd from ecconf
    module unload PrgEnv-cray
    module unload PrgEnv-intel
    module unload PrgEnv-gnu
    module load PrgEnv-intel
    module load cray-hdf5-parallel/1.8.14
    module load cray-netcdf-hdf5parallel/4.3.3.1
    module unload eccodes
    module load grib_api/1.12.3

elif [ ${HPCARCH} == 'finisterrae2' ]
then
  export SCRATCH=${LUSTRE}
  export architecture=cesga-finisterrae2-intel-intelmpi
  module load intel/2016
  module load impi/5.1
  module load netcdf-fortran/4.4.3
  module load grib_api/1.14.5
  module load hdf5/1.8.16
  module load mkl/11.3
  module load gribex/370

elif [ ${HPCARCH} == 'nord3' ]
then
  export SCRATCH=/gpfs/scratch/`id -gn`/${USER}
  export architecture=bsc-nord3-intel-intelmpi
  module unload intel
  module load intel/2017.1

elif [ ${HPCARCH} == 'marenostrum4' ]
then
  export SCRATCH=/gpfs/scratch/`id -gn`/${USER}
  export architecture=bsc-marenostrum4-intel-intelmpi
  module unload intel
  module unload impi
  module unload mkl
  module load intel/2017.4
  module load impi/2018.4
  module load perl/5.26
  module load mkl/2018.4 # require by python/2.7.14
  module load python/2.7.14

else
    error "Unsupported ec-conf architechture: ${HPCARCH}"
    exit 0
fi

module list
set -xuve

#
# Extract Model Sources and copy runtime, only if MODEL_EXTRACT=TRUE
#

if ${MODEL_EXTRACT} ; then

  #
  # Extract Model Sources
  #
  cd ${CURRENT_ROOTDIR}
  if [ -f ${PROJNAME}.tar.gz ]; then
    tar -xvf ${PROJNAME}.tar.gz
    rm ${PROJNAME}.tar.gz
    # move sources into auto-ecearth3/sources to make it consistent with ECEARTH_SRC_DIR
    if $use_autosubmit_simple
    then
        mkdir -p ${CURRENT_ROOTDIR}/auto-ecearth3
        mv ${PROJNAME} ${CURRENT_ROOTDIR}/auto-ecearth3/sources
        ln -sf ${CURRENT_ROOTDIR}/auto-ecearth3 ${PROJNAME}
    fi
  fi

  #
  # Copy runtime
  # TODO check if we can link instead of copy
  #
  cd ${CURRENT_ROOTDIR}
  cp ${PROJNAME}/sources/runtime/autosubmit/ecconf.cfg .
  cp ${PROJNAME}/sources/runtime/classic/lib*.sh .
  cp -r ${PROJNAME}/sources/runtime/classic/ctrl/ .

  # should we need to override any files in classic/ctrl/ put them in the autosubmit/ctrl folder
  # and uncomment this line
  #cp -r ${PROJNAME}/sources/runtime/autosubmit/ctrl/ .

  if [[ -n "${BSC_OUTCLASS-}" && -d ${PROJNAME}/outclass/${BSC_OUTCLASS}/ ]]; then
      [ -f ${PROJNAME}/outclass/${BSC_OUTCLASS}/outclass_extra_nemo_compilation_keys.fcm ] && outclass_extra_nemo_compilation_keys=$(cat ${PROJNAME}/outclass/${BSC_OUTCLASS}/outclass_extra_nemo_compilation_keys.fcm)
   else
      echo "WARNING: you are using the outclass from ec-earth portal"
   fi

fi # $MODEL_EXTRACT


#
# librunscript defines some helper functions
#
cd ${CURRENT_ROOTDIR}
source ./librunscript.sh

cd ${SRCDIR}
if [ ${HPCARCH} == 'ecmwf-xc40' ]
then
  util/ec-conf/ec-conf --platform ${architecture} -o PLT:ACTIVE:ECEARTH_SRC_DIR='${SCRATCH}/${exp_name}/auto-ecearth3/sources/sources' config-build.xml
else
  util/ec-conf/ec-conf --platform ${architecture} config-build.xml
fi


if $(has_config nemo) ; then

    #
    # Select correct nemo_config
    #
    has_config pisces && NEMO_CONFIG=${NEMO_CONFIG}_PISCES
    ! has_config ifs && NEMO_CONFIG=${NEMO_CONFIG}_standalone


    if [[ ${NEMO_resolution-} !=  "%%NEMO_resolution%%" && ! -z ${NEMO_resolution-} ]] ; then
        #
        # NEMO compilation key
        #

        new_nemo_compilation_keys=" key_init_alloc_zero ${outclass_extra_nemo_compilation_keys-}"

        fcm_file=$(ls ${SRCDIR}/nemo-3.6/CONFIG/${NEMO_CONFIG}/*.fcm)

        for key in ${new_nemo_compilation_keys}; do
            if [[ $(grep ${key} ${fcm_file}) == "" ]] ; then
                echo adding ${key}
                sed -i -e 1's/$/'" "${key}' &/' ${fcm_file}
             fi
        done
    fi
fi


#
# Check bin and lib directory (git-svn issue with empty folders)
#
if [ ! -d ifs-36r4/bin ]
then
  mkdir ifs-36r4/bin
  mkdir ifs-36r4/lib
fi
if [ ! -d runoff-mapper/bin ]
then
  mkdir runoff-mapper/bin
  mkdir runoff-mapper/lib
fi
if [ ! -d amip-forcing/bin ]
then
  mkdir amip-forcing/bin
  mkdir amip-forcing/lib
fi
if [ ! -d lpjg/build ]
then
  mkdir lpjg/build
fi

# minimum sanity
#has_config amip nemo && error "Cannot have both nemo and amip in config!!"

#
# Complilation of Model Sources
#




# 1) OASIS
if $(has_config oasis)
then
  cd ${SRCDIR}/oasis3-mct/util/make_dir
  if ${MODEL_CLEAN} ; then make realclean -f TopMakefileOasis3 BUILD_ARCH=ecconf ; fi
  make -f TopMakefileOasis3 BUILD_ARCH=ecconf
  # build lucia with the ifort compiler - modify this if you use another compiler
  cd ${SRCDIR}/oasis3-mct/util/lucia
  F90=ifort ./lucia -c
fi

# 2) XIOS
if $(has_config xios)
then
  cd ${SRCDIR}/xios-2.5
  if ${MODEL_CLEAN} ; then
    ./make_xios --arch ecconf --use_oasis oasis3_mct --netcdf_lib netcdf4_par --job ${MAKE_NUMPROC} --full
  else
    ./make_xios --arch ecconf --use_oasis oasis3_mct --netcdf_lib netcdf4_par --job ${MAKE_NUMPROC}
  fi
  [ -f ${SRCDIR}/xios-2.5/bin/xios_server.exe ] || exit 1
fi

# 3) Runoff-Mapper
if $(has_config rnfmapper)
then
  cd ${SRCDIR}/runoff-mapper/src
  if ${MODEL_CLEAN} ; then make clean ; fi
  make
  [ -f ${SRCDIR}/runoff-mapper/bin/runoff-mapper.exe ] || exit 1
fi

# 4) NEMO
if $(has_config nemo)
then
  cd ${SRCDIR}/nemo-3.6/CONFIG
  # remove old nemo executable to make sure compilation did not fail
  # workaround for the issue that make_nemo does not return an exit code on failure
  rm -f ${SRCDIR}/nemo*/CONFIG/${NEMO_CONFIG}/BLD/bin/*.exe
  if ${MODEL_CLEAN} ; then ./makenemo -m ecconf -n ${NEMO_CONFIG} -j ${MAKE_NUMPROC} clean ; fi
  ./makenemo -m ecconf -n ${NEMO_CONFIG} -j ${MAKE_NUMPROC}
  [ -f ${SRCDIR}/nemo*/CONFIG/${NEMO_CONFIG}/BLD/bin/*.exe ] || exit 1
fi

# 5) IFS
if $(has_config ifs)
then
  set +xuve
  . ${SRCDIR}/util/grib_table_126/define_table_126.sh
  set -xuve
  cd ${SRCDIR}/ifs-36r4
  if ${MODEL_CLEAN} ; then
    make clean BUILD_ARCH=ecconf
    make realclean BUILD_ARCH=ecconf
    make dep-clean BUILD_ARCH=ecconf
  fi
  make -j ${MAKE_NUMPROC} BUILD_ARCH=ecconf lib
  make BUILD_ARCH=ecconf master
  [ -f ${SRCDIR}/ifs*/bin/ifsmaster* ] || exit 1
fi

# 6) Amip
if $(has_config amip)
then
  cd ${SRCDIR}/amip-forcing/src
  if ${MODEL_CLEAN} ; then make clean ; fi
  make
  [ -f ${SRCDIR}/amip*/bin/amip* ] || exit 1
fi

# 7) LPJ-Guess
if $(has_config lpjg)
then
  cd ${SRCDIR}/lpjg/build
  set +xuve
  module load cmake
  [ ${HPCARCH} == 'marenostrum4' ] && module load grib/1.14.0
  set -xuve
  cmake ..
  if ${MODEL_CLEAN} ; then make clean ; fi
  make
  cd ${SRCDIR}/lpjg/offline
  make
  [ -f ${SRCDIR}/lpjg/build/guess_T255 ] || exit 1
fi

# 8) TM5
# currently does not work, make does not expand the $exp_name in ECEARTH_SRC_DIR definition in ecconfig-ecearth3.rc
if $(has_config tm5)
then
  cd ${SRCDIR}/tm5mp/
  #export PATH=${PATH}:${SRCDIR}/util/makedepf90/bin/:$PATH
  ./setup_tm5 -n -v -j ${MAKE_NUMPROC} ecconfig-ecearth3.rc
  [ -f ${PROJDIR}/sources/sources/tm5mp*/build/appl-tm5.x* ] || exit 1
fi

# 9) ELPiN 
if $(has_config elpin)
then
 cd ${SRCDIR}/util/ELPiN/
 make clean
 make
fi

echo "Finished compiling"

set +xuve

#
# workaround for intelremotemond process started when compiling with
# intel compilers (maybe only on MN3 ?)
#
set +e
if [ ${HPCARCH} == 'nord3' ] ; then (! pidof intelremotemond) || killall -u ${USER} intelremotemond ; fi