123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357 |
- #!/usr/bin/env bash
- # remote_setup.sh:
- # Extracts the sources from the Autosubmit project bundle and compiles EC-Earth
- set -xuve
- #
- # Architecture
- #
- HPCARCH=%HPCARCH%
- CURRENT_ROOTDIR=%CURRENT_ROOTDIR%
- PROJNAME=%PROJECT_DESTINATION%
- PROJDIR=${CURRENT_ROOTDIR}/${PROJNAME}
- TEMPLATE_NAME=%TEMPLATE_NAME%
- NEMO_resolution=%NEMO_resolution%
- ICE=%ICE%
- NEMO_CONFIG=${NEMO_resolution}_${ICE}
- MAKE_NUMPROC=%NUMPROC%
- BSC_OUTCLASS=%BSC_OUTCLASS%
- # define which runtime we are using
- use_autosubmit=true
- [[ use_autosubmit && ${PROJNAME} == 'ecearth3' ]] && use_autosubmit_simple=true || use_autosubmit_simple=false
- #when using the simplified AS runtime we use the same ECEARTH_SRC_DIR definition
- #as in the full auto-ecearth AS runtime, to avoid having to change it in config-build and config-run
- #TODO set SRCDIR from ECEARTH_SRC_DIR, requires writing this script as an ec-conf template
- if $use_autosubmit_simple ; then
- SRCDIR=${CURRENT_ROOTDIR}/auto-ecearth3/sources/sources
- else
- SRCDIR=${CURRENT_ROOTDIR}/${PROJNAME}/sources/sources
- fi
- # Variables that could be used from config-build.xml should be exported
- export exp_name=%EXPID%
- # these variables are to be used for development, allowing to retain any local
- # changes to the sources and compile the model more quickly
- # EXTRACT : extract files from uploaded .tar.gz file and copy runtime files (default:true)
- # CLEAN : do 'make clean' for all model components (default:true)
- [ "%MODEL_EXTRACT%" = FALSE ] && MODEL_EXTRACT=false || MODEL_EXTRACT=true
- [ "%MODEL_CLEAN%" = FALSE ] && MODEL_CLEAN=false || MODEL_CLEAN=true
- export MODEL_CLEAN
- #
- # get model config from AS variables (for compilation only)
- #
- #ifs
- [[ "$TEMPLATE_NAME" = ecearth3* || "$TEMPLATE_NAME" = ifs3* ]] && ifs="ifs" || ifs=""
- [[ "$TEMPLATE_NAME" = ifs3* ]] && amip="amip" || amip=""
- #nemo
- if [[ "$TEMPLATE_NAME" = ecearth3* || "$TEMPLATE_NAME" = nemo3* ]]; then nemo="nemo"; lim3="lim3"; xios="xios:detached"; else nemo=""; lim3=""; xios=""; fi
- [[ "$TEMPLATE_NAME" = ecearth3* ]] && rnfmapper="rnfmapper" || rnfmapper=""
- [[ "%NEMO_remove_land%" = TRUE && ! -z $nemo ]] && elpin=":elpin" || elpin=""
- [ "%PISCES%" = TRUE ] && pisces="pisces" || pisces=""
- #others
- [[ "%LPJG%" = TRUE ]] && lpjg=%LPJG_CONFIG% || lpjg=""
- config="${ifs} ${amip} ${nemo}${elpin} ${pisces} ${lim3} ${rnfmapper} ${xios} oasis ${lpjg}"
- # or you can set here the components you want to build
- #config="ifs amip nemo rnfmapper xios oasis lpjg" # for all components (esm runtime)
- #config="oasis lpjg" # for LPJG-offline
- #
- # Loading of modules
- #
- set +xuve
- if [ ${HPCARCH} == 'ecmwf-xc40' ]
- then
- export architecture=ecmwf-cca-intel-mpi
- # TODO use preloadcmd from ecconf
- module unload PrgEnv-cray
- module unload PrgEnv-intel
- module unload PrgEnv-gnu
- module load PrgEnv-intel
- module load cray-hdf5-parallel/1.8.14
- module load cray-netcdf-hdf5parallel/4.3.3.1
- module unload eccodes
- module load grib_api/1.12.3
- elif [ ${HPCARCH} == 'finisterrae2' ]
- then
- export SCRATCH=${LUSTRE}
- export architecture=cesga-finisterrae2-intel-intelmpi
- module load intel/2016
- module load impi/5.1
- module load netcdf-fortran/4.4.3
- module load grib_api/1.14.5
- module load hdf5/1.8.16
- module load mkl/11.3
- module load gribex/370
- elif [ ${HPCARCH} == 'nord3' ]
- then
- export SCRATCH=/gpfs/scratch/`id -gn`/${USER}
- export architecture=bsc-nord3-intel-intelmpi
- module unload intel
- module load intel/2017.1
- elif [ ${HPCARCH} == 'marenostrum4' ]
- then
- export SCRATCH=/gpfs/scratch/`id -gn`/${USER}
- export architecture=bsc-marenostrum4-intel-intelmpi
- module unload intel
- module unload impi
- module unload mkl
- module load intel/2017.4
- module load impi/2018.4
- module load perl/5.26
- module load mkl/2018.4 # require by python/2.7.14
- module load python/2.7.14
- else
- error "Unsupported ec-conf architechture: ${HPCARCH}"
- exit 0
- fi
- module list
- set -xuve
- #
- # Extract Model Sources and copy runtime, only if MODEL_EXTRACT=TRUE
- #
- if ${MODEL_EXTRACT} ; then
- #
- # Extract Model Sources
- #
- cd ${CURRENT_ROOTDIR}
- if [ -f ${PROJNAME}.tar.gz ]; then
- tar -xvf ${PROJNAME}.tar.gz
- rm ${PROJNAME}.tar.gz
- # move sources into auto-ecearth3/sources to make it consistent with ECEARTH_SRC_DIR
- if $use_autosubmit_simple
- then
- mkdir -p ${CURRENT_ROOTDIR}/auto-ecearth3
- mv ${PROJNAME} ${CURRENT_ROOTDIR}/auto-ecearth3/sources
- ln -sf ${CURRENT_ROOTDIR}/auto-ecearth3 ${PROJNAME}
- fi
- fi
- #
- # Copy runtime
- # TODO check if we can link instead of copy
- #
- cd ${CURRENT_ROOTDIR}
- cp ${PROJNAME}/sources/runtime/autosubmit/ecconf.cfg .
- cp ${PROJNAME}/sources/runtime/classic/lib*.sh .
- cp -r ${PROJNAME}/sources/runtime/classic/ctrl/ .
- # should we need to override any files in classic/ctrl/ put them in the autosubmit/ctrl folder
- # and uncomment this line
- #cp -r ${PROJNAME}/sources/runtime/autosubmit/ctrl/ .
- if [[ -n "${BSC_OUTCLASS-}" && -d ${PROJNAME}/outclass/${BSC_OUTCLASS}/ ]]; then
- [ -f ${PROJNAME}/outclass/${BSC_OUTCLASS}/outclass_extra_nemo_compilation_keys.fcm ] && outclass_extra_nemo_compilation_keys=$(cat ${PROJNAME}/outclass/${BSC_OUTCLASS}/outclass_extra_nemo_compilation_keys.fcm)
- else
- echo "WARNING: you are using the outclass from ec-earth portal"
- fi
- fi # $MODEL_EXTRACT
- #
- # librunscript defines some helper functions
- #
- cd ${CURRENT_ROOTDIR}
- source ./librunscript.sh
- cd ${SRCDIR}
- if [ ${HPCARCH} == 'ecmwf-xc40' ]
- then
- util/ec-conf/ec-conf --platform ${architecture} -o PLT:ACTIVE:ECEARTH_SRC_DIR='${SCRATCH}/${exp_name}/auto-ecearth3/sources/sources' config-build.xml
- else
- util/ec-conf/ec-conf --platform ${architecture} config-build.xml
- fi
- if $(has_config nemo) ; then
- #
- # Select correct nemo_config
- #
- has_config pisces && NEMO_CONFIG=${NEMO_CONFIG}_PISCES
- ! has_config ifs && NEMO_CONFIG=${NEMO_CONFIG}_standalone
- if [[ ${NEMO_resolution-} != "%%NEMO_resolution%%" && ! -z ${NEMO_resolution-} ]] ; then
- #
- # NEMO compilation key
- #
- new_nemo_compilation_keys=" key_init_alloc_zero ${outclass_extra_nemo_compilation_keys-}"
- fcm_file=$(ls ${SRCDIR}/nemo-3.6/CONFIG/${NEMO_CONFIG}/*.fcm)
- for key in ${new_nemo_compilation_keys}; do
- if [[ $(grep ${key} ${fcm_file}) == "" ]] ; then
- echo adding ${key}
- sed -i -e 1's/$/'" "${key}' &/' ${fcm_file}
- fi
- done
- fi
- fi
- #
- # Check bin and lib directory (git-svn issue with empty folders)
- #
- if [ ! -d ifs-36r4/bin ]
- then
- mkdir ifs-36r4/bin
- mkdir ifs-36r4/lib
- fi
- if [ ! -d runoff-mapper/bin ]
- then
- mkdir runoff-mapper/bin
- mkdir runoff-mapper/lib
- fi
- if [ ! -d amip-forcing/bin ]
- then
- mkdir amip-forcing/bin
- mkdir amip-forcing/lib
- fi
- if [ ! -d lpjg/build ]
- then
- mkdir lpjg/build
- fi
- # minimum sanity
- #has_config amip nemo && error "Cannot have both nemo and amip in config!!"
- #
- # Complilation of Model Sources
- #
- # 1) OASIS
- if $(has_config oasis)
- then
- cd ${SRCDIR}/oasis3-mct/util/make_dir
- if ${MODEL_CLEAN} ; then make realclean -f TopMakefileOasis3 BUILD_ARCH=ecconf ; fi
- make -f TopMakefileOasis3 BUILD_ARCH=ecconf
- # build lucia with the ifort compiler - modify this if you use another compiler
- cd ${SRCDIR}/oasis3-mct/util/lucia
- F90=ifort ./lucia -c
- fi
- # 2) XIOS
- if $(has_config xios)
- then
- cd ${SRCDIR}/xios-2.5
- if ${MODEL_CLEAN} ; then
- ./make_xios --arch ecconf --use_oasis oasis3_mct --netcdf_lib netcdf4_par --job ${MAKE_NUMPROC} --full
- else
- ./make_xios --arch ecconf --use_oasis oasis3_mct --netcdf_lib netcdf4_par --job ${MAKE_NUMPROC}
- fi
- [ -f ${SRCDIR}/xios-2.5/bin/xios_server.exe ] || exit 1
- fi
- # 3) Runoff-Mapper
- if $(has_config rnfmapper)
- then
- cd ${SRCDIR}/runoff-mapper/src
- if ${MODEL_CLEAN} ; then make clean ; fi
- make
- [ -f ${SRCDIR}/runoff-mapper/bin/runoff-mapper.exe ] || exit 1
- fi
- # 4) NEMO
- if $(has_config nemo)
- then
- cd ${SRCDIR}/nemo-3.6/CONFIG
- # remove old nemo executable to make sure compilation did not fail
- # workaround for the issue that make_nemo does not return an exit code on failure
- rm -f ${SRCDIR}/nemo*/CONFIG/${NEMO_CONFIG}/BLD/bin/*.exe
- if ${MODEL_CLEAN} ; then ./makenemo -m ecconf -n ${NEMO_CONFIG} -j ${MAKE_NUMPROC} clean ; fi
- ./makenemo -m ecconf -n ${NEMO_CONFIG} -j ${MAKE_NUMPROC}
- [ -f ${SRCDIR}/nemo*/CONFIG/${NEMO_CONFIG}/BLD/bin/*.exe ] || exit 1
- fi
- # 5) IFS
- if $(has_config ifs)
- then
- set +xuve
- . ${SRCDIR}/util/grib_table_126/define_table_126.sh
- set -xuve
- cd ${SRCDIR}/ifs-36r4
- if ${MODEL_CLEAN} ; then
- make clean BUILD_ARCH=ecconf
- make realclean BUILD_ARCH=ecconf
- make dep-clean BUILD_ARCH=ecconf
- fi
- make -j ${MAKE_NUMPROC} BUILD_ARCH=ecconf lib
- make BUILD_ARCH=ecconf master
- [ -f ${SRCDIR}/ifs*/bin/ifsmaster* ] || exit 1
- fi
- # 6) Amip
- if $(has_config amip)
- then
- cd ${SRCDIR}/amip-forcing/src
- if ${MODEL_CLEAN} ; then make clean ; fi
- make
- [ -f ${SRCDIR}/amip*/bin/amip* ] || exit 1
- fi
- # 7) LPJ-Guess
- if $(has_config lpjg)
- then
- cd ${SRCDIR}/lpjg/build
- set +xuve
- module load cmake
- [ ${HPCARCH} == 'marenostrum4' ] && module load grib/1.14.0
- set -xuve
- cmake ..
- if ${MODEL_CLEAN} ; then make clean ; fi
- make
- cd ${SRCDIR}/lpjg/offline
- make
- [ -f ${SRCDIR}/lpjg/build/guess_T255 ] || exit 1
- fi
- # 8) TM5
- # currently does not work, make does not expand the $exp_name in ECEARTH_SRC_DIR definition in ecconfig-ecearth3.rc
- if $(has_config tm5)
- then
- cd ${SRCDIR}/tm5mp/
- #export PATH=${PATH}:${SRCDIR}/util/makedepf90/bin/:$PATH
- ./setup_tm5 -n -v -j ${MAKE_NUMPROC} ecconfig-ecearth3.rc
- [ -f ${PROJDIR}/sources/sources/tm5mp*/build/appl-tm5.x* ] || exit 1
- fi
- # 9) ELPiN
- if $(has_config elpin)
- then
- cd ${SRCDIR}/util/ELPiN/
- make clean
- make
- fi
- echo "Finished compiling"
- set +xuve
- #
- # workaround for intelremotemond process started when compiling with
- # intel compilers (maybe only on MN3 ?)
- #
- set +e
- if [ ${HPCARCH} == 'nord3' ] ; then (! pidof intelremotemond) || killall -u ${USER} intelremotemond ; fi
|