1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096 |
- #!/bin/bash
- #
- # Job options
- #
- #SBATCH --job-name=EC00
- #SBATCH --time=01:59:00
- #SBATCH --account=ecearth
- #
- #SBATCH --nodes=4
- #SBATCH --exclusive
- #SBATCH --ntasks-per-node=125
- ##SBATCH --partition=medium
- ##SBATCH --exclude=cnm016
- #SBATCH --partition=debug
- ##SBATCH --switches=1@47:50:00
- #
- set -ueo pipefail
- #
- LOCAL_NODES=4
- LOCAL_TASKS=500
- #
- stdout_file=${SLURM_SUBMIT_DIR-$PWD}/${SLURM_JOB_NAME-"local"}_${SLURM_JOB_ID-"id"}.log
- exec > ${stdout_file}
- echo "------------------ Job Info --------------------"
- echo "jobid : ${SLURM_JOB_ID-"id"}"
- echo "jobname : ${SLURM_JOB_NAME-"local"}"
- echo "nodename : ${SLURMD_NODENAME-"local"}"
- echo "# nodes : ${SLURM_JOB_NUM_NODES-$LOCAL_NODES}"
- echo "# tasks : ${SLURM_NTASKS-$LOCAL_TASKS}"
- echo "submit dir : ${SLURM_SUBMIT_DIR-$PWD}"
- set -ue
- #
- # Cluster variables
- #
- NB_CORES_PER_NODES=128
- MAX_CORES_PER_NODES=123
- LIST_CORES_SOCKET=`seq -s',' 0 $((NB_CORES_PER_NODES-1))`
- #
- # Directories
- start_dir=${SLURM_SUBMIT_DIR-$PWD}
- # librunscript defines some helper functions
- . ${start_dir}/librunscript.sh
- # =============================================================================
- # *** BEGIN User configuration
- # =============================================================================
- # -----------------------------------------------------------------------------
- # *** General configuration
- # -----------------------------------------------------------------------------
- # Component configuration
- # (for syntax of the $config variable, see librunscript.sh)
- config="ifs nemo lim3 rnfmapper xios:detached oasis"
- # Experiment name (exactly 4 letters!)
- export exp_name=EC00
- # Simulation start and end date. Use any (reasonable) syntax you want.
- run_start_date="1970-01-01"
- run_end_date="${run_start_date} + 52 years"
- # Set $force_run_from_scratch to 'true' if you want to force this run to start
- # from scratch, possibly ignoring any restart files present in the run
- # directory. Leave set to 'false' otherwise.
- # NOTE: If set to 'true' the run directory $run_dir is cleaned!
- force_run_from_scratch=true
- special_restart=false
- special_restart_from=ECE3
- special_restart_date="1995-01-01"
- # Resolution (TM5 resolution is set at compilation)
- ifs_grid=T255L91
- nem_grid=ORCA1L75
- # Restart frequency. Use any (reasonable) number and time unit you want.
- # For runs without restart, leave this variable empty
- rst_freq="6 months"
- # Number of restart legs to be run in one go
- run_num_legs=1
- # Directories
- ctrl_file_dir=${start_dir}/ctrl
- output_control_files_dir=${start_dir}/ctrl
- # Architecture
- build_arch=ecconf
- # This file is used to store information about restarts
- ece_info_file="ece.info"
- # -----------------------------------------------------------------------------
- # *** Read platform dependent configuration
- # -----------------------------------------------------------------------------
- #. ${start_dir}/ecconf.cfg
- # This function should configure all settings/modules needed to
- # later prepare the EC-Earth run directory and set variables used
- # in the run script
- # SCRATCH is not defined in MN3, define it here
- # and also make sure it is defined when compiling
- export SCRATCH=/gpfs/scratch/acad/ecearth/${USER}
- # Configure paths for building/running EC-Earth
- ecearth_src_dir=${HOME}/models/ecearth_3.3.1/sources
- run_dir=/gpfs/scratch/acad/ecearth/${USER}/ecearth/run/${exp_name}
- ini_data_dir=/gpfs/scratch/acad/ecearth/data/bsc32/v3.3.1/inidata
- archive_dir=/gpfs/scratch/acad/ecearth/${USER}/ecearth/archive/${exp_name}
- # File for standard output.
- # NOTE: This will be modified for restart jobs!
- stdout_file=${SLURM_SUBMIT_DIR-$PWD}/${SLURM_JOB_NAME-"local"}_${SLURM_JOB_ID-"id"}.log
- # Resubmit this job for automatic restarts? [true/false]
- # Also, add options for the resubmit command here.
- resubmit_job=true
- resubmit_opt=""
- module purge
- module load EasyBuild/2023a
- MODULEPATH=$MODULEPATH:/gpfs/projects/acad/ecearth/softs/easybuild/modules/all
- module load netCDF-Fortran/4.6.1-iompi-2023a
- module load imkl/2023.1.0
- module load grib_api/1.24.0-iompi-2023a
- module load CDO/1.9.10-iompi-2023a
- # Configure grib api paths
- export GRIB_DEFINITION_PATH=${HOME}/models/ecearth_3.3.1/sources/util/grib_table_126:${EBROOTGRIB_API}/share/grib_api/definitions
- export GRIB_SAMPLES_PATH=${EBROOTGRIB_API}/share/grib_api/ifs_samples/grib1
- export GRIB_BIN_PATH=${EBROOTGRIB_API}/bin
- # Configure number of processors per node
- proc_per_node=128
- ulimit -s unlimited
- # -----------------------------------------------------------------------------
- # *** Time step settings
- # -----------------------------------------------------------------------------
- case "${ifs_grid}--${nem_grid}" in
- T159L*--ORCA1L*)
- ifs_time_step_sec=3600; nem_time_step_sec=2700; lim_time_step_sec=2700; cpl_freq_atm_oce_sec=10800
- ;;
- T255L*--ORCA1L*)
- ifs_time_step_sec=2700; nem_time_step_sec=2700; lim_time_step_sec=2700; cpl_freq_atm_oce_sec=2700
- ;;
- T511L*--ORCA025L*)
- ifs_time_step_sec=900 ; nem_time_step_sec=900 ; lim_time_step_sec=900 ; cpl_freq_atm_oce_sec=2700
- ;;
- *) error "Can't set time steps for unknown combination of horizontal grids: ${ifs_grid}-${nem_grid}"
- ;;
- esac
- # -----------------------------------------------------------------------------
- # *** IFS configuration
- # -----------------------------------------------------------------------------
- ifs_version=36r4
- ifs_di_freq=$(( 24 * 3600 / ifs_time_step_sec ))
- ifs_ddh_freq=$(( 120 * 3600 / ifs_time_step_sec ))
- export ifs_res_hor=$(echo ${ifs_grid} | sed 's:T\([0-9]\+\)L\([0-9]\+\):\1:')
- ifs_res_ver=$(echo ${ifs_grid} | sed 's:T\([0-9]\+\)L\([0-9]\+\):\2:')
- ifs_numproc=400
- ifs_exe_file=${ecearth_src_dir}/ifs-${ifs_version}/bin/ifsmaster-${build_arch}
- ifs_lastout=false
- # USE FORCING FROM CMIP5 (SOLAR, GHG, AEROSOL, O3)
- ifs_cmip5=TRUE
- # SWITCH FOR RCP AND HISTORICAL RUNS FOR CMIP5 (0=HISTO 1=RCP 3-PD, 2=RCP 4.5, 3=RCP 6.0, 4=RCP 8.5)
- ifs_cmip5_rcp=0
- # 1PCTCO2 and A4XCO2 should operate together with ifs_cmip_fixyear years, ie., the baseline GHGs level will use that for year ifs_cmip_fixyear
- export ifs_cmip_fixyear=1850
- # USE FORCING FROM CMIP6 (HAS PRIORITY OVER LCMIP5)
- ifs_cmip6=TRUE
- # MAC-SP anthropogenic simple plume model (parameterization of anthropogenic aerosol optical properties)
- ifs_mac2sp=TRUE
- # Use CMIP6 prescribed preindustrial aerosol
- ifs_cmip6piaer=TRUE
- # !! scenario 'historical' max 2014
- # 1850 (o3_pi)
- # 1850 --> 2014 (o3_histo)
- # !! other scenarios (> 2014), default: SSP3-7.0
- # SSP1-1.9, SSP1-2.6, SSP1-2.6-Ext, SSP2-4.5, SSP3-7.0, SSP3-LowNTCF, SSP4-3.4, SSP5-3.4-OS, SSP4-6.0, SSP5-3.4-OS-Ext, SSP5-8.5, SSP5-8.5-Ext
- ifs_cmip6_scenario=historical
- # Read CMIP6 stratospheric aerosol data file, vertically integrated version
- lcmip6_strataer_simp=FALSE
- lcmip6_strataer_full=TRUE
- lcmip6_strataer_bckgd=FALSE
- # for an abrupt increase of CO2 to 4x the starting year (ifs_cmip_fixyear)
- export ifs_A4xCO2=FALSE
- # for an 1% per year increase of CO2 until reaching 4x the starting year (ifs_cmip_fixyear)
- export ifs_1PCTCO2=FALSE
- # Time-varying orbital forcing (Qiong Zhang, SU-2013-09)
- # https://dev.ec-earth.org/projects/ecearth3/wiki/Orbital_forcing_in_EC-Earth_3
- #
- # ifs_orb_switch=false, no orbital calculations applied
- # ifs_orb_switch=true, use orbital calculations according to ifs_orb_mode
- # ifs_orb_mode="fixed_year", or "variable_year", or "fixed_parameters"
- # fixed_year: calculate the orbital parameters at ifs_orb_iyear, e.g.,1850
- # variable_year: calculate orbital parameters annually start from ifs_orb_iyear
- # fixed_parameters: prescribe orbital parameters for given year
- ifs_orb_switch=FALSE
- ifs_orb_mode="variable_year"
- ifs_orb_iyear=$(date -u -d "${run_start_date}" +%Y)
- # IFS tuning parameters
- ifs_tuning_parameter_file=${ctrl_file_dir}/ifs-tuning-parameters-${ifs_grid}.sh
- if [ -f ${ifs_tuning_parameter_file} ]
- then
- source ${ifs_tuning_parameter_file}
- else
- error "Sorry, ${ifs_tuning_parameter_file} not found, exiting."
- fi
- # Select source of vegetation data:
- # ifs climatology from IFS
- # era20c vegetation from an off-line LPJ-Guess run forced with ERA20C
- # (currently available only for T255 and T159)
- #
- ifs_veg_source="era20c"
- case ${ifs_veg_source} in
- "ifs" )
- # Use Lambert-Beer to compute effective vegetation cover
- n_compute_eff_veg_fraction=2
- ;;
- "era20c" )
- # LPJG vegetation is provided as effective cover
- # Don't use Lambert-Beer
- n_compute_eff_veg_fraction=0
- ;;
- * )
- error "Vegetation from ${ifs_veg_source} not implemented"
- ;;
- esac
- # use DMI land ice physics and varying snow albedo
- ifs_landice=false
- # -----------------------------------------------------------------------------
- # *** NEMO/LIM configuration
- # -----------------------------------------------------------------------------
- # This is only needed if the experiment is started from an existing set of NEMO
- # restart files
- nem_restart_file_path=${start_dir}/nemo-rst
- nem_restart_offset=0
- nem_res_hor=$(echo ${nem_grid} | sed 's:ORCA\([0-9]\+\)L[0-9]\+:\1:')
- nem_exe_file=${ecearth_src_dir}/nemo-3.6/CONFIG/${nem_grid}_LIM3/BLD/bin/nemo.exe
- nem_numproc=98
- # -----------------------------------------------------------------------------
- # *** Runoff mapper configuration
- # -----------------------------------------------------------------------------
- rnf_exe_file=${ecearth_src_dir}/runoff-mapper/bin/runoff-mapper.exe
- rnf_numproc=1
- # -----------------------------------------------------------------------------
- # *** OASIS configuration
- # -----------------------------------------------------------------------------
- # Restart files for the coupling fields (note 8 character limit in OASIS)
- # rstas.nc : atmosphere single-category fields
- # rstam.nc : atmosphere multi-category fields
- # rstos.nc : ocean single-category fields
- # rstom.nc : ocean multi-category fields
- oas_rst_files="rstas.nc rstos.nc"
- # Decide whether the OASIS weight files for interpolation should be linked from
- # the setup directory (true) or not (false). In the latter case, the weights
- # are re-computed at the start of the run.
- oas_link_weights=true
- # Flux correction for runoff (not calving) sent from Oasis to ocean.
- # 1.07945 is computed to compensate for a P-E=-0.016 mm/day (valid for std res)
- oas_mb_fluxcorr=1.07945
- # -----------------------------------------------------------------------------
- # *** XIOS configuration
- # -----------------------------------------------------------------------------
- xio_exe_file=${ecearth_src_dir}/xios-2.5/bin/xios_server.exe
- xio_numproc=1
- # -----------------------------------------------------------------------------
- # *** Extra initial conditions saved during the run
- # -----------------------------------------------------------------------------
- if has_config save_ic
- then
- source ./libsave_ic.sh
- declare -a save_ic_date save_ic_date1 save_ic_sec save_ic_day save_ic_ppt_file save_ic_nemo_ts
- fi
- # =============================================================================
- # *** END of User configuration
- # =============================================================================
- # =============================================================================
- # *** This is where the code begins ...
- # =============================================================================
- # -----------------------------------------------------------------------------
- # *** Create the run dir if necessary and go there
- # Everything is done from here.
- # -----------------------------------------------------------------------------
- if [ ! -d ${run_dir} ]
- then
- mkdir -p ${run_dir}
- if $special_restart
- then
- force_run_from_scratch=false
- echo 'rsync -av --delete ${run_dir}/../${special_restart_from}/ --exclude log --exclude output --exclude restart --exclude="${special_restart_from}_*" --exclude="srf*" --exclude="restart_*" --exclude="debug.*" --ex clude="output.*" ${run_dir}'
- rsync -av --delete ${run_dir}/../${special_restart_from}/ --exclude log --exclude output --exclude restart --exclude="${special_restart_from}_*" --exclude="srf*" --exclude="restart_*" --exclude="debug.*" --exclude="output.*" ${run_dir}
- cp -f ${nem_exe_file} ${run_dir}
- cp -f ${ifs_exe_file} ${run_dir}
- cp -f ${rnf_exe_file} ${run_dir}
- cp -f ${xio_exe_file} ${run_dir}
- special_year=${special_restart_date:0:4}
- sed -i "/$special_year/q" ${run_dir}/ece.info
- . ${run_dir}/ece.info
- special_restart_leg=$(printf %03d $((leg_number+1)))
- special_restart_leg_oasis=$(printf %03d $((leg_number+2)))
- # PUT HERE THE INSTRUCTIONS TO COPY THE restart files
- rsync -av ${run_dir}/../../archive/${special_restart_from}/restart/ifs/${special_restart_leg}/ ${run_dir}
- rsync -av ${run_dir}/../../archive/${special_restart_from}/restart/oasis/${special_restart_leg_oasis}/ ${run_dir}
- cd ${run_dir}/../../archive/${special_restart_from}/restart/nemo/${special_restart_leg}
- for f in *.nc; do
- nf=${exp_name}${f:4}
- cp $f ${run_dir}/$nf
- done
- cd -
- cd ${run_dir}
- for f in ${exp_name}_????????_restart_???_????.nc; do
- nf=${f:14}
- ln -s $f $nf
- done
- cd -
- rm -f ${run_dir}/ICMCL${special_restart_from}INIT
- mv ${run_dir}/ICMGG${special_restart_from}INIUA ${run_dir}/ICMGG${exp_name}INIUA
- mv ${run_dir}/ICMGG${special_restart_from}INIT ${run_dir}/ICMGG${exp_name}INIT
- mv ${run_dir}/ICMSH${special_restart_from}INIT ${run_dir}/ICMSH${exp_name}INIT
- fi
- else
- force_run_from_scratch=false
- special_restart=false
- fi
- cd ${run_dir}
- # -----------------------------------------------------------------------------
- # *** Determine the time span of this run and whether it's a restart leg
- # -----------------------------------------------------------------------------
- # Regularise the format of the start and end date of the simulation
- run_start_date=$(date -uR -d "${run_start_date}")
- run_end_date=$(date -uR -d "${run_end_date}")
- # -----------------------------------------------------------------------------
- # *** Set path to grib_set
- # -----------------------------------------------------------------------------
- grib_set=${GRIB_BIN_PATH}${GRIB_BIN_PATH:+/}grib_set
- # Loop over the number of legs
- for (( ; run_num_legs>0 ; run_num_legs-- ))
- do
- # Check for restart information file and set the current leg start date
- # Ignore restart information file if force_run_from_scratch is true
- if ${force_run_from_scratch} || ! [ -r ${ece_info_file} ]
- then
- leg_is_restart=false
- leg_start_date=${run_start_date}
- leg_number=1
- else
- leg_is_restart=true
- . ./${ece_info_file}
- leg_start_date=${leg_end_date}
- leg_number=$((leg_number+1))
- fi
- # Compute the end date of the current leg
- if [ -n "${rst_freq}" ]
- then
- leg_end_date=$(date -uR -d "${leg_start_date} + ${rst_freq}")
- else
- leg_end_date=${run_end_date}
- fi
- if [ $(date -u -d "${leg_end_date}" +%s) -ge $(date -u -d "${run_end_date}" +%s) ]
- then
- leg_end_date=${run_end_date}
- ifs_lastout=true
- fi
- # Some time variables needed later
- leg_length_sec=$(( $(date -u -d "${leg_end_date}" +%s) - $(date -u -d "${leg_start_date}" +%s) ))
- leg_start_sec=$(( $(date -u -d "${leg_start_date}" +%s) - $(date -u -d "${run_start_date}" +%s) ))
- leg_end_sec=$(( $(date -u -d "${leg_end_date}" +%s) - $(date -u -d "${run_start_date}" +%s) ))
- leg_start_date_yyyymmdd=$(date -u -d "${leg_start_date}" +%Y%m%d)
- leg_start_date_yyyymm=$(date -u -d "${leg_start_date}" +%Y%m)
- leg_start_date_yyyy=$(date -u -d "${leg_start_date}" +%Y)
- leg_end_date_yyyy=$(date -u -d "${leg_end_date}" +%Y)
- # Check whether there's actually time left to simulate - exit otherwise
- if [ ${leg_length_sec} -le 0 ]
- then
- info "Leg start date equal to or after end of simulation."
- info "Nothing left to do. Exiting."
- exit 0
- fi
- # Initial conditions saved during the run
- do_save_ic=false
- has_config save_ic && save_ic_get_config
- # if you do not use an option with save_ic, you must define 'do_save_ic' and
- # 'save_ic_date_offset' here or in ../libsave_ic.sh/save_ic_get_config()
- ${do_save_ic} && save_ic_define_vars
- # -------------------------------------------------------------------------
- # *** Prepare the run directory for a run from scratch
- # -------------------------------------------------------------------------
- if ! $leg_is_restart
- then
- # ---------------------------------------------------------------------
- # *** Check if run dir is empty. If not, and if we are allowed to do so
- # by ${force_run_from_scratch}, remove everything
- # ---------------------------------------------------------------------
- if $(ls * >& /dev/null)
- then
- if ${force_run_from_scratch}
- then
- rm -fr ${run_dir}/*
- else
- error "Run directory ${run_dir} not empty and \$force_run_from_scratch not set."
- fi
- fi
- # ---------------------------------------------------------------------
- # *** Copy executables of model components
- # *** Additionally, create symlinks to the original place for reference
- # ---------------------------------------------------------------------
- cp ${ifs_exe_file} .
- ln -s ${ifs_exe_file} $(basename ${ifs_exe_file}).lnk
- cp ${nem_exe_file} .
- ln -s ${nem_exe_file} $(basename ${nem_exe_file}).lnk
- cp ${rnf_exe_file} .
- ln -s ${rnf_exe_file} $(basename ${rnf_exe_file}).lnk
- cp ${xio_exe_file} .
- ln -s ${xio_exe_file} $(basename ${xio_exe_file}).lnk
- # ---------------------------------------------------------------------
- # *** Files needed for IFS (linked)
- # ---------------------------------------------------------------------
- # Initial data
- ln -s \
- ${ini_data_dir}/ifs/${ifs_grid}/${leg_start_date_yyyymmdd}/ICMGGECE3INIUA \
- ICMGG${exp_name}INIUA
- ln -s \
- ${ini_data_dir}/ifs/${ifs_grid}/${leg_start_date_yyyymmdd}/ICMSHECE3INIT \
- ICMSH${exp_name}INIT
- rm -f ICMGG${exp_name}INIT
- cp ${ini_data_dir}/ifs/${ifs_grid}/${leg_start_date_yyyymmdd}/ICMGGECE3INIT \
- ICMGG${exp_name}INIT
- # add bare_soil_albedo to ICMGG*INIT
- tempfile=tmp.$$
- ${grib_set} -s dataDate=$(date -u -d "$run_start_date" +%Y%m%d) \
- ${ini_data_dir}/ifs/${ifs_grid}/climate/bare_soil_albedos.grb \
- ${tempfile}
- cat ${tempfile} >> ICMGG${exp_name}INIT
- rm -f ${tempfile}
- # add land ice mask if needed
- if ${ifs_landice}
- then
- tempfile=tmp.$$
- cdo divc,10 -setcode,82 -selcode,141 ICMGG${exp_name}INIT ${tempfile}
- ${grib_set} -s gridType=reduced_gg ${tempfile} ${tempfile}
- cat ${tempfile} >> ICMGG${exp_name}INIT
- rm -f ${tempfile}
- fi
- # Other stuff
- ln -s ${ini_data_dir}/ifs/rtables/* .
- # Output control (ppt files)
- if [ ! -f ${output_control_files_dir}/pptdddddd0600 ] && [ ! -f ${output_control_files_dir}/pptdddddd0300 ];then
- echo "Error from ece-esm.sh: Neither the file pptdddddd0600 or pptdddddd0300 exists in the directory:"
- echo " " ${output_control_files_dir}
- exit -1
- fi
- mkdir postins
- cp ${output_control_files_dir}/ppt* postins/
- if [ -f postins/pptdddddd0600 ];then
- ln -s pptdddddd0600 postins/pptdddddd0000
- ln -s pptdddddd0600 postins/pptdddddd1200
- ln -s pptdddddd0600 postins/pptdddddd1800
- fi
- if [ -f postins/pptdddddd0300 ];then
- ln -s pptdddddd0300 postins/pptdddddd0900
- ln -s pptdddddd0300 postins/pptdddddd1500
- ln -s pptdddddd0300 postins/pptdddddd2100
- if [ ! -f postins/pptdddddd0600 ];then
- ln -s pptdddddd0300 postins/pptdddddd0000
- ln -s pptdddddd0300 postins/pptdddddd0600
- ln -s pptdddddd0300 postins/pptdddddd1200
- ln -s pptdddddd0300 postins/pptdddddd1800
- fi
- fi
- /bin/ls -1 postins/* > dirlist
- # ---------------------------------------------------------------------
- # *** Files needed for NEMO (linked)
- # ---------------------------------------------------------------------
- # Link initialisation files for matching ORCA grid
- for f in \
- bathy_meter.nc coordinates.nc \
- ahmcoef.nc \
- K1rowdrg.nc M2rowdrg.nc mask_itf.nc \
- decay_scale_bot.nc decay_scale_cri.nc \
- mixing_power_bot.nc mixing_power_cri.nc mixing_power_pyc.nc \
- runoff_depth.nc subbasins.nc
- do
- [ -f ${ini_data_dir}/nemo/initial/${nem_grid}/$f ] && ln -s ${ini_data_dir}/nemo/initial/${nem_grid}/$f
- done
- # Link geothermal heating file (independent of grid) and matching weight file
- ln -s ${ini_data_dir}/nemo/initial/Goutorbe_ghflux.nc
- ln -s ${ini_data_dir}/nemo/initial/weights_Goutorbe1_2_orca${nem_res_hor}_bilinear.nc
- # Link the salinity climatology file (needed for diagnostics)
- ln -s ${ini_data_dir}/nemo/climatology/${nem_grid}/sali_ref_clim_monthly.nc
- # Link either restart files or climatology files for the initial state
- if $(has_config nemo:start_from_restart)
- then
- # When linking restart files, we accept three options:
- # (1) Merged files for ocean and ice, i.e.
- # restart_oce.nc and restart_ice.nc
- # (2) One-file-per-MPI-rank, i.e.
- # restart_oce_????.nc and restart_ice_????.nc
- # No check is done whether the number of restart files agrees
- # with the number of MPI ranks for NEMO!
- # (3) One-file-per-MPI-rank with a prefix, i.e.
- # <exp_name>_<time_step>_restart_oce_????.nc (similar for the ice)
- # The prefix is ignored.
- # The code assumes that one of the options can be applied! If more
- # options are applicable, the first is chosen. If none of the
- # options apply, NEMO will crash with missing restart file.
- if ls -U ${nem_restart_file_path}/restart_[oi]ce.nc > /dev/null 2>&1
- then
- ln -s ${nem_restart_file_path}/restart_[oi]ce.nc ./
- elif ls -U ${nem_restart_file_path}/restart_[oi]ce_????.nc > /dev/null 2>&1
- then
- ln -s ${nem_restart_file_path}/restart_[oi]ce_????.nc ./
- else
- for f in ${nem_restart_file_path}/????_????????_restart_[oi]ce_????.nc
- do
- ln -s $f $(echo $f | sed 's/.*_\(restart_[oi]ce_....\.nc\)/\1/')
- done
- fi
- else
- # Temperature and salinity files for initialisation
- ln -s ${ini_data_dir}/nemo/climatology/absolute_salinity_WOA13_decav_Reg1L75_clim.nc
- ln -s ${ini_data_dir}/nemo/climatology/conservative_temperature_WOA13_decav_Reg1L75_clim.nc
- ln -s ${ini_data_dir}/nemo/climatology/weights_WOA13d1_2_orca${nem_res_hor}_bilinear.nc
- # Grid dependent runoff files
- case ${nem_grid} in
- ORCA1*) ln -s ${ini_data_dir}/nemo/climatology/runoff-icb_DaiTrenberth_Depoorter_ORCA1_JD.nc ;;
- ORCA025*) ln -s ${ini_data_dir}/nemo/climatology/ORCA_R025_runoff_v1.1.nc ;;
- esac
- fi
- # for ocean_nudging
- if $(has_config nemo:ocenudg) ; then
- ln -fs ${ini_data_dir}/nemo/oce_nudg/resto.nc ./
- fi
- # XIOS files
- . ${ctrl_file_dir}/iodef.xml.sh > iodef.xml
- ln -s ${ctrl_file_dir}/context_nemo.xml
- ln -s ${ctrl_file_dir}/domain_def_nemo.xml
- ln -s ${ctrl_file_dir}/axis_def_nemo.xml
- ln -s ${ctrl_file_dir}/grids_def_nemo.xml
- ln -s ${ctrl_file_dir}/field_def_nemo-lim.xml
- ln -s ${ctrl_file_dir}/field_def_nemo-opa.xml
- ln -s ${ctrl_file_dir}/field_def_nemo-pisces.xml
- ln -s ${ctrl_file_dir}/field_def_nemo-inerttrc.xml
- ln -s ${output_control_files_dir}/file_def_nemo-lim3.xml file_def_nemo-lim.xml
- ln -s ${output_control_files_dir}/file_def_nemo-opa.xml
- ln -s ${output_control_files_dir}/file_def_nemo-pisces.xml
- if [ -f ${ini_data_dir}/xios/ORCA${nem_res_hor}/coordinates_xios.nc ]
- then
- cp ${ini_data_dir}/xios/ORCA${nem_res_hor}/coordinates_xios.nc ./
- else
- info "File 'coordinates_xios.nc' not found. NEMO can not be run with land domain removal!"
- fi
- # ---------------------------------------------------------------------
- # *** Files needed for the Runoff mapper (linked)
- # ---------------------------------------------------------------------
- ln -s ${ini_data_dir}/runoff-mapper/runoff_maps.nc
- # ---------------------------------------------------------------------
- # *** Files needed for OASIS (linked)
- # ---------------------------------------------------------------------
- oas_grid_dir=${ini_data_dir}/oasis/T${ifs_res_hor}-ORCA${nem_res_hor}
- # Name table file
- ln -s ${ini_data_dir}/oasis/cf_name_table.txt
- # Grid definition files
- ln -s ${oas_grid_dir}/areas.nc
- ln -s ${oas_grid_dir}/grids.nc
- ln -s ${oas_grid_dir}/masks.nc
- # Weight files
- case ${ifs_res_hor} in
- 159) oas_agrd=080
- ;;
- 255) oas_agrd=128
- ;;
- 511) oas_agrd=256
- ;;
- 799) oas_agrd=400
- ;;
- *) error "Unsupported horizontal resolution (IFS): ${ifs_res_hor}"
- ;;
- esac
- case ${nem_res_hor} in
- 1) oas_ogrd=O1t0
- ;;
- 025) oas_ogrd=Ot25
- ;;
- *) error "Unsupported horizontal resolution (NEMO): ${nem_res_hor}"
- ;;
- esac
- if ${oas_link_weights}
- then
- for f in ${oas_grid_dir}/rmp_????_to_????_GAUSWGT.nc
- do
- ln -s $f
- done
- fi
- for f in ${oas_rst_files}
- do
- cp ${oas_grid_dir}/rst/$f .
- done
- else # i.e. $leg_is_restart == true
- # ---------------------------------------------------------------------
- # *** Remove all leftover output files from previous legs
- # ---------------------------------------------------------------------
- # IFS files
- rm -f ICM{SH,GG}${exp_name}+??????
- # NEMO files
- rm -f ${exp_name}_??_????????_????????_{grid_U,grid_V,grid_W,grid_T,icemod,SBC,scalar,SBC_scalar,diad_T}.nc
- fi # ! $leg_is_restart
- #--------------------------------------------------------------------------
- # *** Surface restoring and ocean nudging options
- #--------------------------------------------------------------------------
- #for ocean_nudging
- if $(has_config nemo:ocenudg) ; then
- ln -fs ${ini_data_dir}/nemo/oce_nudg/temp_sal*.nc ./
- fi
- #for surface restoring
- if $(has_config nemo:surfresto) ; then
- ln -fs ${ini_data_dir}/nemo/surface_restoring/sss_restore_data*.nc ./
- ln -fs ${ini_data_dir}/nemo/surface_restoring/sst_restore_data*.nc ./
- ln -fs ${ini_data_dir}/nemo/surface_restoring/mask_restore*.nc ./
- fi
- # -------------------------------------------------------------------------
- # *** Remove land grid-points
- # -------------------------------------------------------------------------
- if $(has_config nemo:elpin)
- then
- if [ ! -f coordinates_xios.nc ]
- then
- error "ELpIN requested, but file 'coordinates_xios.nc' was not found"
- fi
- jpns=($(${ecearth_src_dir}/util/ELPiN/ELPiNv2.cmd ${nem_numproc}))
- info "nemo domain decompostion from ELpIN: ${jpns[@]}"
- nem_numproc=${jpns[0]}
- nem_jpni=${jpns[1]}
- nem_jpnj=${jpns[2]}
- else
- info "nemo original domain decomposition (not using ELPiN)"
- fi
- # -------------------------------------------------------------------------
- # *** Initial conditions saved during the run
- # -------------------------------------------------------------------------
- ${do_save_ic} && save_ic_prepare_output
- # -------------------------------------------------------------------------
- # *** Create some control files
- # -------------------------------------------------------------------------
- # IFS frequency output for namelist
- if [ -f postins/pptdddddd0300 ]
- then
- ifs_output_freq=$(( 3 * 3600 / ifs_time_step_sec ))
- elif [ -f postins/pptdddddd0600 ]
- then
- ifs_output_freq=$(( 6 * 3600 / ifs_time_step_sec ))
- else
- error "IFS output frequency undefined."
- fi
- # IFS, NEMO, LIM namelist and OASIS namcouple files
- . ${ctrl_file_dir}/namelist.ifs.sh > fort.4
- . ${ctrl_file_dir}/namelist.nemo.ref.sh > namelist_ref
- . ${ctrl_file_dir}/namelist.nemo-${nem_grid}-coupled.cfg.sh > namelist_cfg
- . ${ctrl_file_dir}/namelist.lim3.ref.sh > namelist_ice_ref
- . ${ctrl_file_dir}/namelist.lim3-${nem_grid}.cfg.sh > namelist_ice_cfg
- . ${ctrl_file_dir}/namelist.runoffmapper.sh > namelist.runoffmapper
- . ${ctrl_file_dir}/namcouple.sh > namcouple
- # -------------------------------------------------------------------------
- # *** Create ICMCL file with vegetation fields
- # not needed if LPJG is used with feedback
- # -------------------------------------------------------------------------
- tempfile=tmp.$$
- case ${ifs_veg_source} in
- "ifs" )
- # Vegetation from IFS (climatology)
- icmclfile=${ini_data_dir}/ifs/${ifs_grid}/climate/ICMCL_ONLY_VEG_PD
- # Create data for december, the year before the leg starts
- ${grib_set} \
- -s dataDate=$(printf "%04d" $((leg_start_date_yyyy-1)))1215 \
- ${icmclfile}-12 ICMCL${exp_name}INIT
- # Create data for all month in the years of the leg
- for (( y=${leg_start_date_yyyy} ; y<=${leg_end_date_yyyy} ; y++ ))
- do
- yy=$(printf "%04d" $y)
- for m in {1..12}
- do
- mm=$(printf "%02d" $m)
- ${grib_set} -s dataDate=${yy}${mm}15 ${icmclfile}-${mm} ${tempfile}
- cat ${tempfile} >> ICMCL${exp_name}INIT
- done
- done
- # Create data for january, the year after the leg ends
- ${grib_set} \
- -s dataDate=$(printf "%04d" $((leg_end_date_yyyy+1)))0115 \
- ${icmclfile}-01 ${tempfile}
- cat ${tempfile} >> ICMCL${exp_name}INIT
- ;;
- "era20c" )
- # Vegetation from an off-line LPJG run forced with ERA20C (v16)
- rm -f ICMCL${exp_name}INIT
- # Create data for all years of the leg, including one year
- # before and one year after
- for (( yr=leg_start_date_yyyy-1 ; yr<=leg_end_date_yyyy+1 ; yr+=1 ))
- do
- if [ $ifs_cmip_fixyear -le 0 ]
- then
- cat ${ini_data_dir}/ifs/${ifs_grid}/icmcl_v16/icmcl_$yr.grb >> ICMCL${exp_name}INIT
- else
- # Fixed year forcing, requires cdo! (only when not using ifs_veg_source=custom_exp*)
- # If cdo is not available at runtime you need to fix proper
- # icmcl files beforehand and use them here
- cdo setyear,$yr ${ini_data_dir}/ifs/${ifs_grid}/icmcl_v16/icmcl_${ifs_cmip_fixyear}.grb ${tempfile}
- cat ${tempfile} >> ICMCL${exp_name}INIT
- fi
- done
- ;;
- * )
- error "Vegetation from ${ifs_veg_source} not implemented"
- ;;
- esac
- # Clean up
- rm -f ${tempfile}
- # -------------------------------------------------------------------------
- # *** Link the appropriate NEMO restart files of the previous leg
- # -------------------------------------------------------------------------
- if $leg_is_restart && ! $special_restart
- then
- ns=$(printf %08d $(( leg_start_sec / nem_time_step_sec - nem_restart_offset )))
- for (( n=0 ; n<nem_numproc ; n++ ))
- do
- np=$(printf %04d ${n})
- ln -fs ${exp_name}_${ns}_restart_oce_${np}.nc restart_oce_${np}.nc
- ln -fs ${exp_name}_${ns}_restart_ice_${np}.nc restart_ice_${np}.nc
- done
- # Make sure there are no global restart files
- # If links are found, they will be removed. We are cautious and do
- # _not_ remove real files! However, if real global restart files are
- # present, NEMO/LIM will stop because time stamps will not match.
- [ -h restart_oce.nc ] && rm restart_oce.nc
- [ -h restart_ice.nc ] && rm restart_ice.nc
- fi
- # -------------------------------------------------------------------------
- # *** Remove some OASIS files of the previous leg
- # -------------------------------------------------------------------------
- if $leg_is_restart
- then
- rm -f anaisout_*
- fi
- # -------------------------------------------------------------------------
- # *** Start the run
- # -------------------------------------------------------------------------
- export DR_HOOK_IGNORE_SIGNALS='-1'
- export CPLNG='active'
- # Use the launch function from the platform configuration file
- t1=$(date +%s)
- # Compute and check the node distribution
- info "======================="
- info "Node/proc distribution:"
- info "-----------------------"
- info "IFS: ${ifs_numproc}"
- info "NEMO: ${nem_numproc}"
- info "XIOS: ${xio_numproc}"
- info "RUNOFF: ${rnf_numproc}"
- info "======================="
- pwd
- info "======================="
- info "mpirun \
- -n ${xio_numproc} ./$(basename ${xio_exe_file}) : \
- -n ${nem_numproc} ./$(basename ${nem_exe_file}) : \
- -n ${ifs_numproc} ./$(basename ${ifs_exe_file}) -v ecmwf -e ${exp_name} : \
- -n ${rnf_numproc} ./$(basename ${rnf_exe_file})"
- info "======================="
-
- #exit
- mpirun \
- -n ${xio_numproc} ./$(basename ${xio_exe_file}) : \
- -n ${nem_numproc} ./$(basename ${nem_exe_file}) : \
- -n ${ifs_numproc} ./$(basename ${ifs_exe_file}) -v ecmwf -e ${exp_name} : \
- -n ${rnf_numproc} ./$(basename ${rnf_exe_file})
- t2=$(date +%s)
- tr=$(date -u -d "0 -$t1 sec + $t2 sec" +%T)
- # -------------------------------------------------------------------------
- # *** Check for signs of success
- # Note the tests provide no guarantee that things went fine! They are
- # just based on the IFS and NEMO log files. More tests (e.g. checking
- # restart files) could be implemented.
- # -------------------------------------------------------------------------
- # Checking for IFS success
- if [ -f ifs.stat ]
- then
- if [ "$(awk 'END{print $3}' ifs.stat)" == "CNT0" ]
- then
- info "Leg successfully completed according to IFS log file 'ifs.stat'."
- else
- error "Leg not completed according to IFS log file 'ifs.stat'."
- fi
- else
- error "IFS log file 'ifs.stat' not found after run."
- fi
- # Check for NEMO success
- if [ -f ocean.output ]
- then
- if [ "$(sed -n '/New day/h; ${g;s:.*\([0-9/]\{10\}\).*:\1:;p;}' ocean.output)" == "$(date -u -d "${leg_end_date} - 1 day" +%Y/%m/%d)" ]
- then
- info "Leg successfully completed according to NEMO log file 'ocean.output'."
- else
- error "Leg not completed according to NEMO log file 'ocean.output'."
- fi
- else
- error "NEMO log file 'ocean.output' not found after run."
- fi
- # -------------------------------------------------------------------------
- # *** Post-process initial conditions saved during the run if requested
- # -------------------------------------------------------------------------
- ${do_save_ic} && save_ic_postproc
- # -------------------------------------------------------------------------
- # *** Move IFS output files to archive directory
- # -------------------------------------------------------------------------
- outdir="${archive_dir}/output/ifs/$(printf %03d $((leg_number)))"
- mkdir -p ${outdir}
- prv_leg=$(printf %03d $((leg_number-1)))
- # This takes care of a special IFS feature: The output for the last time
- # step of each leg is written at the first time step of the new leg. The
- # following code makes sure that the output is appended to the appropriate
- # file. Since GRIB files are just streams, its done with a simple cat
- # command.
- for f in ICMSH${exp_name}+?????? ICMGG${exp_name}+??????
- do
- if [ -f output/ifs/${prv_leg}/${f} ]
- then
- cat ${f} >> output/ifs/${prv_leg}/${f}
- rm -f ${f}
- else
- mv ${f} ${outdir}
- fi
- done
- # -------------------------------------------------------------------------
- # *** Move NEMO output files to archive directory
- # -------------------------------------------------------------------------
- outdir="${archive_dir}/output/nemo/$(printf %03d $((leg_number)))"
- mkdir -p ${outdir}
- for v in grid_U grid_V grid_W grid_T icemod SBC scalar SBC_scalar diad_T \
- grid_T_2D grid_U_2D grid_V_2D grid_W_2D grid_T_3D grid_U_3D grid_V_3D grid_W_3D \
- grid_1point grid_T_3D_ncatice vert_sum \
- grid_ptr_W_3basin_3D grid_ptr_T_3basin_2D grid_ptr_T_2D \
- zoom_700_sum zoom_300_sum zoom_2000_sum
- do
- for f in ${exp_name}_*_????????_????????_*${v}.nc
- do
- test -f $f && mv $f $outdir/
- done
- done
- # -------------------------------------------------------------------------
- # *** Move IFS restart files to archive directory
- # -------------------------------------------------------------------------
- if $leg_is_restart
- then
- outdir="${archive_dir}/restart/ifs/$(printf %03d $((leg_number)))"
- mkdir -p ${outdir}
- # Figure out the time part of the restart files (cf. CTIME on rcf files)
- # NOTE: Assuming that restarts are at full days (time=0000) only!
- nd="$(printf %06d $((leg_start_sec/(24*3600))))0000"
- mv srf${nd}.???? ${outdir}
- fi
- # -------------------------------------------------------------------------
- # *** Move NEMO restart files to archive directory
- # -------------------------------------------------------------------------
- if $leg_is_restart && $(has_config nemo)
- then
- outdir="${archive_dir}/restart/nemo/$(printf %03d $((leg_number)))"
- mkdir -p ${outdir}
- ns=$(printf %08d $(( leg_start_sec / nem_time_step_sec - nem_restart_offset )))
- for f in oce ice
- do
- mv ${exp_name}_${ns}_restart_${f}_????.nc ${outdir}
- done
- fi
- # -------------------------------------------------------------------------
- # *** Copy OASIS restart files to archive directory
- # NOTE: These files are copied and not moved as they are used in the
- # next leg!
- # Note also that the OASIS restart files present at the end of
- # the leg correspond to the start of the next leg!
- # -------------------------------------------------------------------------
- outdir="${archive_dir}/restart/oasis/$(printf %03d $((leg_number+1)))"
- mkdir -p ${outdir}
- for f in ${oas_rst_files}
- do
- test -f ${f} && cp ${f} ${outdir}
- done
- # -------------------------------------------------------------------------
- # *** Copy rcf files to the archive directory (of the next leg!)
- # -------------------------------------------------------------------------
- outdir="${archive_dir}/restart/ifs/$(printf %03d $((leg_number+1)))"
- mkdir -p ${outdir}
- for f in rcf
- do
- test -f ${f} && cp ${f} ${outdir}
- done
- # -------------------------------------------------------------------------
- # *** Move log files to archive directory
- # -------------------------------------------------------------------------
- outdir="${archive_dir}/log/$(printf %03d $((leg_number)))"
- mkdir -p ${outdir}
- for f in \
- ifs.log ifs.stat fort.4 ocean.output \
- time.step solver.stat \
- nout.000000 debug.??.?????? A*_??.nc O*_??.nc
- do
- test -f ${f} && mv ${f} ${outdir}
- done
- # -------------------------------------------------------------------------
- # *** Write the restart control file
- # -------------------------------------------------------------------------
- # Compute CPMIP performance
- sypd="$(cpmip_sypd $leg_length_sec $(($t2 - $t1)))"
- chpsy="$(cpmip_chpsy $leg_length_sec $(($t2 - $t1)) $(($ifs_numproc + $nem_numproc + $rnf_numproc + $xio_numproc)))"
- echo "#" | tee -a ${ece_info_file}
- echo "# Finished leg at `date '+%F %T'` after ${tr} (hh:mm:ss)" \
- | tee -a ${ece_info_file}
- echo "# CPMIP performance: $sypd SYPD $chpsy CHPSY"| tee -a ${ece_info_file}
- echo "leg_number=${leg_number}" | tee -a ${ece_info_file}
- echo "leg_start_date=\"${leg_start_date}\"" | tee -a ${ece_info_file}
- echo "leg_end_date=\"${leg_end_date}\"" | tee -a ${ece_info_file}
- # Need to reset force_run_from_scratch in order to avoid destroying the next leg
- force_run_from_scratch=false
- special_restart=false
- done # loop over legs
- # -----------------------------------------------------------------------------
- # *** Platform dependent finalising of the run
- # -----------------------------------------------------------------------------
- #finalise
- if ${resubmit_job} && [ $(date -d "${leg_end_date}" +%s) -lt $(date -d "${run_end_date}" +%s) ]
- then
- info "Resubmitting job for leg $((leg_number+1))"
- # Need to go to start_dir to find the run script
- cd ${start_dir}
- # Submit command
- echo "sbatch -N ${SLURM_JOB_NUM_NODES-"1"} -o ${run_dir}/$(basename ${stdout_file}).$(printf %03d $((leg_number+1))) \
- -e ${run_dir}/$(basename ${stdout_file}).$(printf %03d $((leg_number+1))) -d ${SLURM_JOB_ID-"id"} \
- ./${SLURM_JOB_NAME-"run"}.sh"
- # Note: This does not work if you specify a job name with sbatch -J jobname!
- sbatch -N ${SLURM_JOB_NUM_NODES-"1"} \
- -o ${run_dir}/$(basename ${stdout_file}).$(printf %03d $((leg_number+1))) \
- -e ${run_dir}/$(basename ${stdout_file}).$(printf %03d $((leg_number+1))) \
- -d ${SLURM_JOB_ID-"id"} \
- ./${SLURM_JOB_NAME-"run"}.sh
- fi
- exit 0
|