123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537 |
- #!/usr/bin/env bash
- set -ue
- # autosubmit stuff - this is ignored if using the classic runtime
- if [[ "%%_%%" == "%_%" ]]
- then
- use_autosubmit=true
- set -xuve
- # go to autosubmit root directory
- cd %CURRENT_ROOTDIR%
- # Simulation member. Use any (reasonable) syntax you want.
- member="%MEMBER%"
- else
- use_autosubmit=false
- # Simulation member. Use any (reasonable) syntax you want.
- member="fc0"
- fi
- # librunscript defines some helper functions
- source ./librunscript.sh
- # =============================================================================
- # *** BEGIN User configuration
- # =============================================================================
- # -----------------------------------------------------------------------------
- # *** General configuration
- # -----------------------------------------------------------------------------
- # Component configuration (for syntax of the $config variable, see librunscript.sh)
- #
- if $use_autosubmit
- then
- [[ "%LPJG%" = TRUE ]] && config="%LPJG_CONFIG% lpjg_forcing:IFS oasis" || config=""
- else
- config="lpjg lpjg_forcing:IFS oasis"
- fi
- # minimum sanity
- has_config lpjg_forcing ifs && error "Cannot have both ifs and lpjg_forcing in config!!"
- # Experiment name (exactly 4 letters!)
- export exp_name=[[[MOD:GENERAL:EXP_NAME]]]
- # Simulation start and end date. Use any (reasonable) syntax you want.
- run_start_date="[[[MOD:GENERAL:RUN_START_DATE]]]"
- run_end_date="[[[MOD:GENERAL:RUN_END_DATE]]]"
- # Set $force_run_from_scratch to 'true' if you want to force this run to start
- # from scratch, possibly ignoring any restart files present in the run
- # directory. Leave set to 'false' otherwise.
- # NOTE: If set to 'true' the run directory $run_dir is cleaned!
- force_run_from_scratch=[[[MOD:GENERAL:FORCE_RUN_FROM_SCRATCH]]]
- # Resolution (only T255L91 supported for now)
- ifs_grid=T255L91 #[[[MOD:IFS:GRID]]]
- #atm_grid=A128
- # Restart frequency. Use any (reasonable) number and time unit you want.
- # For runs without restart, leave this variable empty
- rst_freq="[[[MOD:GENERAL:RST_FREQ]]]"
- # Number of restart legs to be run in one go
- run_num_legs=[[[MOD:GENERAL:RUN_NUM_LEGS]]]
- # Coupling frequencies
- has_config lpjg && cpl_freq_atm_lpjg_hrs=[[[MOD:OASIS:CPL_FREQ_ATM_LPJG_HRS]]] #24
- # Directories
- start_dir=${PWD}
- ctrl_file_dir=${start_dir}/ctrl
- # Architecture
- build_arch=[[[PLT:ACTIVE:BUILD_ARCH]]]
- # This file is used to store information about restarts
- ece_info_file="ece.info"
- # -----------------------------------------------------------------------------
- # *** Read platform dependent configuration
- # -----------------------------------------------------------------------------
- source ./ecconf.cfg
- configure
- # -----------------------------------------------------------------------------
- # *** IFS configuration - these variables are needed by LPJG
- # -----------------------------------------------------------------------------
- export ifs_res_hor=$(echo ${ifs_grid} | sed 's:T\([0-9]\+\)L\([0-9]\+\):\1:')
- ifs_res_ver=$(echo ${ifs_grid} | sed 's:T\([0-9]\+\)L\([0-9]\+\):\2:')
- ifs_cmip_fixyear=[[[MOD:IFS:CMIP_FIXYEAR]]]
- ifs_A4xCO2=[[[MOD:IFS:CMIP6_A4xCO2]]]
- ifs_1PCTCO2=[[[MOD:IFS:CMIP6_1PCTCO2]]]
- # -----------------------------------------------------------------------------
- # *** LPJ-GUESS offline forcing configuration
- # -----------------------------------------------------------------------------
- lpjg_forcing_exe_file=${ecearth_src_dir}/lpjg/offline/lpjg_forcing
- has_config lpjg_forcing:IFS && lpjg_forcing_exe_file=${ecearth_src_dir}/lpjg/offline/lpjg_forcing_ifs
- lpjg_forcing_numproc=1
- # TODO add this to config-run.xml ?
- lpjg_forcing_files_path="/gpfs/projects/bsc32/bsc32051/lpjg/forcing/a0pq/out"
- # -----------------------------------------------------------------------------
- # *** LPJ-GUESS configuration
- # -----------------------------------------------------------------------------
- lpjg_time_step_sec=[[[MOD:LPJG:TIME_STEP_SEC]]]
- lpjg_numproc=[[[MOD:LPJG:NUMPROC]]]
- has_config lpjg && lpjg_on=1
- has_config lpjg:fdbck && lpjg_fdbck=1
- has_config tm5 && lpjg_fdbck_tm5=1 || lpjg_fdbck_tm5=0
- export lpjg_fixNdepafter=[[[MOD:LPJG:LPJG_FIXNDEPAFTER]]]
- export lpjg_fixLUafter=[[[MOD:LPJG:LPJG_FIXLUAFTER]]]
- lpjg_exe_file=${ecearth_src_dir}/lpjg/build/guess
- # -----------------------------------------------------------------------------
- # =============================================================================
- # *** END of User configuration
- # =============================================================================
- # =============================================================================
- # *** This is where the code begins ...
- # =============================================================================
- # -----------------------------------------------------------------------------
- # *** Create the run dir if necessary and go there
- # Everything is done from here.
- # -----------------------------------------------------------------------------
- if [ ! -d ${run_dir} ]
- then
- mkdir -p ${run_dir}
- fi
- cd ${run_dir}
- # -----------------------------------------------------------------------------
- # *** Determine the time span of this run and whether it's a restart leg
- # -----------------------------------------------------------------------------
- # Regularise the format of the start and end date of the simulation
- run_start_date=$(date -uR -d "${run_start_date}")
- run_end_date=$(date -uR -d "${run_end_date}")
- # Loop over the number of legs
- for (( ; run_num_legs>0 ; run_num_legs-- ))
- do
- # Check for restart information file and set the current leg start date
- # Ignore restart information file if force_run_from_scratch is true
- if ${force_run_from_scratch} || ! [ -r ${ece_info_file} ]
- then
- leg_is_restart=false
- leg_start_date=${run_start_date}
- leg_number=1
- else
- leg_is_restart=true
- . ./${ece_info_file}
- leg_start_date=${leg_end_date}
- leg_number=$((leg_number+1))
- fi
- # Compute the end date of the current leg
- if [ -n "${rst_freq}" ]
- then
- leg_end_date=$(date -uR -d "${leg_start_date} + ${rst_freq}")
- else
- leg_end_date=${run_end_date}
- fi
- if [ $(date -d "${leg_end_date}" +%s) -gt $(date -d "${run_end_date}" +%s) ]
- then
- leg_end_date=${run_end_date}
- fi
- # Some time variables needed later
- leg_length_sec=$(( $(date -d "${leg_end_date}" +%s) - $(date -d "${leg_start_date}" +%s) ))
- leg_start_sec=$(( $(date -d "${leg_start_date}" +%s) - $(date -d "${run_start_date}" +%s) ))
- leg_end_sec=$(( $(date -d "${leg_end_date}" +%s) - $(date -d "${run_start_date}" +%s) ))
- leg_start_date_yyyymmdd=$(date -u -d "${leg_start_date}" +%Y%m%d)
- leg_start_date_yyyy=$(date -u -d "${leg_start_date}" +%Y)
- leg_end_date_yyyy=$(date -u -d "${leg_end_date}" +%Y)
- leg_length_years=$(( ${leg_end_date_yyyy} - ${leg_start_date_yyyy} ))
- # Check whether there's actually time left to simulate - exit otherwise
- if [ ${leg_length_sec} -le 0 ]
- then
- info "Leg start date equal to or after end of simulation."
- info "Nothing left to do. Exiting."
- exit 0
- fi
- # -------------------------------------------------------------------------
- # *** Prepare the run directory for a run from scratch
- # -------------------------------------------------------------------------
- if ! $leg_is_restart
- then
- # ---------------------------------------------------------------------
- # *** Check if run dir is empty. If not, and if we are allowed to do so
- # by ${force_run_from_scratch}, remove everything
- # ---------------------------------------------------------------------
- if $(ls * >& /dev/null)
- then
- if ${force_run_from_scratch}
- then
- rm -fr ${run_dir}/*
- else
- error "Run directory ${run_dir} not empty and \$force_run_from_scratch not set."
- fi
- fi
- # ---------------------------------------------------------------------
- # *** Copy executables of model components
- # *** Additionally, create symlinks to the original place for reference
- # ---------------------------------------------------------------------
- if $(has_config lpjg)
- then
- cp ${lpjg_exe_file} .
- ln -s ${lpjg_exe_file} $(basename ${lpjg_exe_file}).lnk
- fi
- if $(has_config lpjg_forcing)
- then
- cp ${lpjg_forcing_exe_file} .
- fi
- # ---------------------------------------------------------------------
- # *** Files needed for LPJ-GUESS
- # ---------------------------------------------------------------------
- if $(has_config lpjg)
- then
- # Initial data - saved state for LPJ-GUESS (.bin format)
- lpjgstartdir=$(printf "lpjg_state_%04d" $leg_start_date_yyyy)
- ln -sf ${ini_data_dir}/lpjg/${lpjgstartdir}/ ${run_dir}/${lpjgstartdir}
- # Control files (i.e. .ins, landuse, N deposition, soil type files etc.)
- #FIXME: the ins files should not come from two sources and even if, the source dir should be the first that is copied
- #and then insfiles out of the ini or some experiment data dir could overwrite some of the defaults.
- #cp ${ini_data_dir}/lpjg/insfiles/* ${run_dir}
- cp -f ${ecearth_src_dir}/lpjg/data/ins/*.ins .
- mkdir -p ${run_dir}/landuse
- cp ${ini_data_dir}/lpjg/landuse/* ${run_dir}/landuse
- mkdir -p ${run_dir}/ndep
- cp ${ini_data_dir}/lpjg/ndep/* ${run_dir}/ndep
- cp ${ini_data_dir}/lpjg/ndep/* ${run_dir}
- cp ${ini_data_dir}/ifs/cmip6-data/CMIP6_histo_mole_fraction_of_carbon_dioxide_in_air_input4MIPs_gr1-GMNHSH.nc ${run_dir}
- fi
- #TODO ET ask Anthoni for this file
- # if $(has_config GSWP3)
- # then
- # cp -f ${ecearth_src_dir}/lpjg/offline/guess_GSWP3.ins ${run_dir}/guess.ins
- # fi
- else # i.e. $leg_is_restart == true
- # ---------------------------------------------------------------------
- # *** Remove all leftover output files from previous legs
- # ---------------------------------------------------------------------
- # OSM lpjg files ????
- echo nothing to do
- fi # ! $leg_is_restart
- # -------------------------------------------------------------------------
- # *** Create some control files
- # -------------------------------------------------------------------------
- # namelist and namcouple files
- has_config lpjg_forcing && . ${ctrl_file_dir}/namelist.lpjg_forcing.sh > namelist.lpjg_forcing
- . ${ctrl_file_dir}/namcouple.sh > namcouple
- # -------------------------------------------------------------------------
- # *** LPJ-GUESS initial data
- # -------------------------------------------------------------------------
- if $(has_config lpjg)
- then
- # LPJG runtime rcfile - update with leg dates
- . ${ctrl_file_dir}/namelist.lpjg.sh > lpjg_steps.rc
- # Populate or update LPJG run directories
- for (( n=1; n<=${lpjg_numproc}; n++ ))
- do
- if ! $leg_is_restart
- then
- # Make output directories
- # add the ioopt config option if running on one node to speed up output
- if $(has_config lpjg:ioopt)
- then
- mkdir -p $TMPDIR/lpjg/run${n}/output
- mkdir -p ${run_dir}/run${n}
- ln -sf $TMPDIR/lpjg/run${n}/output ${run_dir}/run${n}/output
- else
- mkdir -p ${run_dir}/run${n}/output
- fi
- # Copy *.ins, lpjg_steps.rc and OASIS-MCT grid and restart files
- cp ${run_dir}/*.ins ${run_dir}/run${n}
- cp ${ini_data_dir}/lpjg/oasismct/*.nc ${run_dir}
- cp ${ini_data_dir}/lpjg/oasismct/*.txt ${run_dir}/run${n}
- # Link OASIS-MCT grid definition files
- # would normally come from oasis/T255-... dir
- for f in areas grids masks
- do
- ln -sf ${f}lpjg.nc ${f}.nc
- done
- # If num_proc has increased provide additional rundirs.
- elif [ ! -d ${run_dir}/run${n} ]
- then
- mkdir -p ${run_dir}/run${n}/output
- cp ${run_dir}/*.ins ${run_dir}/run${n}
- cp ${ini_data_dir}/lpjg/oasismct/*.txt ${run_dir}/run${n}
- fi
- # Refresh dirs after they hav been removed after end of leg
- mkdir -p ${run_dir}/run${n}/output/CMIP6
- done
- if $leg_is_restart
- then
- lpjg_restart_dir="restart/lpjg/$(printf %03d $((leg_number-1)))"
- lpjg_rst_state="${lpjg_restart_dir}/lpjg_state_${leg_start_date_yyyy}"
- if [ -d "$lpjg_rst_state" ]
- then
- ln -sf $lpjg_rst_state
- else
- echo "lpjg restart dir $lpjg_rst_state not available"
- exit -1
- fi
- # get oasis restart files
- oasis_restart_dir="restart/oasis/$(printf %03d $((leg_number)))"
- cp -f ${oasis_restart_dir}/*.nc ${run_dir}/
- fi
- fi
- # -------------------------------------------------------------------------
- # *** Remove some OASIS files of the previous leg
- # -------------------------------------------------------------------------
- # TODO check oasis restart files
- #if $leg_is_restart
- #then
- # rm -f anaisout_*
- #fi
- # -------------------------------------------------------------------------
- # *** Start the run
- # -------------------------------------------------------------------------
- #needed for osm???
- # export DR_HOOK_IGNORE_SIGNALS='-1'
- # export CPLNG='active'
- has_config lpjg && cmd="${lpjg_numproc} ${lpjg_exe_file} guess.ins -parallel" || error "no lpjg in config"
- has_config lpjg_forcing && cmd=${cmd}" -- ${lpjg_forcing_numproc} ${lpjg_forcing_exe_file}"
- t1=$(date +%s)
- launch $cmd
- t2=$(date +%s)
- tr=$(date -d "0 -$t1 sec + $t2 sec" +%T)
- # -------------------------------------------------------------------------
- # *** Check for signs of success
- # Note the tests provide no guarantee that things went fine! They are
- # just based on the IFS, NEMO and TM5 log files. More tests (e.g. checking
- # restart files) could be implemented.
- # -------------------------------------------------------------------------
- # Check for LPJG success ???
- # -------------------------------------------------------------------------
- # *** Move LPJ-GUESS output files to archive directory
- # -------------------------------------------------------------------------
- if $(has_config lpjg)
- then
- outdir="output/lpjg/$(printf %03d $((leg_number)))"
- if [ -d ${outdir} ]
- then
- rm -rf ${outdir}
- fi
- mkdir -p ${outdir}
- # LPJG run directories
- # concatenate *.out files from each run* into output dir
- # DMG: adding escaped % for autosubmit, undo when issue #42 is resolved
- if $use_autosubmit
- then
- flist=$(find ${run_dir}/run1/output/ -type f | sed "s%${run_dir}/run1/output\/%%%%")
- else
- flist=$(find ${run_dir}/run1/output/ -type f | sed "s%${run_dir}/run1/output\/%%")
- fi
- mkdir ${outdir}/CMIP6
- for (( n=1; n<=${lpjg_numproc}; n++ ))
- do
- for ofile in $flist
- do
- if (( n == 1 ))
- then
- cat ${run_dir}/run${n}/output/${ofile} > ${outdir}/$ofile
- else
- awk '(FNR!=1){print $0}' ${run_dir}/run${n}/output/${ofile} >> ${outdir}/$ofile
- fi
- done
- rm -rf ${run_dir}/run${n}/output
- done
- # move monthly file if available
- if [ -f ${run_dir}/LPJ-GUESS_monthlyoutput.txt ]
- then
- mv ${run_dir}/LPJ-GUESS_monthlyoutput.txt ${outdir}
- fi
- fi
- #the output might be large, lets gzip those text files
- # this is not in the esm runscript anymore
- find ${outdir} -type f | xargs -I %% gzip %%
- # -------------------------------------------------------------------------
- # *** Move LPJ-GUESS restart files to archive directory
- # -------------------------------------------------------------------------
- if $(has_config lpjg)
- then
- outdir="restart/lpjg/$(printf %03d $((leg_number)))"
- if [ -d ${outdir} ]
- then
- rm -rf ${outdir}
- fi
- mkdir -p ${outdir}
- state_dir="./lpjg_state_$(printf %04d $((leg_end_date_yyyy)))"
- mv ${state_dir} ${outdir}
- # LPJG writes into run1 dir, so mv to main rundir
- mv -f run1/lpjgv.nc .
- # remove restart link
- if $leg_is_restart
- then
- old_state_dir="./lpjg_state_$(printf %04d $((leg_start_date_yyyy)))"
- if [ -L $old_state_dir ]
- then
- rm -f "$old_state_dir"
- fi
- fi
- fi
- # -------------------------------------------------------------------------
- # *** Move log files to archive directory
- # -------------------------------------------------------------------------
- outdir="log/$(printf %03d $((leg_number)))"
- mkdir -p ${outdir}
- for f in \
- guess.log \
- nout.000000 debug.??.?????? A*_??.nc O*_??.nc
- do
- test -f ${f} && mv ${f} ${outdir}
- done
- # lpjg run logs
- # this is not in the esm runscript anymore
- # other (lpjg_forcing osm) logs ???
- if $(has_config lpjg)
- then
- mkdir -p ${outdir}/lpjg
- for (( n=1; n<=${lpjg_numproc}; n++ ))
- do
- #concat the lpjg run logs
- if (( n == 1 ))
- then
- echo "--------- lpjg run${n} log --------" > ${outdir}/lpjg/guess_runs.log
- else
- echo "--------- lpjg run${n} log --------" >> ${outdir}/lpjg/guess_runs.log
- fi
- cat ${run_dir}/run${n}/guess$((n-1)).log >> ${outdir}/lpjg/guess_runs.log
- done
- fi
- # -------------------------------------------------------------------------
- # *** Copy OASIS restart files to archive directory
- # NOTE: These files are copied and not moved as they are used in the
- # next leg!
- # Note also that the OASIS restart files present at the end of
- # the leg correspond to the start of the next leg!
- # -------------------------------------------------------------------------
- outdir="restart/oasis/$(printf %03d $((leg_number+1)))"
- mkdir -p ${outdir}
- for f in vegin.nc lpjgv.nc
- do
- test -f ${f} && cp ${f} ${outdir}
- done
- # -------------------------------------------------------------------------
- # *** Write the restart control file
- # -------------------------------------------------------------------------
- # Compute CPMIP performance
- sypd="$(cpmip_sypd $leg_length_sec $(($t2 - $t1)))"
- ncores=0
- # has_config osm && (( ncores+=${osm_numproc} )) || :
- has_config lpjg && (( ncores+=${lpjg_numproc} )) || :
- has_config lpjg_forcing && (( ncores+=${lpjg_forcing_numproc} )) || :
- chpsy="$(cpmip_chpsy $leg_length_sec $(($t2 - $t1 +1)) $ncores)" #t_run+1 prevent div zero, just in case
- echo "#" | tee -a ${ece_info_file}
- echo "# Finished leg at `date '+%F %T'` after ${tr} (hh:mm:ss)" \
- | tee -a ${ece_info_file}
- echo "# CPMIP performance: $sypd SYPD $chpsy CHPSY"| tee -a ${ece_info_file}
- echo "leg_number=${leg_number}" | tee -a ${ece_info_file}
- echo "leg_start_date=\"${leg_start_date}\"" | tee -a ${ece_info_file}
- echo "leg_end_date=\"${leg_end_date}\"" | tee -a ${ece_info_file}
- # Need to reset force_run_from_scratch in order to avoid destroying the next leg
- force_run_from_scratch=false
- done # loop over legs
- # -----------------------------------------------------------------------------
- # *** Platform dependent finalising of the run
- # -----------------------------------------------------------------------------
- finalise
- $use_autosubmit || exit 0
|