#!/usr/bin/env bash set -ue # librunscript defines some helper functions source ./librunscript.sh # ============================================================================= # *** BEGIN User configuration # ============================================================================= # ----------------------------------------------------------------------------- # *** General configuration # ----------------------------------------------------------------------------- # Component configuration (for syntax of the $config variable, see librunscript.sh) # # Currently maintained: # config="osm" # OSM (Offline Surface Model, H-TESSEL), forced by IFS output or reanalysis data # config="lpjg" # LPJG-Offline, forced by IFS or OSM output (lpjg_forcing) # In this config lpjg can take the option lpjg:spinup to run an offline spinup # config="lpjg:fdbck osm" # LPJG & OSM Coupled via OASIS - option lpjg:fdbck is used to feedback on the OSM config="lpjg:fdbck osm" # dummy tm5 coupling - co2 ppm + fluxes ccycle_co2_init_ppm=0 oas_rst_ifs_tm5='' if has_config lpjg:cc_coup then # define initial CO2 concentrations in ppm ccycle_co2_init_ppm=284.32 # set to true to write co2 fluxes ccycle_debug_fluxes=true config+=" tm5:co2 cc_coup" export tm5_exch_nlevs=10 tm5_time_step_sec=[[[MOD:LPJG:TIME_STEP_SEC]]] oas_rst_ifs_tm5=$oas_rst_ifs_tm5' l_co2.nc rlpjg.nc' fi # minimum sanity has_config ifs && error "Cannot have ifs in config" has_config nemo && error "Cannot have nemo in config" has_config lpjg:spinup osm && error "Cannot have both lpjg:spinup and osm in config" # add lpjg_forcing to config when needed (lpjg non-spinup and no osm) - do not specify it in the config directly has_config lpjg_forcing && error "lpjg:lpjg_forcing not allowed in user config" has_config lpjg && ! has_config lpjg:spinup && ! has_config osm && config+=" lpjg_forcing" has_config lpjg_forcing osm && error "Cannot have both lpjg_forcing and osm in config" has_config lpjg:spinup lpjg_forcing && error "Cannot have both lpjg:spinup and lpjg_forcing in config" # liblsm.sh defines some OSM and LPJG pre/post-processing functions has_config any osm lpjg lpjg_forcing && source ./liblsm.sh # Experiment name (exactly 4 letters!) export exp_name=[[[MOD:GENERAL:EXP_NAME]]] # Simulation start and end date. Use any (reasonable) syntax you want. run_start_date="[[[MOD:GENERAL:RUN_START_DATE]]]" # define run_start_date_as with proper format e.g. 19900101 export run_start_date_as=$(date -u -d "${run_start_date}" +%Y%m%d) run_end_date="[[[MOD:GENERAL:RUN_END_DATE]]]" # Set $force_run_from_scratch to 'true' if you want to force this run to start # from scratch, possibly ignoring any restart files present in the run # directory. Leave set to 'false' otherwise. # NOTE: If set to 'true' the run directory $run_dir is cleaned! force_run_from_scratch=[[[MOD:GENERAL:FORCE_RUN_FROM_SCRATCH]]] # Resolution (only T159 and T255 supported) ifs_grid=[[[MOD:IFS:GRID]]] #atm_grid=A128 # Restart frequency. Use any (reasonable) number and time unit you want. # For runs without restart, leave this variable empty rst_freq="[[[MOD:GENERAL:RST_FREQ]]]" # Number of restart legs to be run in one go run_num_legs=[[[MOD:GENERAL:RUN_NUM_LEGS]]] # Coupling frequencies has_config lpjg && cpl_freq_atm_lpjg_hrs=[[[MOD:OASIS:CPL_FREQ_ATM_LPJG_HRS]]] #24 # Directories start_dir=${PWD} ctrl_file_dir=${start_dir}/ctrl output_control_files_dir=${start_dir}/[[[MOD:GENERAL:OUTPUT_CONTROL_FILES_DIR]]] # Architecture build_arch=[[[PLT:ACTIVE:BUILD_ARCH]]] use_machinefile=[[[PLT:ACTIVE:USE_MACHINEFILE]]] # This file is used to store information about restarts ece_info_file="ece.info" # ----------------------------------------------------------------------------- # *** Read platform dependent configuration # ----------------------------------------------------------------------------- source ./ecconf.cfg configure [ -z $TMPDIR ] && error "you must define TMPDIR for your platform!!!" # ----------------------------------------------------------------------------- # *** IFS/OSM configuration - some variables are needed by LPJG # ----------------------------------------------------------------------------- ifs_version=36r4 export ifs_res_hor=$(echo ${ifs_grid} | sed 's:T\([0-9]\+\)L\([0-9]\+\):\1:') ifs_res_ver=$(echo ${ifs_grid} | sed 's:T\([0-9]\+\)L\([0-9]\+\):\2:') osm_numproc=[[[MOD:OSM:NUMPROC]]] # has_config lpjg && osm_exe_file=${ecearth_src_dir}/ifs-${ifs_version}/src/surf/offline/osm/build/bin/master1s_cpl.exe \ || osm_exe_file=${ecearth_src_dir}/ifs-${ifs_version}/src/surf/offline/osm/build/bin/master1s.exe osm_forcing_type="[[[MOD:OSM:OSM_FORCING_TYPE]]]" osm_ifs_output_dir="[[[MOD:OSM:OSM_IFS_OUTPUT_DIR]]]" osm_forcing_dir="[[[MOD:OSM:OSM_FORCING_DIR]]]" #uncomment these lines to use era20c/erai forcings, which are generated with the generate_forcing.sh script #osm_forcing_type="era20c" #osm_ifs_output_dir="" #osm_forcing_dir="${SCRATCH}/ECEARTH-RUNS/OSM_FORCING/${osm_forcing_type}/${ifs_grid}" export ifs_cmip_fixyear=[[[MOD:IFS:CMIP_FIXYEAR]]] ifs_cmip6=[[[MOD:IFS:CMIP6]]] ifs_cmip6_scenario=[[[MOD:IFS:CMIP6_SCENARIO]]] ifs_cmip_fixyear_ch4=[[[MOD:IFS:CMIP_FIXYEAR_CH4]]] ifs_A4xCO2=[[[MOD:IFS:CMIP6_A4xCO2]]] ifs_1PCTCO2=[[[MOD:IFS:CMIP6_1PCTCO2]]] bgc_1PCTCO2=[[[MOD:IFS:CMIP6_1PCTCO2_BGC]]] # *** Time step settings # setting ifs_time_step_sec breaks LPJG-offline oasis config so only set it when osm is active for now # usually we use the IFS time step setting for the target resolution, but for ERA5 forcing # we use 1h which is the resolution of the forcing if has_config osm then case "${ifs_grid}" in T159L*) ifs_time_step_sec=3600 ;; T255L*) ifs_time_step_sec=2700 ;; T511L*) ifs_time_step_sec=900 ;; *) error "Can't set time steps for unknown horizontal grid: ${ifs_grid}" ;; esac if [[ "$osm_forcing_type" != "ifs" ]] then info "Setting OSM time step to 1h because we are using reanalysis forcing" ifs_time_step_sec=3600 fi # ifs_output_freq is defined below for ifs in the esm runscript, here it is hard-coded for osm ifs_output_freq=$(( 3 * 3600 / ifs_time_step_sec )) #ifs_di_freq=$(( 24 * 3600 / ifs_time_step_sec )) #ifs_ddh_freq=$(( 120 * 3600 / ifs_time_step_sec )) fi # Select source of vegetation data: # ifs climatology from IFS # era20c vegetation from an off-line LPJ-Guess run forced with ERA20C # (currently available only for T255 and T159) # cmip6 vegetation from an EC-Earth3-Veg (interactive LPJ-Guess) run # (currently available only for T255) # custom_exp vegetation from any EC-Earth3-Veg run exp # (must contain same variables as era20c & cmip6 and located in icmcl_exp folder) # none don't create an ICMCL file with vegetation data (this is set # automatically if LPJG is used with feedback) # ifs_veg_source="cmip6" has_config lpjg:fdbck && ! has_config osm && ifs_veg_source="none" case ${ifs_veg_source} in "ifs" ) # Use Lambert-Beer to compute effective vegetation cover n_compute_eff_veg_fraction=2 ;; "era20c" ) # LPJG vegetation is provided as effective cover # Don't use Lambert-Beer n_compute_eff_veg_fraction=0 case "${ifs_grid}" in T159L*) veg_version=v29 ;; T255L*) veg_version=v16 ;; *) error "Vegetation from off-line LPJ-Guess not available for ${ifs_grid}" ;; esac ;; "cmip6" ) # LPJG vegetation is provided as effective cover # Don't use Lambert-Beer n_compute_eff_veg_fraction=0 case "${ifs_grid}" in T255L*) veg_version=v32 ;; *) error "Vegetation from CMIP6 EC-Earth3-Veg not available for ${ifs_grid}" ;; esac ;; "custom_"* ) # LPJG vegetation is provided as effective cover # Don't use Lambert-Beer n_compute_eff_veg_fraction=0 veg_version=${ifs_veg_source:7} if [ ! -d ${ini_data_dir}/ifs/${ifs_grid}/icmcl_${veg_version} ] then error "requested IFS_VEG_SOURCE = ${ifs_veg_source} but not found in ${ini_data_dir}/ifs/${ifs_grid}/icmcl_${veg_version}" fi ;; "none" ) # LPJG with feedback n_compute_eff_veg_fraction=0 has_config osm && ! has_config lpjg:fdbck && error "OSM requires an offline source of vegetation" ;; * ) error "Vegetation from ${ifs_veg_source} not implemented" ;; esac # use DMI land ice physics and varying snow albedo ifs_landice=false # ----------------------------------------------------------------------------- # *** LPJ-GUESS configuration # ----------------------------------------------------------------------------- lpjg_time_step_sec=[[[MOD:LPJG:TIME_STEP_SEC]]] lpjg_numproc=[[[MOD:LPJG:NUMPROC]]] has_config lpjg && lpjg_on=1 has_config lpjg:fdbck && lpjg_fdbck=1 has_config tm5:co2 && lpjg_fdbck_tm5=1 || lpjg_fdbck_tm5=0 export lpjg_fixNdepafter=[[[MOD:LPJG:LPJG_FIXNDEPAFTER]]] export lpjg_fixLUafter=[[[MOD:LPJG:LPJG_FIXLUAFTER]]] info '!!!! CMIP FIX YEAR SETTINGS:' info "ifs_cmip_fixyear: $ifs_cmip_fixyear" info "lpjg_fixNDepAfter: $lpjg_fixNdepafter" info "lpjg_fixLUAfter: $lpjg_fixLUafter" info '!!!!' lpjg_res=T${ifs_res_hor} lpjg_exe_file=${ecearth_src_dir}/lpjg/build/guess_${lpjg_res} # ----------------------------------------------------------------------------- # *** LPJ-GUESS offline forcing configuration # ----------------------------------------------------------------------------- lpjg_forcing_exe_file=${ecearth_src_dir}/lpjg/offline/lpjg_forcing if $(has_config lpjg_forcing) then case "${ifs_grid}" in T159L*) lpjg_forcing_exe_file+="_ifs_T159" ;; T255L*) lpjg_forcing_exe_file+="_ifs_T255" ;; *) has_config lpjg && error "LPJG offline forcing doesn't exist for ifs-grid: ${ifs_grid}" ;; esac fi lpjg_forcing_numproc=1 ccycle_coupling_exe_file=${ecearth_src_dir}/lpjg/offline/ccycle_coupling ccycle_coupling_numproc=1 if $(has_config lpjg) then lpjg_ifs_output_dir="[[[MOD:LPJG:LPJG_IFS_OUTPUT_DIR]]]" lpjg_forcing_dir="[[[MOD:LPJG:LPJG_FORCING_DIR]]]" fi # Decide whether the OASIS weight files for interpolation should be linked from # the setup directory (true) or not (false). In the latter case, the weights # are re-computed at the start of the run. oas_link_weights=true # ============================================================================= # *** END of User configuration # ============================================================================= # ============================================================================= # *** This is where the code begins ... # ============================================================================= # ----------------------------------------------------------------------------- # *** Create the run dir if necessary and go there # Everything is done from here. # ----------------------------------------------------------------------------- if [ ! -d ${run_dir} ] then mkdir -p ${run_dir} fi cd ${run_dir} # ----------------------------------------------------------------------------- # *** Determine the time span of this run and whether it's a restart leg # ----------------------------------------------------------------------------- # Regularise the format of the start and end date of the simulation run_start_date=$(date -uR -d "${run_start_date}") run_end_date=$(date -uR -d "${run_end_date}") # ----------------------------------------------------------------------------- # *** Set path to grib_set # ----------------------------------------------------------------------------- grib_set=${GRIB_BIN_PATH}${GRIB_BIN_PATH:+/}grib_set grib_copy=${GRIB_BIN_PATH}${GRIB_BIN_PATH:+/}grib_copy # Loop over the number of legs for (( ; run_num_legs>0 ; run_num_legs-- )) do # Check for restart information file and set the current leg start date # Ignore restart information file if force_run_from_scratch is true if ${force_run_from_scratch} || ! [ -r ${ece_info_file} ] then leg_is_restart=false leg_start_date=${run_start_date} leg_number=1 else leg_is_restart=true . ./${ece_info_file} leg_start_date=${leg_end_date} leg_number=$((leg_number+1)) fi # Compute the end date of the current leg if [ -n "${rst_freq}" ] then leg_end_date=$(date -uR -d "${leg_start_date} + ${rst_freq}") else leg_end_date=${run_end_date} fi # Check if legs are integer multiples of full years if LPJG is used if has_config lpjg then if [[ $(date +%m%d%T -u -d "${leg_start_date}") != "010100:00:00" || \ $(date +%m%d%T -u -d "${leg_start_date} + ${rst_freq}") != "010100:00:00" ]] then error "LPJ-GUESS runs must start on Jan 1 and end on Dec 31. Multi-year legs are allowed." fi fi if [ $(date -u -d "${leg_end_date}" +%s) -ge $(date -u -d "${run_end_date}" +%s) ] then leg_end_date=${run_end_date} ifs_lastout=true fi # Some time variables needed later leg_length_sec=$(( $(date -u -d "${leg_end_date}" +%s) - $(date -u -d "${leg_start_date}" +%s) )) leg_start_sec=$(( $(date -u -d "${leg_start_date}" +%s) - $(date -u -d "${run_start_date}" +%s) )) leg_end_sec=$(( $(date -u -d "${leg_end_date}" +%s) - $(date -u -d "${run_start_date}" +%s) )) leg_start_date_yyyymmdd=$(date -u -d "${leg_start_date}" +%Y%m%d) leg_start_date_yyyy=$(date -u -d "${leg_start_date}" +%Y) leg_end_date_yyyy=$(date -u -d "${leg_end_date}" +%Y) # compute last full year of the leg, used for ICMCL data and OSM forcing when leg ends on Jan 1st if [ $(date -u -d "${leg_end_date}" +%m%d) == "0101" ] then leg_end_date_yyyy_full=$(( leg_end_date_yyyy - 1 )) else leg_end_date_yyyy_full=$leg_end_date_yyyy fi # Check whether there's actually time left to simulate - exit otherwise if [ ${leg_length_sec} -le 0 ] then info "Leg start date equal to or after end of simulation." info "Nothing left to do. Exiting." exit 0 fi # ------------------------------------------------------------------------- # *** Prepare the run directory for a run from scratch # ------------------------------------------------------------------------- if ! $leg_is_restart then # --------------------------------------------------------------------- # *** Check if run dir is empty. If not, and if we are allowed to do so # by ${force_run_from_scratch}, remove everything # --------------------------------------------------------------------- if $(ls * >& /dev/null) then if ${force_run_from_scratch} then rm -fr ${run_dir}/* else error "Run directory ${run_dir} not empty and \$force_run_from_scratch not set." fi fi # --------------------------------------------------------------------- # *** Copy executables of model components # *** Additionally, create symlinks to the original place for reference # --------------------------------------------------------------------- if $(has_config osm) then cp ${osm_exe_file} . ln -s ${osm_exe_file} $(basename ${osm_exe_file}).lnk fi if $(has_config lpjg) then cp ${lpjg_exe_file} . ln -s ${lpjg_exe_file} $(basename ${lpjg_exe_file}).lnk fi if $(has_config lpjg_forcing) then cp ${lpjg_forcing_exe_file} . fi if $(has_config cc_coup) then cp ${ccycle_coupling_exe_file} . fi # --------------------------------------------------------------------- # *** Files needed for IFS/OSM (linked) # --------------------------------------------------------------------- if $(has_config osm) then # Initial data ln -s \ ${ini_data_dir}/ifs/${ifs_grid}/${leg_start_date_yyyymmdd}/ICMGGECE3INIUA \ ICMGG${exp_name}INIUA ln -s \ ${ini_data_dir}/ifs/${ifs_grid}/${leg_start_date_yyyymmdd}/ICMSHECE3INIT \ ICMSH${exp_name}INIT rm -f ICMGG${exp_name}INIT cp ${ini_data_dir}/ifs/${ifs_grid}/${leg_start_date_yyyymmdd}/ICMGGECE3INIT \ ICMGG${exp_name}INIT # add bare_soil_albedo to ICMGG*INIT tempfile=tmp.$$ ${grib_set} -s dataDate=$(date -u -d "$run_start_date" +%Y%m%d) \ ${ini_data_dir}/ifs/${ifs_grid}/climate/bare_soil_albedos.grb \ ${tempfile} cat ${tempfile} >> ICMGG${exp_name}INIT rm -f ${tempfile} # add land ice mask if needed - not used in OSM yet if ${ifs_landice} then tempfile=tmp.$$ cdo divc,10 -setcode,82 -selcode,141 ICMGG${exp_name}INIT ${tempfile} ${grib_set} -s gridType=reduced_gg ${tempfile} ${tempfile} cat ${tempfile} >> ICMGG${exp_name}INIT rm -f ${tempfile} fi fi #osm # --------------------------------------------------------------------- # *** Files needed for LPJ-GUESS # --------------------------------------------------------------------- if $(has_config lpjg) then # Check for valid grid if [ $lpjg_res != "T255" -a $lpjg_res != "T159" ] then error "LPJG-gridlist doesn't exist for ifs-grid: ${ifs_grid}" fi # Initial data - saved state for LPJ-GUESS (.bin format) lpjgstartdir=$(printf "lpjg_state_%04d" $leg_start_date_yyyy) if ! $(has_config lpjg:spinup) then ln -sf ${ini_data_dir}/lpjg/ini_state/${lpjg_res}/${lpjgstartdir} ${run_dir}/${lpjgstartdir} fi # Control files (i.e. .ins, landuse, N deposition, soil type files etc.) cp -f ${ecearth_src_dir}/lpjg/data/ins/*.ins . # activate the new litterfall scheme for C4MIP - for the coupled model this is done when both pisces and lpjg are activated has_config cc_coup && echo -e "!override for EC-Earth-CC in runscript\nifpftlitterfall 1\ncalc_phen_after_restart 0" >> global.ins mkdir -p ${run_dir}/landuse fi # if running an LPJG spinup, create fast forcing files and link to the ifs_spinup_dir expected by LPJG if $(has_config lpjg:spinup) then lpjg_gen_forcing_spinup ln -sf $lpjg_forcing_dir ifs_spinup_forcing fi # --------------------------------------------------------------------- # *** Files needed for OASIS (linked) # --------------------------------------------------------------------- # Name table file ln -s ${ini_data_dir}/oasis/cf_name_table.txt # we should use the L128 grid (only in AMIP grid file) # but for now let's use the same files as for the coupled runs #oas_grid_dir=${ini_data_dir}/oasis/AMIP oas_grid_dir=${ini_data_dir}/oasis/${lpjg_res}-ORCA1 mycp='cp' # Grid definition files if $(has_config tm5) then ${mycp} ${oas_grid_dir}/areas.nc gcm_areas.nc ${mycp} ${oas_grid_dir}/grids.nc gcm_grids.nc ${mycp} ${oas_grid_dir}/masks.nc gcm_masks.nc oas_grid_dir=${ini_data_dir}/oasis/T${ifs_res_hor}-TM5-LPJG cp ${oas_grid_dir}/tm5_areas.nc . cp ${oas_grid_dir}/tm5_grids.nc . cp ${oas_grid_dir}/tm5_masks.nc . if ${oas_link_weights} then for f in ${oas_grid_dir}/rmp_????_to_????_*.nc do ln -s $f done fi # -- Get restart files for TM5-IFS/LPJG/PISCES for f in ${oas_rst_ifs_tm5} do cp ${oas_grid_dir}/rst/${tm5_exch_nlevs}-levels/$f . done # set CO2 concentrations to 1850 levels as in TM5 code if true then for f in l_co2.nc do [ -f $f ] && mv -f $f $f.bak && cdo setrtoc,0,1000,${ccycle_co2_init_ppm} $f.bak $f done fi # -- Merge grid definition files cdo merge gcm_areas.nc tm5_areas.nc areas.nc cdo merge gcm_grids.nc tm5_grids.nc grids.nc cdo merge gcm_masks.nc tm5_masks.nc masks.nc else ln -s ${oas_grid_dir}/areas.nc ln -s ${oas_grid_dir}/grids.nc ln -s ${oas_grid_dir}/masks.nc fi else # i.e. $leg_is_restart == true # --------------------------------------------------------------------- # *** Remove all leftover output files from previous legs # --------------------------------------------------------------------- # IFS/OSM files rm -f ICM{SH,GG}${exp_name}+?????? fi # ! $leg_is_restart # ------------------------------------------------------------------------- # *** Create some control files # ------------------------------------------------------------------------- # namelist and namcouple files has_config lpjg_forcing && . ${ctrl_file_dir}/namelist.lpjg_forcing.sh > namelist.lpjg_forcing has_config cc_coup && . ${ctrl_file_dir}/namelist.ccycle_coupling.sh > namelist.ccycle_coupling . ${ctrl_file_dir}/namcouple.sh > namcouple # ------------------------------------------------------------------------- # *** LPJ-GUESS initial data # ------------------------------------------------------------------------- if $(has_config lpjg) then # LPJG runtime rcfile - update with leg dates . ${ctrl_file_dir}/namelist.lpjg.sh > lpjg_steps.rc # determine lpjg scenario-name and co2-file middle-fix case $(echo ${ifs_cmip6_scenario} | tr '[:upper:]' '[:lower:]') in hist*) lpjg_scen="ssp370"; lu_src="AIM" ; lpjg_sco2_mfix="AIM-ssp370-1-2-1" ; lu_file_posfix="2018_10_08.txt";; ssp1-1.9*) lpjg_scen="ssp119"; lu_src="IMAGE" ; lpjg_sco2_mfix="IMAGE-ssp119-1-2-1" ; lu_file_posfix="2019_03_13.txt";; ssp1-2.6*) lpjg_scen="ssp126"; lu_src="IMAGE" ; lpjg_sco2_mfix="IMAGE-ssp126-1-2-1" ; lu_file_posfix="2018_10_08.txt";; ssp2-4.5*) lpjg_scen="ssp245"; lu_src="MESSAGE"; lpjg_sco2_mfix="MESSAGE-GLOBIOM-ssp245-1-2-1"; lu_file_posfix="2018_10_08.txt";; ssp3-7.0*) lpjg_scen="ssp370"; lu_src="AIM" ; lpjg_sco2_mfix="AIM-ssp370-1-2-1" ; lu_file_posfix="2018_10_08.txt";; ssp4-3.4*) lpjg_scen="ssp434"; lu_src="GCAM" ; lpjg_sco2_mfix="GCAM4-ssp434-1-2-1" ; lu_file_posfix="2018_10_08.txt";; ssp5-3.4*) lpjg_scen="ssp534os"; lu_src="MAGPIE"; lpjg_sco2_mfix="REMIND-MAGPIE-ssp534-over-1-2-1" ; lu_file_posfix="2019_05_10.txt";; ssp5-8.5*) lpjg_scen="ssp585"; lu_src="MAGPIE" ; lpjg_sco2_mfix="REMIND-MAGPIE-ssp585-1-2-1" ; lu_file_posfix="2018_10_08.txt";; *) error "Scenario ${ifs_cmip6_scenario} not defined for LPJ-GUESS" ;; esac lpjg_scenario_new="historical + $lpjg_sco2_mfix" lpjg_scenario_info=${run_dir}/lpjg_scenario.info if [ -f $lpjg_scenario_info ] then source $lpjg_scenario_info if [ "$lpjg_scenario_new" = "$lpjg_scenario" ] then lpjg_copy_rte=false else lpjg_copy_rte=true fi else lpjg_copy_rte=true fi # copy RTE only if necessary (at beginning of a run or when scenario changes) if $lpjg_copy_rte then # write info about installed scenarios to file echo "lpjg_scenario=\"historical + $lpjg_sco2_mfix\"" > $lpjg_scenario_info # set file prefixes depending on scenario lu_file_prefix="1850_2100_luh2_Hist_ScenarioMIP_UofMD" lu_file_midfix="2_1_f" lu_path="${ini_data_dir}/lpjg/landuse/${lpjg_res}/${lpjg_scen}" # copy and reference-link landuse, gross transitions, crops, n-fertilisation for inp in lu gross crop nfert do if [ $inp == "crop" -o $inp == "nfert" ] then lu_src_file="${lu_path}/${inp}_rfirr_${lu_file_prefix}_${lu_src}_${lpjg_scen}_${lu_file_midfix}_${lpjg_res}_${lu_file_posfix}" else lu_src_file="${lu_path}/${inp}_${lu_file_prefix}_${lu_src}_${lpjg_scen}_${lu_file_midfix}_${lpjg_res}_${lu_file_posfix}" fi cp -f $lu_src_file ${run_dir}/landuse/${inp}_luh2.txt ln -fs $lu_src_file ${run_dir}/landuse/${inp}_luh2.txt.lnk done # nitrogen deposition files mkdir -p ${run_dir}/ndep for inp in drynhx2 drynoy2 wetnhx2 wetnoy2 do ndep_src_file="${ini_data_dir}/lpjg/ndep/${lpjg_res}/${lpjg_scen}/${lpjg_scen}_${lpjg_res}_${inp}.nc" cp -f $ndep_src_file ${run_dir}/ndep/${inp}.nc ln -fs $ndep_src_file ${run_dir}/ndep/${inp}.nc.lnk done # concatenate historic and scenario (2015+) co2 concentration file # hist co2 file lpjg_hco2_file="${ini_data_dir}/ifs/cmip6-data/mole-fraction-of-carbon-dioxide-in-air_input4MIPs_GHGConcentrations_CMIP_UoM-CMIP-1-2-0_gr1-GMNHSH_0000-2014.nc" # scenario co2 file lpjg_sco2_file="${ini_data_dir}/ifs/cmip6-data/mole-fraction-of-carbon-dioxide-in-air_input4MIPs_GHGConcentrations_ScenarioMIP_UoM-${lpjg_sco2_mfix}_gr1-GMNHSH_2015-2500.nc" # combined file lpjg_co2_file="${run_dir}/mole_fraction_of_carbon_dioxide_in_air_input4MIPs_lpjg.nc" rm -f $lpjg_co2_file cdo mergetime $lpjg_hco2_file $lpjg_sco2_file $lpjg_co2_file fi # Populate or update LPJG run directories if $(has_config lpjg:ioopt) then export lpjg_numproc run_node 'echo creating lpjg temp dirs on `hostname` ; rm -rf $TMPDIR/lpjg/run* ; for (( n=1; n<=${lpjg_numproc}; n++ )) ; do mkdir -p $TMPDIR/lpjg/run${n}/output/CMIP6 ; done ; ls -l $TMPDIR/lpjg/' fi for (( n=1; n<=${lpjg_numproc}; n++ )) do # if run from scratch or number of procs has been extended if ! $leg_is_restart || [ ! -d ${run_dir}/run${n} ] then # Make output directories # add the ioopt config option if running on one node to speed up output if $(has_config lpjg:ioopt) then mkdir -p ${run_dir}/run${n} ln -sf $TMPDIR/lpjg/run${n}/output ${run_dir}/run${n}/output else mkdir -p ${run_dir}/run${n}/output/CMIP6 fi # Copy *.ins, lpjg_steps.rc and OASIS-MCT restart files cp ${run_dir}/*.ins ${run_dir}/run${n} # Copy output control files cp ${output_control_files_dir}/lpjg_cmip6_output.ins ${run_dir}/run${n} ln -s ${output_control_files_dir}/lpjg_cmip6_output.ins ${run_dir}/run${n}/lpjg_cmip6_output.ins.lnk cp ${ini_data_dir}/lpjg/oasismct/ghg*.txt ${run_dir}/run${n} cp ${ini_data_dir}/lpjg/oasismct/${lpjg_res}/ece_gridlist_${lpjg_res}.txt ${run_dir}/run${n}/ece_gridlist.txt ln -s ${ini_data_dir}/lpjg/oasismct/${lpjg_res}/ece_gridlist_${lpjg_res}.txt ${run_dir}/run${n}/ece_gridlist.txt.lnk # Data only needed by master or spinup if [ $n == 1 ] then cp ${ini_data_dir}/lpjg/oasismct/${lpjg_res}/*.nc ${run_dir} fi fi # Refresh output-dirs after they hav been removed at end of the last leg mkdir -p ${run_dir}/run${n}/output/CMIP6 mkdir -p ${run_dir}/run${n}/output/CRESCENDO done if $leg_is_restart then lpjg_restart_dir="restart/lpjg/$(printf %03d $((leg_number-1)))" lpjg_rst_state="${lpjg_restart_dir}/lpjg_state_${leg_start_date_yyyy}" if [ -d "$lpjg_rst_state" ] then ln -sf $lpjg_rst_state else echo "lpjg restart dir $lpjg_rst_state not available" exit -1 fi # get oasis restart files oasis_restart_dir="restart/oasis/$(printf %03d $((leg_number)))" cp -f ${oasis_restart_dir}/*.nc ${run_dir}/ fi # generate lpjg_forcing files if $(has_config lpjg_forcing) then lpjg_gen_forcing $leg_start_date_yyyy $leg_end_date_yyyy_full fi fi # ------------------------------------------------------------------------- # *** Create ICMCL file with vegetation fields # not needed if LPJG is used with feedback # ------------------------------------------------------------------------- if $(has_config osm) then tempfile=tmp.$$ case ${ifs_veg_source} in "ifs" ) # Vegetation from IFS (climatology) icmclfile=${ini_data_dir}/ifs/${ifs_grid}/climate/ICMCL_ONLY_VEG_PD # Create data for december, the year before the leg starts ${grib_set} \ -s dataDate=$(printf "%04d" $((leg_start_date_yyyy-1)))1215,stepRange=0 \ ${icmclfile}-12 ICMCL${exp_name}INIT # Create data for all month in the years of the leg for (( y=${leg_start_date_yyyy} ; y<=${leg_end_date_yyyy_full} ; y++ )) do yy=$(printf "%04d" $y) for m in {1..12} do mm=$(printf "%02d" $m) ${grib_set} -s dataDate=${yy}${mm}15,stepRange=0 ${icmclfile}-${mm} ${tempfile} cat ${tempfile} >> ICMCL${exp_name}INIT done done # Create data for january, the year after the leg ends ${grib_set} \ -s dataDate=$(printf "%04d" $((leg_end_date_yyyy_full+1)))0115,stepRange=0 \ ${icmclfile}-01 ${tempfile} cat ${tempfile} >> ICMCL${exp_name}INIT ;; "era20c"|"cmip6"|"custom_"* ) # Vegetation from a LPJG run (off-line or EC-Earth3-Veg) rm -f ICMCL${exp_name}INIT # Create data for all years of the leg, including one year # before and one year after for (( yr=leg_start_date_yyyy-1 ; yr<=leg_end_date_yyyy_full+1 ; yr+=1 )) do case ${ifs_veg_source} in 'era20c' ) # no scenario needed with era20c icmcl_scenario="" ;; 'custom_'* ) # no scenario implemented yet with custom_dir icmcl_scenario="" ;; 'cmip6' ) # select scenario, use SSP3-7.0 as default # if not otherwise specified icmcl_scenario="historical" if ( [ $ifs_cmip_fixyear -le 0 ] && [ $yr -ge 2015 ] ) || \ [ $ifs_cmip_fixyear -ge 2015 ] then [[ ${ifs_cmip6_scenario} =~ ^SSP ]] \ && icmcl_scenario=${ifs_cmip6_scenario} \ || icmcl_scenario='SSP3-7.0' fi ;; esac if [ $ifs_cmip_fixyear -le 0 ] then cat ${ini_data_dir}/ifs/${ifs_grid}/icmcl_${veg_version}/${icmcl_scenario}/icmcl_$yr.grb >> ICMCL${exp_name}INIT else # Fixed year forcing, requires cdo! # If cdo is not available at runtime you need to fix proper # icmcl files beforehand and use them here cdo setyear,$yr ${ini_data_dir}/ifs/${ifs_grid}/icmcl_${veg_version}/${icmcl_scenario}/icmcl_${ifs_cmip_fixyear}.grb ${tempfile} cat ${tempfile} >> ICMCL${exp_name}INIT fi done ;; "none" ) info "no ICMCL file is created" ;; * ) error "Vegetation from ${ifs_veg_source} not implemented" ;; esac # Clean up rm -f ${tempfile} fi #osm # --------------------------------------------------------------------- # *** Create initial & climatological conditions for osm # --------------------------------------------------------------------- if $(has_config osm) then ## create initial conditions if ! $leg_is_restart then osm_gen_init_clim fi # used to generate daily veg from an IFS experiment to force the OSM osm_icmcl_ifs=false if $osm_icmcl_ifs then if ! [ -r ${osm_forcing_dir}/ICMCL_${leg_start_date_yyyy} ] then osm_gen_icmcl_from_ifs $leg_start_date_yyyy cp ICMCL${exp_name}INIT ${osm_forcing_dir}/ICMCL_${leg_start_date_yyyy} else # or get it from osm_forcing dir mv ICMCL${exp_name}INIT ICMCL${exp_name}INIT_ORI ln -sf ${osm_forcing_dir}/ICMCL_${leg_start_date_yyyy} ICMCL${exp_name}INIT fi fi # generate surfveg for osm simulation osm_gen_veg ## create forcing osm_gen_forcing $leg_start_date_yyyy $leg_end_date_yyyy_full if [[ $leg_start_date_yyyy == $leg_end_date_yyyy_full ]] then ln -sf ${osm_forcing_dir}/${leg_start_date_yyyy}/* . else ln -sf ${osm_forcing_dir}/${leg_start_date_yyyy}-${leg_end_date_yyyy_full}/* . fi ## generate osm namelist . ${ctrl_file_dir}/namelist.osm.sh > input fi # ------------------------------------------------------------------------- # *** Remove some OASIS files of the previous leg # ------------------------------------------------------------------------- # TODO check oasis restart files #if $leg_is_restart #then # rm -f anaisout_* #fi # ------------------------------------------------------------------------- # *** Start the run # ------------------------------------------------------------------------- # Activate OASIS coupling in OSM if necessary has_config osm lpjg && export CPLNG='active' # Use the launch function from the platform configuration file cmd="" if $(has_config lpjg) then [ "${cmd}" != "" ] && cmd=${cmd}" -- " has_config lpjg:spinup && lpjg_spinup="-islpjgspinup" || lpjg_spinup="" cmd=${cmd}"${lpjg_numproc} ${lpjg_exe_file} guess.ins ${lpjg_spinup} -parallel" has_config lpjg_forcing && cmd=${cmd}" -- ${lpjg_forcing_numproc} ${lpjg_forcing_exe_file}" has_config cc_coup && cmd=${cmd}" -- ${ccycle_coupling_numproc} ${ccycle_coupling_exe_file}" fi if $(has_config osm) then # TODO pass this only to the osm process? export OMP_NUM_THREADS=${osm_numproc} ### to do: special "launch" function for OpenMP only run [ "${cmd}" != "" ] && cmd=${cmd}" -- " cmd=${cmd}"1 ${osm_exe_file}" fi t1=$(date +%s) launch $cmd t2=$(date +%s) tr=$(date -d "0 -$t1 sec + $t2 sec" +%T) # ------------------------------------------------------------------------- # *** Check for signs of success # Note the tests provide no guarantee that things went fine! They are # just based on the IFS, NEMO and TM5 log files. More tests (e.g. checking # restart files) could be implemented. # ------------------------------------------------------------------------- # Check for LPJG/OSM success ??? # ------------------------------------------------------------------------- # *** Move LPJ-GUESS output files to archive directory # ------------------------------------------------------------------------- if $(has_config lpjg) then outdir="output/lpjg/$(printf %03d $((leg_number)))" if [ -d ${outdir} ] then rm -rf ${outdir} fi mkdir -p ${outdir} if $(has_config lpjg:ioopt) then for (( n=1; n<=${lpjg_numproc}; n++ )) ; do [ -L ${run_dir}/run${n}/output ] && rm -f ${run_dir}/run${n}/output ; done export lpjg_numproc run_dir run_node 'echo copying lpjg temp dirs from `hostname` ; for (( n=1; n<=${lpjg_numproc}; n++ )) ; do cp -rvf $TMPDIR/lpjg/run${n}/output/ ${run_dir}/run${n} ; done' fi # LPJG run directories # concatenate *.out (or compressed *.out.gz) files from each run* into output dir flist=$(cd ${run_dir}/run1/output && find *.out.gz -type f 2>/dev/null || true) if [ "$flist" = "" ] then lpjg_compress_output=false flist=$(cd ${run_dir}/run1/output && find *.out -type f 2>/dev/null) else lpjg_compress_output=true fi mkdir ${outdir}/CMIP6 for (( n=1; n<=${lpjg_numproc}; n++ )) do for ofile in $flist do if $lpjg_compress_output then [ $n == 1 ] && gzip -c ${run_dir}/run${n}/output/`basename ${ofile} .gz`.hdr > ${outdir}/$ofile cat ${run_dir}/run${n}/output/${ofile} >> ${outdir}/$ofile else if (( n == 1 )) then cat ${run_dir}/run${n}/output/${ofile} > ${outdir}/$ofile else awk '(FNR!=1){print $0}' ${run_dir}/run${n}/output/${ofile} >> ${outdir}/$ofile fi fi done rm -rf ${run_dir}/run${n}/output done # move monthly file if available if [ -f ${run_dir}/LPJ-GUESS_monthlyoutput.txt ] then mv ${run_dir}/LPJ-GUESS_monthlyoutput.txt ${outdir} fi fi # ------------------------------------------------------------------------- # *** Move OSM output files to archive directory # ------------------------------------------------------------------------- if $(has_config osm) then outdir="output/osm/$(printf %03d $((leg_number)))" mkdir -p ${outdir} mv o_*.nc $outdir rm -f Wind.nc Tair.nc Snowf.nc SWdown.nc Rainf.nc Qair.nc PSurf.nc LWdown.nc #------------------------------------------------------------------------ # ** Post-process OSM data #------------------------------------------------------------------------ osm_post_gen_script ${outdir} ${leg_number} # run osm_post script now if post_run is in config has_config osm:post_run && . ${exp_name}_osm_post_${leg_number}.sh fi # ------------------------------------------------------------------------- # *** Move ccycle_coupling output files to archive directory # ------------------------------------------------------------------------- if $(has_config tm5:co2) then outdir="output/ccycle/$(printf %03d $((leg_number)))" mkdir -p ${outdir} rm -f GUE_{CNAT,CANT,CNPP}_*.nc mv TM5_Land{CNAT,CANT,CNPP}_*.nc ${outdir} fi # ------------------------------------------------------------------------- # *** Move LPJ-GUESS restart files to archive directory # ------------------------------------------------------------------------- if $(has_config lpjg) && ! $(has_config lpjg:spinup) then outdir="restart/lpjg/$(printf %03d $((leg_number)))" if [ -d ${outdir} ] then rm -rf ${outdir} fi mkdir -p ${outdir} state_dir="./lpjg_state_$(printf %04d $((leg_end_date_yyyy)))" mv ${state_dir} ${outdir} # LPJG writes into run1 dir, so mv to main rundir mv -f run1/lpjgv.nc . has_config tm5:co2 lpjg && mv -f run1/rlpjg.nc . # remove restart link if $leg_is_restart then old_state_dir="./lpjg_state_$(printf %04d $((leg_start_date_yyyy)))" if [ -L $old_state_dir ] then rm -f "$old_state_dir" fi fi fi # ------------------------------------------------------------------------- # *** Move OSM restart files to archive directory # ------------------------------------------------------------------------- if $(has_config osm) then outdir="restart/osm/$(printf %03d $((leg_number)))" mkdir -p ${outdir} mv restartout.nc $outdir/ rm -f restartin.nc restartin.nc.lnk cp -f $outdir/restartout.nc restartin.nc ln -sf $outdir/restartout.nc restartin.nc.lnk fi # ------------------------------------------------------------------------- # *** Move log files to archive directory # ------------------------------------------------------------------------- outdir="log/$(printf %03d $((leg_number)))" mkdir -p ${outdir} for f in \ guess.log run1/guess0.log \ nout.000000 debug.??.?????? A*_??.nc O*_??.nc \ osm.log input do test -f ${f} && mv ${f} ${outdir} done # ------------------------------------------------------------------------- # *** Copy OASIS restart files to archive directory # NOTE: These files are copied and not moved as they are used in the # next leg! # Note also that the OASIS restart files present at the end of # the leg correspond to the start of the next leg! # ------------------------------------------------------------------------- outdir="restart/oasis/$(printf %03d $((leg_number+1)))" mkdir -p ${outdir} for f in vegin.nc lpjgv.nc do test -f ${f} && cp ${f} ${outdir} done # ------------------------------------------------------------------------- # *** Write the restart control file # ------------------------------------------------------------------------- # Compute CPMIP performance sypd="$(cpmip_sypd $leg_length_sec $(($t2 - $t1)))" ncores=0 has_config lpjg && (( ncores+=${lpjg_numproc} )) || : has_config lpjg_forcing && (( ncores+=${lpjg_forcing_numproc} )) || : has_config osm && (( ncores+=${osm_numproc} )) || : chpsy="$(cpmip_chpsy $leg_length_sec $(($t2 - $t1 +1)) $ncores)" #t_run+1 prevent div zero, just in case echo "#" | tee -a ${ece_info_file} echo "# Finished leg at `date '+%F %T'` after ${tr} (hh:mm:ss)" \ | tee -a ${ece_info_file} echo "# CPMIP performance: $sypd SYPD $chpsy CHPSY"| tee -a ${ece_info_file} echo "leg_number=${leg_number}" | tee -a ${ece_info_file} echo "leg_start_date=\"${leg_start_date}\"" | tee -a ${ece_info_file} echo "leg_end_date=\"${leg_end_date}\"" | tee -a ${ece_info_file} # Need to reset force_run_from_scratch in order to avoid destroying the next leg force_run_from_scratch=false done # loop over legs # ----------------------------------------------------------------------------- # *** Platform dependent finalising of the run # ----------------------------------------------------------------------------- finalise exit 0