#! /bin/sh # ----------------------------------------------------- # init # ----------------------------------------------------- # exit on error set -e # program name and location: call="$0 $*" case $0 in /* ) script=$0 ;; * ) script="`/bin/pwd`/$0" ;; esac bindir=`/usr/bin/dirname ${script}` prog=`basename ${script}` # ----------------------------------------------------- # help # ----------------------------------------------------- DisplayHelp () { ${PAGER:-less} << EOF NAME $prog - setup meteo files for TM5 run USAGE ('runtm5.py' scripting) $prog [-v] [] USAGE ('pycasso' scripting) $prog [-v] DESCRIPTION Install meteo files for all days covering the requested time interval. The horizontal and vertical grids that should be installed are read from the rcfile, as well as the type of files. Daily meteo files are installed in a buffer directory, either as link or unpacked from a monthly tarchive. ARGUMENTS -v Verbose mode; messages about progress are written. Text file with settings, see below. For 'runtm5.py' scripting : Start and end time (day and hour). Forecast start day (ccyymmdd). RCFILE ! ! Meteo files are linked to or unpacked in a buffer directory. ! o the clean flag (T|F) such that files that have not been accessed ! for a long time are removed if a maximum buffer usage is exceeded. ! o specify a maximum size in Mb ! tmm.dir : /scratch/TM5/tmm-buf tmm.dir.clean : T tmm.dir.size : 500 ! ! Meteo files could be setup before the actual program is started. ! First set the main switch to apply this or not. ! If to be applied, specify details for a number of sets; ! each set can be applied by a seperate flag. ! tmm.setup.apply : T ! display progress ? tmm.setup.verbose : T ! file permission mode for newly created files: ! o user archive : 644 ! o archive shared by group : 664 tmm.setup.mode : 664 ! number of sets defined below: tmm.setup.nset : 6 ! ! setup by TMPP produced hdf files: tmm.setup.001.apply : T tmm.setup.001.files : uvsp w spm t q cld sub surf tmm.setup.001.sources : tmpp:class=od;type=fc;levs=ml60;grid=glb3x2;tres=_21p06 tmm.setup.001.search : ecfs:/nlh/TM/meteo ! ! setup by TM5 produced hdf files: tmm.setup.002.apply : T tmm.setup.002.files : mfuv mfw sp t q cld sub tmm.setup.002.sources : tm5:mdir=era2003-ml60-glb3x2;tres=_fc012up2tr3 tmm.setup.002.search : ecfs:/nl5/gems/meteo ! ! extracted from MARS archive tmm.setup.003.apply : T tmm.setup.003.files : T/Q/W/CLWC/CIWC/CC tmm.setup.003.sources : ecmwf-mars:class=od;type=fc;levs=${LEVS};grid=N80 tmm.setup.003.search : ! ! extracted from MARS archive: MACC fire emissions tmm.setup.003.apply : T tmm.setup.003.files : CH4FIRE tmm.setup.003.sources : ecmwf-mars:class=rd;type=fc;levs=sfc;grid=N80;tres=_fc000up24tr1 tmm.setup.003.search : ! ! NCEP re-analysis data (cdc) tmm.setup.004.apply : T tmm.setup.004.files : pres.nlog vort div vair shum hgt land sfcr pres icec skt uwnd.10m vwnd.10m lhtfl shtfl ugwd vgwd prate cprat tmm.setup.004.sources : ncep-cdc: tmm.setup.004.search : /co2/data/ncep.reanalysis ! ! NCEP operational data (gfs) tmm.setup.005.apply : T tmm.setup.005.files : SF SFLUXGrbF tmm.setup.005.sources : ncep-gfs: tmm.setup.005.search : /co2/data/ncep.gfs ! ! Olsson surface roughness files tmm.setup.006.apply : T tmm.setup.006.files : SR tmm.setup.006.sources : olsson:mdir=OLSSON tmm.setup.006.search : ecfs:/nl5/MARS EOF exit 0 } # err 'help text' err () { echo "$1" 1>&2 } # errit errit () { err "$prog - ERROR - in call : ${call}" err "$prog - ERROR - use '$prog --help' for more info." exit $1 } # ----------------------------------------------------- # arguments # ----------------------------------------------------- rcfile='' ccyymmddhh1='' ccyymmddhh2='' fcday0='' verbose='' # extract settings for arg in "$@" ; do case ${arg} in -h | --help ) DisplayHelp ;; -* ) # option arguments options="${options} ${arg}" case ${arg} in -v | --verbose ) verbose='true' ;; esac ;; * ) # non-option arguments if [ -z "${rcfile}" ]; then rcfile=${arg} elif [ -z "${ccyymmddhh1}" ]; then ccyymmddhh1=${arg} elif [ -z "${ccyymmddhh2}" ]; then ccyymmddhh2=${arg} elif [ -z "${fcday0}" ]; then fcday0=${arg} else err "$prog - ERROR - unknown argument : ${arg}" errit 1 fi ;; esac done # not complete ? if [ -z "${rcfile}" ]; then err "$prog - ERROR - missing arguments" errit 1 fi # full path: case ${rcfile} in /* ) rcfile=${rcfile} ;; * ) rcfile="`/bin/pwd`/${rcfile}" ;; esac # ----------------------------------------------------- # settings # ----------------------------------------------------- # info ... echo "$prog - start" # should meteo files be setup ? tmm_setup_apply=`${bindir}/go_readrc ${rcfile} 'tmm.setup.apply' 'F'` # if not ... if [ "${tmm_setup_apply}" != "T" ] ; then # info ... echo "$prog - flag to setup meteo files is false; return" # return without problems ... exit 0 fi # time range not specified yet ? if [ -z "${ccyymmddhh1}${ccyymmddhh2}" ]; then # time range start and end in "ccyy-mm-dd hh:mn:ss' format ; use up to hours only ccyymmddhh1=`${bindir}/go_readrc $rcfile 'jobstep.timerange.start' | cut -d ':' -f 1 | tr -d '\-/ '` ccyymmddhh2=`${bindir}/go_readrc $rcfile 'jobstep.timerange.end' | cut -d ':' -f 1 | tr -d '\-/ '` fi # no forecast day specified yet ? then try to read from rcfile (pycasso scripting): if [ -z "${fcday0}" ]; then # optional forecast day in "ccyy-mm-dd" format; might be missing .. fcday0=`${bindir}/go_readrc $rcfile 'time.fc.day0' 'None'` # None specified ? if [ "${fcday0}" == "None" ]; then # set empty: fcday0='' else # convert to ccyymmdd format: fcday0=`echo ${fcday0} | tr -d '\-/'` fi fi # fcmode set in rc file (pls) fcmode_read=`${bindir}/go_readrc $rcfile 'time.fc' 'F'` tmm_dir=`${bindir}/go_readrc $rcfile 'tmm.dir'` tmm_dir_clean=`${bindir}/go_readrc $rcfile 'tmm.dir.clean'` tmm_dir_size=`${bindir}/go_readrc $rcfile 'tmm.dir.size'` verboseTF=`${bindir}/go_readrc $rcfile 'tmm.setup.verbose'` nset=`${bindir}/go_readrc $rcfile 'tmm.setup.nset'` # file creation mode: tmm_mode=`${bindir}/go_readrc $rcfile 'tmm.setup.mode' '644'` # file transfer mode: transfermode=`${bindir}/go_readrc $rcfile 'tmm.setup.transfermode' 'link -s -f'` # ----------------------------------------------------- # begin # ----------------------------------------------------- test "${verbose}" -o "${verboseTF}" = "T" && verbose='--verbose' test ${verbose} && echo "$prog - Start at `date`" # info if [ ${verbose} ]; then echo "$prog - " echo "$prog - time range : ${ccyymmddhh1} ${ccyymmddhh2}" echo "$prog - fc day 0 : ${fcday0}" fi # create buffer if not present yet: test ! -d ${tmm_dir} && /bin/mkdir -p ${tmm_dir} # change to buffer; store current directory: owd=`/bin/pwd` cd ${tmm_dir} echo "$prog - " echo "$prog - changed to : ${tmm_dir}" # create target directory if necessary: test ! -d ${tmm_dir} && /bin/mkdir -p ${tmm_dir} # ************************************* # step 1 : pre # ************************************* # initiate mars requests mars_jb='./mars.jb' /bin/rm -f ${mars_jb} # ************************************* # step 2 : loop over sets # ************************************* iset=0 while true ; do # next set iset=`expr ${iset} + 1` # leave ? test ${iset} -gt ${nset} && break # 001, 002, etc sset=`/usr/bin/printf '%3.3i' ${iset}` # skip ? apply=`${bindir}/go_readrc $rcfile "tmm.setup.${sset}.apply"` test "${apply}" != "T" && continue # set specific variables: mfiles=`${bindir}/go_readrc $rcfile "tmm.setup.${sset}.files"` msources=`${bindir}/go_readrc $rcfile "tmm.setup.${sset}.sources"` search_locs=`${bindir}/go_readrc $rcfile "tmm.setup.${sset}.search"` # skip ? test "${apply}" != "T" && continue # info if [ ${verbose} ]; then echo "$prog - " echo "$prog - sources : ${msources}" echo "$prog - files : ${mfiles}" fi # loop over files for mfile in ${mfiles} ; do # loop over file sources for msource in ${msources} ; do # reset dates to init values; # days might be changed in order to cover selected time range # first day ccyymmdd1=`echo ${ccyymmddhh1} | /usr/bin/cut -c 1-8` ccyy1=`echo ${ccyymmddhh1} | /usr/bin/cut -c 1-4` mm1=`echo ${ccyymmddhh1} | /usr/bin/cut -c 5-6` dd1=`echo ${ccyymmddhh1} | /usr/bin/cut -c 7-8` hh1=`echo ${ccyymmddhh1} | /usr/bin/cut -c 9-10` # last day ccyymmdd2=`echo ${ccyymmddhh2} | /usr/bin/cut -c 1-8` ccyy2=`echo ${ccyymmddhh2} | /usr/bin/cut -c 1-4` mm2=`echo ${ccyymmddhh2} | /usr/bin/cut -c 5-6` dd2=`echo ${ccyymmddhh2} | /usr/bin/cut -c 7-8` hh2=`echo ${ccyymmddhh2} | /usr/bin/cut -c 9-10` # setup files given source key: case ${msource} in # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ tmpp:* ) # TMPP hdf files # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # tmpp files: valid for [21,21] test ${hh1} -ge 21 && ccyymmdd1=`${bindir}/date_add ${ccyymmdd1} 1` test ${hh2} -ge 21 && ccyymmdd2=`${bindir}/date_add ${ccyymmdd2} 1` # loop over days ccyymmdd=${ccyymmdd1} while [ ${ccyymmdd} -le ${ccyymmdd2} ]; do # split current day: ccyy=`echo ${ccyymmdd} | /usr/bin/cut -c 1-4` mm=`echo ${ccyymmdd} | /usr/bin/cut -c 5-6` dd=`echo ${ccyymmdd} | /usr/bin/cut -c 7-8` # # define filename etc # # class=od;type=fc;levs=ml60;grid=glb3x2;tres=_21p06 # default keys: class='od' type='fc' levs='ml60' grid='glb3x2' tres='_21p06' # split in 'tm-hdf' and keys : keys=`echo ${msource} | /usr/bin/cut -d ':' -f 2-` # evaluate keys : form=tm5; mdir=od-fc-etc; tres=_21p06 eval ${keys} # actual level key test "${mfile}" = "spm" && levs='ml1' test "${mfile}" = "surf" && levs='sfc' # actual grid key test "${mfile}" = "surf" && grid='glb1x1' # time resolution: test "${mfile}" = "spm" && tres='_00p06' test "${mfile}" = "surf" && tres='_21p03' # directory: mdir="${class}/${type}/${ccyy}/${mm}/${levs}/${grid}" # directory and file names: meteodir=`echo ${mdir} | /usr/bin/tr '-' '/'` meteofile="${mfile}_${ccyymmdd}${tres}.hdf" # try tar files .. tarfile="${mfile}_${ccyy}${mm}${tres}.tar" trkfile="${mfile}.tar" # expand directory: meteodir_long=`echo ${meteodir} | /usr/bin/tr '/' '-'` # expand day file: meteofile_full="${meteodir}/${meteofile}" meteofile_long="${meteodir_long}-${meteofile}" # expand tar file: tarfile_full="${meteodir}/${tarfile}" tarfile_long="${meteodir_long}-${tarfile}" trkfile_full="${meteodir}/${trkfile}" trkfile_long="${meteodir_long}-${trkfile}" # not found yet ... found='' # # present in buffer ? # # not found yet ? if [ -z "${found}" ]; then fname=./${meteofile_long} test ${verbose} && echo "$prog - try ${fname}" if [ -f ${fname} ]; then test ${verbose} && echo "$prog - found ..." test -w ${fname} && touch ${fname} test -O ${fname} && chmod ${tmm_mode} ${fname} found='true' fi fi # not found yet ... # # search locations # # not found yet ? if [ -z "${found}" ]; then # loop over search locations for loc in ${search_locs} ; do # *** full file fname=${loc}/${meteofile_full} test ${verbose} && echo "$prog - try ${fname} ..." if ${bindir}/gss exist ${fname} ; then # create link in buffer: test ${verbose} && echo "$prog - link ..." bname=./${meteofile_long} ${bindir}/gss ${transfermode} ${fname} ${bname} # found ! test -w ${bname} && touch ${bname} test -O ${bname} && chmod ${tmm_mode} ${bname} found='true' break fi # *** long file fname=${loc}/${meteofile_long} test ${verbose} && echo "$prog - try ${fname} ..." if ${bindir}/gss exist ${fname} ; then # create link in buffer: test ${verbose} && echo "$prog - link ..." bname=./${meteofile_long} ${bindir}/gss ${transfermode} ${fname} ${bname} # found ! test -w ${bname} && touch ${bname} test -O ${bname} && chmod ${tmm_mode} ${bname} found='true' break fi # *** tar file fname=${loc}/${tarfile_full} test ${verbose} && echo "$prog - try ${fname} ..." if ${bindir}/gss exist ${fname} ; then # create link in buffer: test ${verbose} && echo "$prog - link ..." ${bindir}/gss ${transfermode} ${fname} ${tarfile_long} # extract, rename: test ${verbose} && echo "$prog - unpack ..." # create temporary unpacking dir: ndir=0 while /bin/true ; do ndir=`/usr/bin/expr ${ndir} + 1` tmpdir="./tmp.${ndir}" test ! -d ${tmpdir} && break done /bin/mkdir -p ${tmpdir} cd ${tmpdir} # unpack ${bindir}/gss tar-extract ../${tarfile_long} # move and rename: for mf in `/bin/ls *.hdf` ; do /bin/mv ${mf} ../${meteodir_long}-${mf} done # remove tmpdir cd .. /bin/rm -r -f ${tmpdir} # remove linked tar file: ${bindir}/gss unlink ${tarfile_long} # check ... bname=./${meteofile_long} if [ ! -f ${bname} ]; then err "$prog - ERROR - meteofile not found after unpacking tarfile:" err "$prog - ERROR - pwd : `/bin/pwd`" err "$prog - ERROR - tarfile : ${tarfile_long}" err "$prog - ERROR - meteofile : ${bname}" errit 1 fi # found! test -w ./${meteofile_long} && touch ./${meteofile_long} test -O ${meteofile_long} && chmod ${tmm_mode} ${meteofile_long} for file2 in *${class}*${type}*${ccyy}*${mm}*${levs}*${grid}* ; do test -O ${file2} && chmod ${tmm_mode} ${file2} done found='true' break fi # *** Krolse tar file fname=${loc}/${trkfile_full} test ${verbose} && echo "$prog - try ${fname} ..." if ${bindir}/gss exist ${fname} ; then # create link in buffer: test ${verbose} && echo "$prog - link ..." ${bindir}/gss ${transfermode} ${fname} ${trkfile_long} # extract, rename: test ${verbose} && echo "$prog - unpack ..." # create temporary unpacking dir: ndir=0 while /bin/true ; do ndir=`/usr/bin/expr ${ndir} + 1` tmpdir="./tmp.${ndir}" test ! -d ${tmpdir} && break done /bin/mkdir -p ${tmpdir} cd ${tmpdir} # unpack ${bindir}/gss tar-extract ../${trkfile_long} # move and rename: for mf in `/bin/ls *.hdf` ; do /bin/mv ${mf} ../${meteodir_long}-${mf} done # remove tmpdir cd .. /bin/rm -r -f ${tmpdir} # remove linked tar file: ${bindir}/gss unlink ${trkfile_long} # found! test -w ./${meteofile_long} && touch ./${meteofile_long} test -O ${meteofile_long} && chmod ${tmm_mode} ${meteofile_long} for file2 in *${class}*${type}*${ccyy}*${mm}*${levs}*${grid}* ; do test -O ${file2} && chmod ${tmm_mode} ${file2} done found='true' break fi done # loop over locations fi # not found yet ... # not found: error ... if [ -z "${found}" ]; then err "$prog - ERROR : do not know how to retrieve meteo file:" err "$prog - ERROR : directory : `/bin/pwd`" err "$prog - ERROR : meteofile : ${meteofile_long}" errit 1 fi # next day: ccyymmdd=`${bindir}/date_add ${ccyymmdd} 1` done # loop over days ;; # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ tm5:* | tm5-hdf:* | tm5-nc:* ) # TM5 produced files # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # split in 'tm5' and keys : keys=`echo ${msource} | /usr/bin/cut -d ':' -f 2-` # evaluate keys : form=tm5; mdir=od-fc-etc; tres=_21p06 # surround values with quotes to handle '<' and other strange characters for keyval in `echo ${keys} | tr ';' ' '` ; do key=`echo ${keyval} | cut -d '=' -f 1` val=`echo ${keyval} | cut -d '=' -f 2` eval "${key}='${val}'" done # file extension: case "${msource}" in tm5:* | tm5-hdf:* ) mext='hdf' ;; tm5-nc:* ) mext='nc' ;; * ) err "$prog - ERROR - could not set meteo file extension for source : ${msource}" ; errit 1 ;; esac # extend or limit day range ? case ${mfile} in # surface pressure is always interpolated in time, # thus 00:00 of next day required if end hour is after 18:00 or 21:00 'sp' ) case ${tres} in '_00p06' ) test ${hh2} -gt 18 && ccyymmdd2=`${bindir}/date_add ${ccyymmdd2} 1` ;; '_00p03' ) test ${hh2} -gt 21 && ccyymmdd2=`${bindir}/date_add ${ccyymmdd2} 1` ;; esac ;; # if an surface fields are kept 6 hourly constant around 00/06/12/18, # next day is required for end times after 21:00 'albedo' | 'sr' | 'veg' ) case ${tres} in '_00p06' ) test ${hh2} -gt 18 && ccyymmdd2=`${bindir}/date_add ${ccyymmdd2} 1` ;; esac ;; # midnight is included in accumulated fields, so one day less if ends at 00:00 : 'cp' | 'lsp' | 'slhf' | 'sshf' | 'ssr' | 'ewss' | 'nsss' ) test ${hh2} -eq 0 && ccyymmdd2=`${bindir}/date_add ${ccyymmdd2} -1` ;; esac # loop over times (days?) ccyymmdd=${ccyymmdd1} while [ ${ccyymmdd} -le ${ccyymmdd2} ]; do # split current day: ccyy=`echo ${ccyymmdd} | /usr/bin/cut -c 1-4` mm=`echo ${ccyymmdd} | /usr/bin/cut -c 5-6` dd=`echo ${ccyymmdd} | /usr/bin/cut -c 7-8` # longer day parts for tar files: ccyymm=`echo ${ccyymmdd} | /usr/bin/cut -c 1-6` ccyymmd=`echo ${ccyymmdd} | /usr/bin/cut -c 1-7` d=`echo ${ccyymmdd} | /usr/bin/cut -c 8` # # define filename etc # # trap change from fg to fc data: if [ "${tres}" = "_fg006up4tr3" ] && [ ${ccyymmdd} -ge 20000912 ] ; then tres='_fc012up2tr3' fi # directory and file names: meteodir=`echo ${mdir} | /usr/bin/tr '-' '/'` meteofile="${mfile}_${ccyymmdd}${tres}.${mext}" # extension of tarfiles : tarext='tar' # try tar files .. # t_200102_00p03.tar : month file # t_200102?_00p03.tar : 10 days (?0,?1,..,?9) # t_200102?0_00p03.tar : first five days (?0,?1,?2,?3,?4) # t_200102?5_00p03.tar : last five days (?5,?6,?7,?8,?9) tarfiles="${mfile}_${ccyymm}${tres}.${tarext}" tarfiles="${tarfiles} ${mfile}_${ccyymmd}${tres}.${tarext}" if [ ${d} -lt 5 ]; then tarfiles="${tarfiles} ${mfile}_${ccyymmd}0${tres}.${tarext}" else tarfiles="${tarfiles} ${mfile}_${ccyymmd}5${tres}.${tarext}" fi # specials ... case ${mfile} in 'oro' | 'lsm' ) meteofile="${mfile}.${mext}" tarfiles="${mfile}.tar" mdir_yyyy='0000' mdir_mm='00' ;; 'srols' ) meteofile="${mfile}_${ccyy}${mm}.${mext}" tarfiles="${mfile}_${ccyy}${mm}.${tarext}" mdir_yyyy="${ccyy}" mdir_mm="${mm}" ;; * ) mdir_yyyy="${ccyy}" mdir_mm="${mm}" ;; esac # replace keys in directory name: meteodir=`echo ${meteodir} | sed -e "s//${mdir_yyyy}/"` meteodir=`echo ${meteodir} | sed -e "s//${mdir_mm}/"` # expand directory: meteodir_long=`echo ${meteodir} | /usr/bin/tr '/' '-'` # expand day file: meteofile_full="${meteodir}/${meteofile}" meteofile_long="${meteodir_long}-${meteofile}" # not found yet ... found='' # # present in buffer ? # # not found yet ? if [ -z "${found}" ]; then fname=./${meteofile_long} test ${verbose} && echo "$prog - try ${fname}" if [ -f ${fname} ]; then test ${verbose} && echo "$prog - found ..." test -w ${fname} && touch ${fname} found='true' fi fi # not found yet ... # # search locations # # not found yet ? if [ -z "${found}" ]; then # loop over search locations for loc in ${search_locs} ; do # *** full file fname=${loc}/${meteofile_full} test ${verbose} && echo "$prog - try full ${fname} ..." if ${bindir}/gss exist ${fname} ; then # create link in buffer: test ${verbose} && echo "$prog - ${transfermode} ..." bname=./${meteofile_long} ${bindir}/gss ${transfermode} ${fname} ${bname} # found ! test -w ${bname} && touch ${bname} found='true' break fi # *** long file fname=${loc}/${meteofile_long} test ${verbose} && echo "$prog - try long ${fname} ..." if ${bindir}/gss exist ${fname} ; then # create link in buffer: test ${verbose} && echo "$prog - ${transfermode} ..." bname=./${meteofile_long} ${bindir}/gss ${transfermode} ${fname} ${bname} # found ! test -w ${bname} && touch ${bname} found='true' break fi # *** tar file # loop over possible tar files: for tarfile in ${tarfiles} ; do # expand tar file: tarfile_full="${meteodir}/${tarfile}" tarfile_long="${meteodir_long}-${tarfile}" # file in archive could have full path or long name: fnames="${loc}/${tarfile_full} ${loc}/${tarfile_long}" # loop over possible archived files: for fname in ${fnames} ; do test ${verbose} && echo "$prog - try tar ${fname} ..." if ${bindir}/gss exist ${fname} ; then # create link in buffer: test ${verbose} && echo "$prog - link ..." ${bindir}/gss ${transfermode} ${fname} ${tarfile_long} # extract; tar file contains long names: test ${verbose} && echo "$prog - unpack ..." ${bindir}/gss tar-extract ${tarfile_long} # unzip ? test -f "${meteofile_long}.gz" && gunzip -f *.gz test -f "${meteofile_long}.bz2" && bunzip2 -f *.bz2 # check ... bname=./${meteofile_long} if [ ! -f ${bname} ]; then err "$prog - ERROR - meteofile not found after unpacking tarfile:" err "$prog - ERROR - pwd : `/bin/pwd`" err "$prog - ERROR - tarfile : ${tarfile_long}" err "$prog - ERROR - meteofile : ${bname}" errit 1 fi # remove linked tar file: ${bindir}/gss unlink ${tarfile_long} # found! test -w ${bname} && touch ${bname} test -O ${bname} && chmod ${tmm_mode} ${bname} for file2 in *${mfile}_${ccyymm}* ; do test -O ${file2} && chmod ${tmm_mode} ${file2} done found='true' break fi # found ? test "${found}" && break done # full path or long name in archive # found ? test "${found}" && break done # loop over tar files # found ? test "${found}" && break done # loop over locations fi # not found yet ... # not found: error ... if [ -z "${found}" ]; then err "$prog - ERROR : do not know how to retrieve meteo file:" err "$prog - ERROR : directory : `/bin/pwd`" err "$prog - ERROR : meteofile : ${meteofile_long}" errit 1 fi # next time case ${mfile} in # constant files 'oro' | 'lsm' ) # exit time loop break ;; # month files 'srols' ) # next month (don't mind about day part, is not used for this files) # WP: replaced by one day at a time because adding 31 days could bring # WP: you to exceed the enddate, and then the next month file would not # WP: be retrieved. #ccyymmdd=`${bindir}/date_add ${ccyy}${mm}01 31` ccyymmdd=`${bindir}/date_add ${ccyy}${mm}${dd} 1` ;; # day files * ) # next day: ccyymmdd=`${bindir}/date_add ${ccyymmdd} 1` ;; esac done # time loop ;; # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ecmwf:* ) # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # valid for (00,24] : # o if start at 00:00, retrieve file for previous day: test ${hh1} -eq 00 && ccyymmdd1=`${bindir}/date_add ${ccyymmdd1} -1` # loop over days ccyymmdd=${ccyymmdd1} while [ ${ccyymmdd} -le ${ccyymmdd2} ]; do # split current day: ccyy=`echo ${ccyymmdd} | /usr/bin/cut -c 1-4` mm=`echo ${ccyymmdd} | /usr/bin/cut -c 5-6` dd=`echo ${ccyymmdd} | /usr/bin/cut -c 7-8` # # define filename etc # # split in 'grib' and keys : keys=`echo ${msource} | /usr/bin/cut -d ':' -f 2-` # evaluate keys : class=od;type=fc;levs=ml60;sh=T159 class='class' type='type' levs='ml60' sh=0 gg=0 tres='_tres' eval ${keys} # actual type case ${mfile} in S0 ) type='an' ;; esac # set level key case ${mfile} in S1 | S0 | Z | LSM ) levs='sfc' ;; LNSP ) levs='ml1' ;; esac # actual grid case ${mfile} in Q | CLD | S1 | S0 | Z | LSM ) grid="N${gg}" ;; * ) grid="T${sh}" ;; esac # directory and file names: case ${mfile} in Z | LSM ) meteodir="${class}/${type}/0000/00/${levs}/${grid}" meteofile="${mfile}.gb" ;; * ) meteodir="${class}/${type}/${ccyy}/${mm}/${levs}/${grid}" meteofile="${mfile}_${ccyy}${mm}${dd}${tres}.gb" ;; esac # expand directory: meteodir_long=`echo ${meteodir} | /usr/bin/tr '/' '-'` # expand day file: meteofile_full="${meteodir}/${meteofile}" meteofile_long="${meteodir_long}-${meteofile}" # not found yet ... found='' # # present in buffer ? # # not found yet ? if [ -z "${found}" ]; then fname=./${meteofile_long} test ${verbose} && echo "$prog - try ${fname}" if [ -f ${fname} ]; then test ${verbose} && echo "$prog - found ..." test -w ${fname} && touch ${fname} found='true' fi fi # not found yet ... # # search locations # # not found yet ? if [ -z "${found}" ]; then # loop over search locations for loc in ${search_locs} ; do # *** full file fname=${loc}/${meteofile_full} test ${verbose} && echo "$prog - try ${fname} ..." if ${bindir}/gss exist ${fname} ; then # create link in buffer: test ${verbose} && echo "$prog - link ..." bname=./${meteofile_long} ${bindir}/gss ${transfermode} ${fname} ${bname} # found ! found='true' break fi # *** long file fname=${loc}/${meteofile_long} test ${verbose} && echo "$prog - try ${fname} ..." if ${bindir}/gss exist ${fname} ; then # create link in buffer: test ${verbose} && echo "$prog - link ..." bname=./${meteofile_long} ${bindir}/gss ${transfermode} ${fname} ${bname} # found ! found='true' break fi done # loop over locations fi # not found yet ... # not found: error ... if [ -z "${found}" ]; then err "$prog - ERROR : do not know how to retrieve meteo file:" err "$prog - ERROR : directory : `/bin/pwd`" err "$prog - ERROR : meteofile : ${meteofile_long}" errit 1 fi # next day: ccyymmdd=`${bindir}/date_add ${ccyymmdd} 1` done # loop over days ;; # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ecmwf-mars:* | ecmwf-tmpp:* | ecmwf-tm5:* ) # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # split in 'mars' and keys : keys=`echo ${msource} | /usr/bin/cut -d ':' -f 2-` # evaluate keys : class=od;type=fc;nlev=60;grid=T159 class='class' stream='oper' expver=1 type='type' levs='ml60' grid='X00' eval ${keys} # start at 00:00 ? if [ ${hh1} -eq 00 ]; then # fc files will be retrieved for (00,24] or (00,27] # and therefore include the final midnight already; # therefor start with the day before: ccyymmdd1=`${bindir}/date_add ${ccyymmdd1} -1` fi #------------------------ # 24 Feb 2011 - P. Le Sager - Commented the following block for # EI preprocessing. See ecmwf:* case above. Note that, for "ei-an", we # substract another day, with dates=`${bindir}/date_add ${fcday00} # -1` further down in the script. # 13 Apr 2011 - P. Le Sager - Commented also for OD pre-processing to # work. if [ "${class}" != 'ei' -a "${class}" != 'od' ]; then # end at 00:00 ? if [ ${hh2} -eq 00 ]; then # fc files will be retrieved for (00,24] or (00,27] # and therefore include the final midnight already; # therefor end with the day before: ccyymmdd2=`${bindir}/date_add ${ccyymmdd2} -1` fi fi #------------------------ ## forecast mode ? then single day only: #if [ "${fcday0}" ]; then # ccyymmdd_end=${ccyymmdd1} #else # ccyymmdd_end=${ccyymmdd2} #fi # loop over days ccyymmdd=${ccyymmdd1} fcmode='' while [ ${ccyymmdd} -le ${ccyymmdd2} ]; do # forecast series only once .. test "${fcmode}" && break # forecast mode ? if [ "${fcday0}" -a "${fcmode_read}" == "T" ]; then # older forecast ? if [ ${ccyymmdd} -lt ${fcday0} ]; then fcday00=${ccyymmdd} else fcday00=${fcday0} fcmode='true' fi else # dummy fcday00=${ccyymmdd} fi # re-evaluate keys; some might have been changed for previous day ... eval ${keys} # adhoc reset of experiment version: test "${expver}" == "29" -a ${ccyymmdd} -gt 20060201 && expver='1' # # od, since 20040628 ? # --------------------- # # ][ forecast day ][ ][ ][ ] # 12 15 18 21 00 03 06 09 12 15 18 21 00 03 06 09 12 15 18 21 00 03 06 09 12 15 18 21 00 03 06 09 12 15 18 21 0 # # an an an # fc 00:00 + 00 03 06 09 12 15 18 21 24 27 30 33 36 39 42 45 48 51 54 57 60 63 66 69 72 78 84 90 96 ... 216 222 228 234 240 # -- -- -- -- -- # an an an # fc 12:00 + 00 03 06 09 12 15 18 21 24 27 30 33 36 39 42 45 48 51 54 57 60 63 66 69 72 78 84 ... 204 210 216 222 228 234 240 # -- -- -- -- -- == == == == == == == == ++ ++ ++ ++ ++ ++ ++ ++ # # # "--" means "selected for day 0" ; "==" means "selected for day 1" ; "++" means "selected for day 2" ; # # # era2003 (expver=500) # -------------------- # # 2002-10-31 ][ 2002-11-01 ][ 2002-11-02 ] # 12 15 18 21 00 03 06 09 12 15 18 21 00 03 06 09 12 15 18 21 00 # # an an an # fc 12:00 + 03 06 09 12 # # an an an # fc 00:00 + 03 06 09 12 15 18 21 24 30 36 # # an an an # fc 12:00 + 03 06 09 12 15 18 21 24 30 36 # # --- # 1) experimenatal data such as e4 convective fields # are available for fc steps 3/6/9/12 only # # # # ml91 setup (expver=29) # ---------------------- # # 2005-09-30 ][ 2005-10-01 ][ 2005-10-02 ] # 03 06 09 12 15 18 21 00 03 06 09 12 15 18 21 00 03 06 09 12 15 18 21 00 # # 4v [----------] # an an an # fc 12:00 + 00 03 06 09 12 15 18 21 24 ... # # 4v [----------] # an an an # fc 00:00 + 00 03 06 09 12 15 18 21 24 ... # # 4v [----------] # an an an # fc 12:00 + 00 03 06 09 12 15 18 21 24 ... # # 4v [----------] # an an an # fc 00:00 + 00 03 06 09 12 15 18 21 24 ... # # [--first-complete-day--] # # Last forecast for expver 29 : 2006-02-01 00:00 # # From 2006-03-14 : 3 hourly upto 96 # # General notes # ------------- # # o 4v data is often chaninging, use an instead # o SP available for od as fc-sfc, but for e4 only as an-sfc .. # # date, time and steps case ${class}-${type} in # ~~~~~~~~~~~~~~~~~~ 'od-an' | 'e4-an' | 'ei-an' | 'rd-an' ) # ~~~~~~~~~~~~~~~~~~ # analysed fields are (at least) 24 hours old, # since not available in forecast mode: dates=`${bindir}/date_add ${fcday00} -1` # ensure that all fields within [00,24] are available (from previous day!); # therefore the time 2400 is added which is treated specially below: times="0000/0600/1200/1800 2400" # an fields always have step 000 : steps="000 000" # specify the experiment version for each times sequence: expvers="${expver} ${expver}" # # adhoc for ml91 test suit: if [ "${class}" = "od" -a ${expver} -eq 29 ]; then if [ ${dates} -eq 20050930 ] ; then times="0600/1200/1800 2400" fi fi ;; # ~~~~~~~~~~~~~~~~~~ 'ea-an' ) # ~~~~~~~~~~~~~~~~~~ # ERA5: retrieve hourly surface fields # re-analysis, use curent day: dates="${ccyymmdd}" # hourly from midnight onwards: times="0000/0100/0200/0300/0400/0500/0600/0700/0800/0900/1000/1100/1200/1300/1400/1500/1600/1700/1800/1900/2000/2100/2200/2300" # an fields always have step 000 : steps="000" # specify the experiment version for each times sequence: expvers="${expver}" ;; # ~~~~~~~~~~~~~~~~~~ 'od-fc' | 'e4-fc' | 'ei-fc' ) # ~~~~~~~~~~~~~~~~~~ # forecast mode ? if [ "${fcmode}" ]; then # in forecast mode, the date is when the forecast started: dates="${fcday00}" # determine forecast periode range 0 .. 10 fcp1=`${bindir}/date_diff ${fcday00} ${ccyymmdd}` fcp2=`${bindir}/date_diff ${fcday00} ${ccyymmdd2}` if [ ${fcp1} -lt 0 -o ${fcp1} -gt 10 -o ${fcp2} -lt 0 -o ${fcp2} -gt 10 ]; then err "$prog - ERROR - found obscure forecast period range: ${fcp1} ${fcp2}" err "$prog - ERROR - fcday0 : ${fcday00}" err "$prog - ERROR - ccyymmdd : ${ccyymmdd}" err "$prog - ERROR - ccyymmdd2 : ${ccyymmdd2}" errit 1 fi else # forecast day '0' dates="${ccyymmdd}" # no forecast perode range: fcp1=0 fcp2=0 fi # fc periode '0' uses the 00:00 and 12:00 forecast, # fc periode 1, 2, ... use the 12:00 forecast only if [ ${fcp1} -eq 0 -a ${fcp2} -eq 0 ] ; then # single mars request for first day: times="0000/1200" steps="003/006/009/012" expvers="${expver}" # for surface fields, also read step 015 since we need # accumulated surface heat fluxes for convection for the # 3-hourly intervals around 12:00 and 24:00: test "${levs}" = "sfc" && steps="003/006/009/012/015" # adhoc for ml91 test suit: if [ "${class}" = "od" -a ${expver} -eq 29 ]; then if [ ${ccyymmdd} -eq 20050930 ] ; then times="1200" steps="003/006/009/012" test "${levs}" = "sfc" && steps="003/006/009/012/015" fi if [ ${ccyymmdd} -eq 20060201 ] ; then times="0000 1200" steps="003/006/009/012 003/006/009/012/" test "${levs}" = "sfc" && steps="003/006/009/012/015 003/006/009/012/015" expvers="29 1" fi fi # adhoc for era2003: if [ "${class}" = "e4" -a ${expver} -eq 500 ]; then # experimental fields up to step 12 only: steps="003/006/009/012" fi else # multiple forecast days, or one forecast day after day 0 # retrieve day 0 ? if [ ${fcp1} -eq 0 ] ; then # two mars requests needed: # (also step 015 for accum fields around 12:00 and 24:00) times="0000 1200" steps="003/006/009/012/015 003/006/009/012/015" # now # steps="003/006/009/012 003/006/009/012/015" # test "${levs}" = "sfc" && steps="003/006/009/012/015 003/006/009/012/015" expvers="${expver} ${expver}" # adhoc ... test ${expver} -eq 29 -a ${ccyymmdd} -eq 20060201 && expvers="29 1" else # one mars request for 12:00 forecast only: times="1200" steps="" expvers="${expver}" # adhoc ... test ${expver} -eq 29 -a ${ccyymmdd} -eq 20060201 && expvers="1" fi # add other steps; for fcp 0 already filled above ... fcp=${fcp1} test ${fcp} -eq 0 && fcp=1 while [ ${fcp} -le ${fcp2} ]; do # add seperation / if necessary: test "${steps}" && steps="${steps}/" # add steps (015 already added above) if [ ${ccyymmdd} -lt 20060314 ]; then case ${fcp} in 1 ) steps="${steps}018/021/024/027/030/033/036" ;; 2 ) steps="${steps}039/042/045/048/051/054/057/060" ;; 3 ) steps="${steps}063/066/069/072/078/084" ;; 4 ) steps="${steps}090/096/102/108" ;; 5 ) steps="${steps}114/120/126/132" ;; 6 ) steps="${steps}138/144/150/156" ;; 7 ) steps="${steps}162/168/174/180" ;; 8 ) steps="${steps}186/192/198/204" ;; 9 ) steps="${steps}210/216/222/228" ;; 10 ) steps="${steps}234/240" ;; * ) err "$prog - ERROR - forecast periode ${fcp} not supported for steps" errit 1 ;; esac else case ${fcp} in # 30 Nov 2010 - P. Le Sager - # before: # 1 ) steps="${steps}018/021/024/027/030/033/036" ;; # 2 ) steps="${steps}039/042/045/048/051/054/057/060" ;; # 3 ) steps="${steps}063/066/069/072/075/078/081/084" ;; # 4 ) steps="${steps}087/090/093/096/102/108" ;; # 5 ) steps="${steps}114/120/126/132" ;; # 6 ) steps="${steps}138/144/150/156" ;; # 7 ) steps="${steps}162/168/174/180" ;; # 8 ) steps="${steps}186/192/198/204" ;; # 9 ) steps="${steps}210/216/222/228" ;; # 10 ) steps="${steps}234/240" ;; # now: 1 ) steps="${steps}018/021/024/027/030/033/036/039" ;; 2 ) steps="${steps}042/045/048/051/054/057/060/063" ;; 3 ) steps="${steps}066/069/072/075/078/081/084/087" ;; 4 ) steps="${steps}090/093/096/102/108" ;; 5 ) steps="${steps}114/120/126/132" ;; 6 ) steps="${steps}138/144/150/156" ;; 7 ) steps="${steps}162/168/174/180" ;; 8 ) steps="${steps}186/192/198/204" ;; 9 ) steps="${steps}210/216/222/228" ;; 10 ) steps="${steps}234/240" ;; * ) err "$prog - ERROR - forecast periode ${fcp} not supported for steps" errit 1 ;; esac fi # next fc periode: fcp=`/usr/bin/expr ${fcp} + 1` done fi # fcp 0 only or one or forecast periode range ;; # ~~~~~~~~~~~~~~~~~~ 'ea-fc' ) # ~~~~~~~~~~~~~~~~~~ # ERA5: retrieve hourly surface fields, others at 3 hourly case "${levs}" in 'sfc' ) expvers="${expver}" dates="${ccyymmdd}" times="0600/1800" steps="001/002/003/004/005/006/007/008/009/010/011/012" ;; * ) expvers="${expver}" dates="${ccyymmdd}" times="0600/1800" steps="003/006/009/012" ;; esac ;; # ~~~~~~~~~~~~~~~~~~ 'rd-fc' ) # ~~~~~~~~~~~~~~~~~~ # a keyword 'tres' should be used .. if [ -z "${tres}" ] ; then err "$prog - ERROR - sources definition line should contain a 'tres' value" errit 1 fi # fill per tres: case ${tres} in '_fc000up24tr1' ) expvers="${expver}" dates="${ccyymmdd}" times="0030/0130/0230/0330/0430/0530/0630/0730/0830/0930/1030/1130/1230/1330/1430/1530/1630/1730/1830/1930/2030/2130/2230/2330" steps="000" ;; '_fc000up1tr24' ) expvers="${expver}" dates="${ccyymmdd}" times="1200" steps="000" ;; * ) err "$prog - ERROR - tres value '${tres}' not supported for class-type : ${class}-${type}" errit 1 ;; esac ;; # ~~~~~~~~~~~~~~~~~~ * ) # ~~~~~~~~~~~~~~~~~~ err "$prog - unsupported class-type for mars data : ${class}-${type}" errit 1 ;; esac # levels case ${levs} in sfc ) levtype='sfc' ;; ml* ) nlev=`echo ${levs} | /usr/bin/tr -d 'ml'` levtype="ml,levelist=1/to/${nlev}" ;; * ) err "$prog - levs '${levs}' not supported ..." errit 1 ;; esac # parameters to retrieve: params=${mfile} # horizontal grid case ${grid} in T* ) resol=`echo ${grid} | /usr/bin/tr -d 'T'` repres="sh,resol=${resol},grid=off" ;; N* ) resol=`echo ${grid} | /usr/bin/tr -d 'N'` repres="gg,grid=${resol},gaussian=reduced" ;; R0.5 ) # regular lon/lat grid, retrieve as archived, # therfore limit area to the centers of first/last rows/columns ; # explicitly define area for [-180,180], otherwise the # default [0,360] is returned: #repres="ll,area=89.75/-179.75/-89.75/179.75,qgrid=0.5/0.5" # mars seems to return always the archived field, # thus [0,360] ... repres="ll,grid=off,resol=off,gaussian=off" ;; * ) err "$prog - grid '${grid}' not supported ..." errit 1 ;; esac # define target file including fields expanded by mars: targets="${class}-${type}-[date]-[time]-${levs}-[param]-${grid}.gb" # To have param numbers including table (173.128 instead of 173) # and times in 4 digits (0000 and 0600 instead of 0 and 600), # use the following trick given by the mars info messages: # # In order to get filenames according to MARS values, # please, set env. var. MARS_MULTITARGET_STRICT_FORMAT=1 # before executing your MARS request # export MARS_MULTITARGET_STRICT_FORMAT=1 # # request could be skipped if all target files are already present; # expand the 'targets' to indvidual file and test existence: # # initially not found: notfound=0 # # loop over dates: for date in `echo ${dates} | /usr/bin/tr '/' ' '` ; do # loop over times: for time in `echo ${times} | /usr/bin/tr '/' ' '` ; do # adhoc: era2003 (expver=500) starts at 2002-10-31 12:00 test "${class}-${expver}" = "e4-500" -a ${date} -lt 20021031 && continue # adhoc: ml91 spin-up (expver=29) between 2005-09-30 12:00 and 2006-02-01 00:00 test "${class}-${expver}" = "od-29" -a ${date}${time} -lt 200509301200 && continue # adhoc fix to be able to read an fields over [00,24] : if [ "${time}" = "2400" ]; then xdate=`${bindir}/date_add ${date} 1` xtime='0000' else xdate=${date} xtime=${time} fi # init list of mars parameter numbers "27.128/28.128/..." : mars_params='' # loop over parameters: for param in `echo ${params} | /usr/bin/tr '/' ' '` ; do # translate param name to grib code: mars_param=${param} case ${param} in CVL ) mars_param=27.128 ;; CVH ) mars_param=28.128 ;; TVL ) mars_param=29.128 ;; TVH ) mars_param=30.128 ;; CI ) mars_param=31.128 ;; SSTK ) mars_param=34.128 ;; SWVL1 ) mars_param=39.128 ;; SWVL2 ) mars_param=40.128 ;; SWVL3 ) mars_param=41.128 ;; SWVL4 ) mars_param=42.128 ;; STL1 ) mars_param=139.128 ;; STL2 ) mars_param=170.128 ;; STL3 ) mars_param=183.128 ;; STL4 ) mars_param=236.128 ;; 10FG ) mars_param=49.128 # check if [ "${class}" == 'od' -a ${date} -ge 20111115 ] ; then echo " WARNING : unless you have modified TM5 code, " echo " WARNING : you are supposed to request 10FG3. " echo " WARNING : It replaces 10FG as variable name for" echo " WARNING : the 3-hourly accumulated gust wind." exit 1 fi ;; 10FG3 ) mars_param=28.228 # check if [ "${class}" == 'od' -a ${date} -lt 20111115 ] ; then echo " WARNING : unless you have modified TM5 code, " echo " WARNING : you are supposed to request 10FG. " echo " WARNING : It was the variable name for the 3-hourly" echo " WARNING : accumulated gust wind before 2011/11/15." exit 1 fi ;; Z ) mars_param=129.128 ;; T ) mars_param=130 ;; #.128 ;; Q ) mars_param=133 ;; #.128 ;; SP ) mars_param=134.128 ;; W ) mars_param=135 ;; #.128 ;; VO ) mars_param=138 ;; #.128 ;; STL1 ) mars_param=139.128 ;; SD ) mars_param=141.128 ;; LSP ) mars_param=142.128 ;; CP ) mars_param=143.128 ;; SF ) mars_param=144.128 ;; SSHF ) mars_param=146.128 ;; SLHF ) mars_param=147.128 ;; LNSP ) mars_param=152 ;; #.128 ;; D ) mars_param=155 ;; #.128 ;; BLH ) mars_param=159.128 ;; 10U ) mars_param=165.128 ;; 10V ) mars_param=166.128 ;; 2T ) mars_param=167.128 ;; 2D ) mars_param=168.128 ;; SSRD ) mars_param=169.128 ;; STL2 ) mars_param=170.128 ;; LSM ) mars_param=172.128 ;; SR ) mars_param=173.128 ;; AL ) mars_param=174.128 ;; STRD ) mars_param=175.128 ;; SSR ) mars_param=176.128 ;; STR ) mars_param=177.128 ;; EWSS ) mars_param=180.128 ;; NSSS ) mars_param=181.128 ;; STL3 ) mars_param=183.128 ;; SRC ) mars_param=198.128 ;; TP ) mars_param=228.128 ;; LSRH ) mars_param=234.128 ;; SKT ) mars_param=235.128 ;; STL4 ) mars_param=236.128 ;; FAL ) mars_param=243.128 ;; FSR ) mars_param=244.128 ;; CLWC ) mars_param=246 ;; #.128 ;; CIWC ) mars_param=247 ;; #.128 ;; CC ) mars_param=248 ;; #.128 ;; # ea accumulated surface fields: MLSPR ) mars_param=029.235 ;; MCPR ) mars_param=030.235 ;; MSR ) mars_param=031.235 ;; METSS ) mars_param=041.235 ;; MNTSS ) mars_param=042.235 ;; MSLHF ) mars_param=034.235 ;; MSSHF ) mars_param=033.235 ;; MSNSWRF ) mars_param=037.235 ;; MSDWSWRF ) mars_param=035.235 ;; MSNLWRF ) mars_param=038.235 ;; MSDWLWRF ) mars_param=036.235 ;; # e4 convective fields, local table should have been 162 ... 104.128 ) mars_param=104.128 ;; 105.128 ) mars_param=105.128 ;; 106.128 ) mars_param=106.128 ;; 107.128 ) mars_param=107.128 ;; # ei convective fields: 104.162 ) mars_param=104.162 ;; 105.162 ) mars_param=105.162 ;; 106.162 ) mars_param=106.162 ;; 107.162 ) mars_param=107.162 ;; # ei diffusion: 109.162 ) mars_param=109.162 ;; # ea convective fields, param codes as written by mars: MUMF ) mars_param=235009 ;; #mars_param=009.235 ;; MDMF ) mars_param=235010 ;; #mars_param=010.235 ;; MUDR ) mars_param=235011 ;; #mars_param=011.235 ;; MDDR ) mars_param=235012 ;; #mars_param=012.235 ;; # ea diffusion, param codes as written by mars: MTDCH ) mars_param=235014 ;; #mars_param=014.235 ;; # macc emissions: CH4FIRE ) mars_param='82.210' ;; # not supported yet ... * ) err "$prog - no grib code implemented for param '${param}'" errit 1 ;; esac # add to list: if [ -z "${mars_params}" ]; then mars_params="${mars_param}" else mars_params="${mars_params}/${mars_param}" fi # target file name: fname=`echo ${targets} | /usr/bin/sed -e "s^\[date\]^${xdate}^" -e "s^\[time\]^${xtime}^" -e "s^\[param\]^${mars_param}^"` # found ? if [ -f ${fname} ]; then echo "$prog - found ${fname}" # touch to prevent removal because of quota test -w ${fname} && touch ${fname} else echo "$prog - missing ${fname}" # at least one file is not found yet, # so further testing is not necessary notfound=1 # do not leave the 'param' loop, need to fill 'mars_params' list ... #break fi done # loop over params test ${notfound} -gt 0 && break done # loop over times test ${notfound} -gt 0 && break done # loop over dates # add mars request ? if [ ${notfound} -gt 0 ]; then # add line to mars job for each time: istep=0 for time in ${times} ; do # next step: istep=`/usr/bin/expr ${istep} + 1` # corresponding steps etc: stepi=`echo ${steps} | /usr/bin/cut -d ' ' -f ${istep}` expveri=`echo ${expvers} | /usr/bin/cut -d ' ' -f ${istep}` # adhoc fix to be able to read an fields over [00,24] : if [ "${time}" = "2400" ]; then # next day, one time only: xdate=`${bindir}/date_add ${date} 1` xtime='0000' else # just copy: xdate=${dates} xtime=${time} fi # add line: echo "$prog - add line to mars job for ${xdate} ${xtime} ..." echo "retrieve,class=${class},stream=${stream},expver=${expveri},type=${type},date=${xdate},time=${xtime},step=${stepi},levtype=${levtype},param=${mars_params},repres=${repres},target="'"'"${targets}"'"' >> ${mars_jb} done else # all found ... echo "$prog - all found for ${ccyymmdd} ..." fi # next day: ccyymmdd=`${bindir}/date_add ${ccyymmdd} 1` done # loop over days set +x ;; # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ncep-cdc:* ) # NCEP RA NetCDF files # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # year files, valid for valid for [00,24) # retrieve previous year if necessary case ${mfile} in 'lhtfl' ) test "${mm1}${dd1}${hh1}" -eq "010100" && ccyy1=`/usr/bin/expr ${ccyy1} - 1` ;; esac # loop over years ccyy=${ccyy1} while [ ${ccyy} -le ${ccyy2} ]; do # # define filename etc # # # spectral /div .spec .2000.nc # spectral /pres.nlog.sfc.spec .2000.nc # spectral /vort .spec .2000.nc # surface_gauss/lhtfl.sfc .gauss.2000.nc # pressure /omega .2000.nc # ## split in 'ncep-cdc' and keys : ##keys=`echo ${msource} | /usr/bin/cut -d ':' -f 2-` # ## evaluate keys : mdir=spectral ##eval ${keys} # file and directory name: case ${mfile} in 'hgt' | 'land' ) meteodir='surface_gauss' meteofile="${mfile}.sfc.gauss.nc" ;; 'sfcr' | 'pres' | 'icec' | 'skt' | 'lhtfl' | 'shtfl' | 'ugwd' | 'vgwd' | 'prate' | 'cprat' ) meteodir='surface_gauss' meteofile="${mfile}.sfc.gauss.${ccyy}.nc" ;; 'uwnd.10m' | 'vwnd.10m' ) meteodir='surface_gauss' meteofile="${mfile}.gauss.${ccyy}.nc" ;; 'pres.nlog' ) meteodir='spectral' meteofile="${mfile}.sfc.spec.${ccyy}.nc" ;; 'vort' | 'div' | 'vair' | 'shum' ) meteodir='spectral' meteofile="${mfile}.spec.${ccyy}.nc" ;; * ) err "$prog - ERROR - unsupported ncep file '${mfile}' ..." errit 1 ;; esac # expand directory: meteodir_long=`echo ${meteodir} | /usr/bin/tr '/' '-'` # expand day file: meteofile_full="${meteodir}/${meteofile}" meteofile_long="${meteodir_long}-${meteofile}" # not found yet ... found='' # # present in buffer ? # # not found yet ? if [ -z "${found}" ]; then fname=./${meteofile_long} #test ${verbose} && echo "$prog - try ${fname}" if [ -f ${fname} ]; then #test ${verbose} && echo "$prog - found ..." test -w ${fname} && touch ${fname} found='true' fi fi # not found yet ... # # search locations # # not found yet ? if [ -z "${found}" ]; then # loop over search locations for loc in ${search_locs} ; do # *** full file fname=${loc}/${meteofile_full} test ${verbose} && echo "$prog - try ${fname} ..." if ${bindir}/gss exist ${fname} ; then # create link in buffer: test ${verbose} && echo "$prog - link ..." bname=./${meteofile_long} ${bindir}/gss ${transfermode} ${fname} ${bname} # found ! found='true' break fi # *** long file fname=${loc}/${meteofile_long} test ${verbose} && echo "$prog - try ${fname} ..." if ${bindir}/gss exist ${fname} ; then # create link in buffer: test ${verbose} && echo "$prog - link ..." bname=./${meteofile_long} ${bindir}/gss ${transfermode} ${fname} ${bname} # found ! found='true' break fi done # loop over locations fi # not found yet ... # not found: error ... if [ -z "${found}" ]; then err "$prog - ERROR : do not know how to retrieve meteo file:" err "$prog - ERROR : directory : `/bin/pwd`" err "$prog - ERROR : meteofile : ${meteofile_long}" errit 1 fi # next year: ccyy=`/usr/bin/expr ${ccyy} + 1` done # loop over years ;; # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ncep-gfs:* ) # NCEP GFS binary and grib files # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # files in directory valid for [12,12] test ${hh1} -lt 12 && ccyymmdd1=`${bindir}/date_add ${ccyymmdd1} -1` test ${hh2} -lt 12 && ccyymmdd2=`${bindir}/date_add ${ccyymmdd2} -1` # loop over days ccyymmdd=${ccyymmdd1} while [ ${ccyymmdd} -le ${ccyymmdd2} ]; do # split current day: ccyy=`echo ${ccyymmdd} | /usr/bin/cut -c 1-4` mm=`echo ${ccyymmdd} | /usr/bin/cut -c 5-6` dd=`echo ${ccyymmdd} | /usr/bin/cut -c 7-8` # hour list: hours='12.00 12.03 12.06 12.09 12.12 12.15 12.18 12.21 12.24' # loop over hours for hour in ${hours} ; do # # define filename etc # # # 20040614/20040614.00.00.SF # 20040614/20040614.00.00.SFLUXGrbF # # split in 'ncep-gfs' and keys : #keys=`echo ${msource} | /usr/bin/cut -d ':' -f 2-` # evaluate keys : mdir=spectral #eval ${keys} # file and directory name: meteodir="${ccyy}${mm}${dd}" meteofile="${ccyy}${mm}${dd}.${hour}.${mfile}" # expand directory: meteodir_long=`echo ${meteodir} | /usr/bin/tr '/' '-'` # expand day file: meteofile_full="${meteodir}/${meteofile}" meteofile_long="${meteodir_long}-${meteofile}" # not found yet ... found='' # # present in buffer ? # # not found yet ? if [ -z "${found}" ]; then fname=./${meteofile_long} #test ${verbose} && echo "$prog - try ${fname}" if [ -f ${fname} ]; then #test ${verbose} && echo "$prog - found ..." test -w ${fname} && touch ${fname} found='true' fi fi # not found yet ... # # search locations # # not found yet ? if [ -z "${found}" ]; then # loop over search locations for loc in ${search_locs} ; do # *** full file fname=${loc}/${meteofile_full} test ${verbose} && echo "$prog - try ${fname} ..." if ${bindir}/gss exist ${fname} ; then # create link in buffer: test ${verbose} && echo "$prog - link ..." bname=./${meteofile_long} ${bindir}/gss ${transfermode} ${fname} ${bname} # found ! found='true' break fi # *** long file fname=${loc}/${meteofile_long} test ${verbose} && echo "$prog - try ${fname} ..." if ${bindir}/gss exist ${fname} ; then # create link in buffer: test ${verbose} && echo "$prog - link ..." bname=./${meteofile_long} ${bindir}/gss ${transfermode} ${fname} ${bname} # found ! found='true' break fi done # loop over locations fi # not found yet ... # not found: error ... if [ -z "${found}" ]; then err "$prog - ERROR : do not know how to retrieve meteo file:" err "$prog - ERROR : directory : `/bin/pwd`" err "$prog - ERROR : meteofile : ${meteofile_long}" errit 1 fi done # hours # next day: ccyymmdd=`${bindir}/date_add ${ccyymmdd} 1` done # loop over days ;; # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ msc-data:* ) # MSC data files # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # loop over days ccyymmdd=${ccyymmdd1} while [ ${ccyymmdd} -le ${ccyymmdd2} ]; do # split current day: ccyy=`echo ${ccyymmdd} | /usr/bin/cut -c 1-4` mm=`echo ${ccyymmdd} | /usr/bin/cut -c 5-6` dd=`echo ${ccyymmdd} | /usr/bin/cut -c 7-8` # # define filename etc # # example: msc-data:nlev=71;sh=47;mdir=cmam;tres=_1dag_6hrly # default keys: mdir='' tres='' # split in 'msc-data' and keys : keys=`echo ${msource} | /usr/bin/cut -d ':' -f 2-` # evaluate keys : form=tm5; mdir=od-fc-etc; tres=_21p06 eval ${keys} # define directory and file: meteodir=${mdir}'/'${type} meteofile="${mfile}_${ccyymmdd}${tres}.dat" # expand directory: meteodir_long=`echo ${meteodir} | /usr/bin/tr '/' '-'` # expand day file: meteofile_full="${meteodir}/${meteofile}" meteofile_long="${meteodir_long}-${meteofile}" # not found yet ... found='' # # present in buffer ? # # not found yet ? if [ -z "${found}" ]; then fname=./${meteofile_long} #test ${verbose} && echo "$prog - try ${fname}" if [ -f ${fname} ]; then #test ${verbose} && echo "$prog - found ..." test -w ${fname} && touch ${fname} found='true' fi fi # not found yet ... # # search locations # # not found yet ? if [ -z "${found}" ]; then # loop over search locations for loc in ${search_locs} ; do # *** full file fname=${loc}/${meteofile_full} test ${verbose} && echo "$prog - try ${fname} ..." if ${bindir}/gss exist ${fname} ; then # create link in buffer: test ${verbose} && echo "$prog - link ..." bname=./${meteofile_long} ${bindir}/gss ${transfermode} ${fname} ${bname} # found ! found='true' break fi # *** long file fname=${loc}/${meteofile_long} test ${verbose} && echo "$prog - try ${fname} ..." if ${bindir}/gss exist ${fname} ; then # create link in buffer: test ${verbose} && echo "$prog - link ..." bname=./${meteofile_long} ${bindir}/gss ${transfermode} ${fname} ${bname} # found ! found='true' break fi done # loop over locations fi # not found yet ... # not found: error ... if [ -z "${found}" ]; then err "$prog - ERROR : do not know how to retrieve meteo file:" err "$prog - ERROR : directory : `/bin/pwd`" err "$prog - ERROR : meteofile : ${meteofile_long}" errit 1 fi # next day: ccyymmdd=`${bindir}/date_add ${ccyymmdd} 1` done # loop over days ;; # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ olsson:* ) # Olsson sr files # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # month files # loop over months ccyy=${ccyy1} mm=${mm1} while [ ${ccyy}${mm} -le ${ccyy2}${mm2} ]; do # # define filename etc # # example: olsson:mdir=SR_OLSSON # default keys: mdir='OLSSON' # split in 'olsson' and keys : keys=`echo ${msource} | /usr/bin/cut -d ':' -f 2-` # evaluate keys : mdir=oSR_OLSSON eval ${keys} # define directory and file: meteodir="${mdir}" meteofile="${mfile}_OLSSON_360_180_${mm}.d" # expand directory: meteodir_long=`echo ${meteodir} | /usr/bin/tr '/' '-'` # expand day file: meteofile_full="${meteodir}/${meteofile}" meteofile_long="${meteodir_long}-${meteofile}" # not found yet ... found='' # # present in buffer ? # # not found yet ? if [ -z "${found}" ]; then fname=./${meteofile_long} #test ${verbose} && echo "$prog - try ${fname}" if [ -f ${fname} ]; then #test ${verbose} && echo "$prog - found ..." test -w ${fname} && touch ${fname} found='true' fi fi # not found yet ... # # search locations # # not found yet ? if [ -z "${found}" ]; then # loop over search locations for loc in ${search_locs} ; do # *** just file (ie not assumption on parent dir) fname=${loc}/${meteofile} test ${verbose} && echo "$prog - try ${fname} ..." if ${bindir}/gss exist ${fname} ; then # create link in buffer: test ${verbose} && echo "$prog - link ..." bname=./${meteofile_long} ${bindir}/gss ${transfermode} ${fname} ${bname} # found ! found='true' break fi # *** full file fname=${loc}/${meteofile_full} test ${verbose} && echo "$prog - try ${fname} ..." if ${bindir}/gss exist ${fname} ; then # create link in buffer: test ${verbose} && echo "$prog - link ..." bname=./${meteofile_long} ${bindir}/gss ${transfermode} ${fname} ${bname} # found ! found='true' break fi # *** long file fname=${loc}/${meteofile_long} test ${verbose} && echo "$prog - try ${fname} ..." if ${bindir}/gss exist ${fname} ; then # create link in buffer: test ${verbose} && echo "$prog - link ..." bname=./${meteofile_long} ${bindir}/gss ${transfermode} ${fname} ${bname} # found ! found='true' break fi done # loop over locations fi # not found yet ... # not found: error ... if [ -z "${found}" ]; then err "$prog - ERROR : do not know how to retrieve meteo file:" err "$prog - ERROR : directory : `/bin/pwd`" err "$prog - ERROR : meteofile : ${meteofile_long}" errit 1 fi # next month: mm=`/usr/bin/expr ${mm} + 1` if [ ${mm} -gt 12 ]; then ccyy=`/usr/bin/expr ${ccyy} + 1` mm=1 fi # 2 digits ... mm=`/usr/bin/printf '%2.2i' ${mm}` done # loop over days ;; # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ * ) # unsupported ... # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ err "$prog - ERROR - unsupported source:" err "$prog - ERROR - ${msource}" errit 1 ;; esac # setup given source done # loop over sources done # loop over files done # loop over sets # ************************************* # step 3 : post # ************************************* # retrieve files from mars ? if [ -f ${mars_jb} ]; then # clean up ? mars_clean='false' # mars output file: mars_out="`/bin/pwd`/mars.out" ## adhoc: keep output files for profiling: mars_out="${mars_out}.$$" mars_clean='' echo "$prog - retrieve files from mars ..." echo "$prog - start at `/bin/date`" if ( /usr/bin/sort ${mars_jb} | mars > ${mars_out} 2>&1 ) ; then echo "$prog - end at `/bin/date`" test "${mars_clean}" && /bin/rm -f ${mars_jb} ${mars_out} # # force conversion from grib2 to grib1 until grib_api is implemented in TM5 # if ( hostname | grep c2 ); then # echo " editionNumber=1;" > gb2to1 # for i in *.gb; do # # if (grib_dump $i | grep "editionNumber.*=.*2"); then # grib_convert -M -G -7 gb2to1 $i dummy # mv -f dummy $i # # fi # done # fi else err "$prog - error from mars request; see:" err "$prog - ${mars_out}" errit 1 fi fi # ************************************* # step 4 : clean # ************************************* # clean flag ? if [ "${tmm_dir_clean}" = "T" ]; then # limit disk usage: ${bindir}/gss ${verbose} limit ${tmm_dir_size} ${tmm_dir} fi # # back to origin # cd ${owd} # ----------------------------------------------------- # end # ----------------------------------------------------- test ${verbose} && echo "$prog - end" # ok exit 0