ecearth3/sources/tm5mp/bin/tm5-tmm-store

#! /bin/sh

# This script is called if added to the list of postprocessing scripts in your
# main rc file. It is dedicated to the STORAGE of WRITTEN MET FIELDS. It
# requires several "tmm.output.store.*" keys, "tmm.output.dir", and
# "tmm.output"
# It relies on the set of bin/gss* script to tar/zip/move-to-destination


# -------------------------------------------------------------
# --- init
# -------------------------------------------------------------

# leave on error
set -e

# set program name and location:
call="$0 $*"
case $0 in
  /* ) script=$0 ;;
  *  ) script="`pwd`/$0" ;;
esac
bindir=`dirname ${script}`
prog=`basename ${script}`


# -------------------------------------------------------------
# --- help
# -------------------------------------------------------------

DisplayUsage ()
{
${PAGER:-less} << EOF
NAME
  ${prog} -  store meteo created by TM5
  
USAGE
  ${prog} <rcfile>
  
EOF
exit 0
}


# err 'help text'
err ()
{
  echo "$1" 1>&2
}


# errit <exit-status>
errit ()
{
  err "$prog - ERROR - from : ${call}"
  err "$prog - ERROR - Use '$prog --help' for more info."
  exit $1
}


# -------------------------------------------------------------
# --- arguments
# -------------------------------------------------------------

rcfile=''

for arg in "$@" ; do
  case ${arg} in
    -h | --help ) DisplayUsage  ;;
    -* )
      err "$prog - ERROR - unknown option '${arg}'"
      errit 1
      ;;
    * )
      if [ -z "${rcfile}" ] ; then
        rcfile=${arg}
      else
        err "$prog - ERROR - unknown argument '${arg}'"
        errit 1
      fi
      ;;
  esac
done

# not complete ?
if [ -z "${rcfile}" ] ; then
  err "$prog - ERROR - no rcfile specified"
  errit 1
fi


# -------------------------------------------------------------
# --- settings
# -------------------------------------------------------------

# info ...
echo "$prog - start"

# location of readrc script:
go_readrc="${bindir}/go_readrc"

# are meteo files actually written ?
tmm_output=`${go_readrc} ${rcfile} 'tmm.output' 'F'`
# if not ...
if [ "${tmm_output}" != "T" ] ; then
  # info ...
  echo "$prog - no meteo files written; return"
  # return without problems ...
  exit 0
fi

# directory where TM5 has stored the files:
tmm_output_dir=`${go_readrc} ${rcfile} 'tmm.output.dir' './tmm_output_dir'`

# archive directories (gss)
archives=`${go_readrc} ${rcfile} 'tmm.output.store.archives' ' '`
archives_fc=`${go_readrc} ${rcfile} 'tmm.output.store.archives.fc' ' '`

# messages ?
verboseTF=`${go_readrc} ${rcfile} 'tmm.output.store.verbose' ''`
if [ ${verboseTF} = T ] ; then verbose='true' ; else verbose='' ; fi

# zipper ?
zipper=`${go_readrc} ${rcfile} 'tmm.output.store.zipper' ''`

# gss messages ?
verb=''
verb='--verbose'


# -------------------------------------------------------------
# --- begin
# -------------------------------------------------------------

echo "$prog - store meteo output ..."

# output dir present ?
if [ ! -d ${tmm_output_dir} ] ; then
  err "$prog - ERROR - output directory not found:"
  err "$prog - ERROR -   ${tmm_output_dir}"
  errit 1
fi

# change to output directory:
owd=`/bin/pwd`
cd ${tmm_output_dir}

# not stored in tarfiles yet ...
tarfile_latest=''

# loop over files in output buffer:
for mfile in `/bin/ls` ; do

  # skip status files, and oro.lsm which never change in ERA-Interim
  case ${mfile} in
    *.hdf.status ) continue ;;
    *.nc.status ) continue ;;
    *oro.nc* ) continue ;;
    *lsm.nc* ) continue ;;
  esac

  echo "$prog -   ${mfile} ..."
  
  # no status file ? then skip
  mstatfile="${mfile}.status"
  if [ ! -f ${mstatfile} ]; then
    echo "$prog -     no status file; skip"
    continue
  fi
  
  # completed files should always replace older ones;
  # uncompleted files should only be added if older files do not exist:
  if [ "`/bin/cat ${mstatfile}`" = "completed" ]; then
    verb_keep='replace'
    keep_old_files=''
  else
    verb_keep='keep'
    keep_old_files='--keep-old-files'
  fi
  
  # zip the file ?
  mfileZ="${mfile}"
  if [ -n "${zipper}" ]; then
    case ${zipper} in
      'gzip' )
        gzip ${mfile}
        mfileZ="${mfile}.gz"
        ;;
      'bzip2' )
        gzip ${mfile}
        mfileZ="${mfile}.bz2"
        ;;
      'internal' )
            # This may be the best option for netCDF files. Instead of
            # compressing in the Fortran code, it is more flexible to do it
            # here: we can for example remove the unlimited dimension, switch
            # to netcdf4_classic, ...
            # Switch to netcdf4_classic with efficient (using -s) L1 compression
            #err "Internal compression not supported yet"
            rm -f aap.nc
            # note that we cannot use "nccopy" without specifying the path
            # depending on the library loaded (if any!) this can be problematic 
#            /opt/cray/netcdf/4.4.0/bin/nccopy -k 4 -d 1 -s ${mfile} aap.nc
            # here is what works with: load cray-hdf5-parallel/1.8.14  & cray-netcdf-hdf5parallel/4.3.3.1
            /opt/cray/netcdf/4.3.0/bin/nccopy -k 4 -d 1 -s ${mfile} aap.nc

            mv -f aap.nc ${mfile}
            ;;
      * )
        err "ERROR - unsupported zipper : ${zipper}"
        errit 1
        ;;
    esac
  fi    
  
  # default destination archives:
  archivesX="${archives}"
  
  # forecast files might be send to different archives ...
  case ${mfileZ} in
    *_[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]f[0-9]* ) 
      test "${archives_fc}"  &&  archivesX="${archives_fc}"
      ;;
  esac
    
  # loop over archives:
  for archiveX in ${archivesX} ; do
  
    echo "$prog -     archive in ${archiveX} ..."
  
    # different archive methods:
    case ${archiveX} in

      # if archive ends on '/*.tar'*, then store in tarfiles:
      */\*.tar* )

        # skip '*.tar' part from archive name:
        archive=`/usr/bin/dirname ${archiveX}`
        #echo "$prog -       tarchive dir ${archive} ..."

        # split mfile in 'od-fc-...' and 'uvsp_20001201_21p06' parts:
        n=`echo ${mfileZ} | /usr/bin/tr '-' ' ' | /usr/bin/wc -w`
        mdir=`echo ${mfileZ} | /usr/bin/cut -d '-' -f -$((${n}-1))`
        mfil=`echo ${mfileZ} | /usr/bin/cut -d '-' -f ${n}`

        # extract parameter name from mfil: 'uvsp', 'T', etc:
        mfil_base=${mfil}
        mfil_base=`basename ${mfil_base} '.gz'`
        mfil_base=`basename ${mfil_base} '.bz2'`
        mfil_base=`basename ${mfil_base} '.hdf'`
        mfil_base=`basename ${mfil_base} '.nc'`
        tarname=`echo ${mfil_base} | /usr/bin/cut -d '_' -f 1`

        # tar file in long format:
        #   od-fc-xxxx-oro.tar
        #   od-fc-xxxx-uvsp_200012_21p06.tar
        case ${tarname} in
          'oro' | 'lsm' )
            tarfile="${mdir}-${tarname}.tar"
            ;;
          'srols' )
            # extract '200012'
            ccyymm=`echo ${mfil_base} | /usr/bin/cut -d '_' -f 2`
            tarfile="${mdir}-${tarname}_${ccyymm}.tar"
            ;;
          * )
            # extract '20001201':
            ccyymmdd=`echo ${mfil} | /usr/bin/cut -d '_' -f 2`
            ccyymmd=`echo ${ccyymmdd} | /usr/bin/cut -c 1-7`
            ccyymm=`echo ${ccyymmdd} | /usr/bin/cut -c 1-6`
            d=`echo ${ccyymmdd} | /usr/bin/cut -c 8`
            # large 3D files ?
            case ${tarname} in
              'cld' | 'mfuv' | 'mfw' | 'q' | 'sub' | 't' )
                # tar per month, 10 days, 5 days ...
                case ${archiveX} in
                  */\*.tar )
                    # year and month:
                    tardate=`echo ${ccyymmdd} | /usr/bin/cut -c 1-6`
                    ;;
                  */\*.tar10 )
                    # year, month, 0|1|2|3
                    tardate=`echo ${ccyymmdd} | /usr/bin/cut -c 1-7`
                    ;;
                  */\*.tar5 )
                    # year, month, 00|05|10|15|20|25|30
                    case ${d} in
                      [0-4] ) tardate="${ccyymmd}0" ;;
                      [5-9] ) tardate="${ccyymmd}5" ;;
                    esac
                    ;;
                  * )
                    err "$prog - ERROR - unsupported archive for large tar files :"
                    err "$prog - ERROR -   ${archive}"
                    errit 1
                    ;;
                esac
                ;;
              * )
                # year and month:
                tardate=`echo ${ccyymmdd} | /usr/bin/cut -c 1-6`
                ;;
            esac
            # rest of the name (time resolution, extension):
            mend=`echo ${mfil} | /usr/bin/cut -d '_' -f 3-`
            # rest of the file without extension:
            tarend=${mend}
            tarend=`basename ${tarend} '.gz'`
            tarend=`basename ${tarend} '.bz2'`
            tarend=`basename ${tarend} '.hdf'`
            tarend=`basename ${tarend} '.nc'`
            # name of tarfile:
            tarfile="${mdir}-${tarname}_${tardate}_${tarend}.tar"
            ;;
        esac
        # tarfile with full path:
        tarfile_full=`echo ${tarfile} | tr '-' '/'`
        
        # unlink previous tar file if necessary:
        if [ "${tarfile_latest}" ]; then
          # different from current tarfile ?
          if [ "${tarfile}" != "${tarfile_latest}" ]; then
            # unlink previous tarfile:
            echo "$prog -       unlink ${tarfile_latest} ..."
            ${bindir}/gss ${verb} unlink ${tarfile_latest}
          fi
        fi

        # link tarfile if not present yet:
        if [ ! -f ${tarfile} ] ; then
          echo "$prog -       link ${tarfile} ..."
          ${bindir}/gss ${verb} link --new ${archive}/${tarfile_full} ${tarfile}
        fi

        # add to (linked) tarfile:
        echo "$prog -       add ${mfileZ} to ${tarfile} (${verb_keep}) ..."
        ${bindir}/gss ${verb} ${keep_old_files} tar-update ${tarfile} ${mfileZ}
        
        # store name of current tarfile:
        tarfile_latest=${tarfile}
        
        ;;

      # store directly (not in tarfile)
      * )

        # file name with full path:
        mfile_full=`echo ${mfileZ} | tr '-' '/'`
        
        # test on existing file ?
        if [ "${keep_old_files}" ]; then
          if ( ${bindir}/gss exist ${archiveX}/${mfile_full} ) ; then
            echo "$prog -       keep existing ${archiveX}/${mfile_full} ..."
          else
            # copy:
            echo "$prog -       copy to ${archiveX}/${mfile_full} (${verb_keep}) ..."
            ${bindir}/gss ${verb} copy --mkdir ${mfileZ} ${archiveX}/${mfile_full}
          fi
        else
          # copy, eventually overwrite existing file:
          echo "$prog -       copy to ${archiveX}/${mfile_full} (${verb_keep}) ..."
          ${bindir}/gss ${verb} copy --mkdir ${mfileZ} ${archiveX}/${mfile_full}
        fi

        ;;

    esac  # tarfile or not
    
  done  # loop over archives

  # remove meteo and status file:
  rm -f ${mfileZ}
  rm -f ${mstatfile}

done  # loop over meteo files

# unlink tarfile if necessary:
if [ "${tarfile_latest}" ]; then
  # unlink (store on tape ?)
  echo "$prog -   unlink ${tarfile_latest} ..."
  ${bindir}/gss ${verb} unlink ${tarfile_latest}
fi

# change back:
cd ${owd}


# -------------------------------------------------------------
# --- end
# -------------------------------------------------------------

echo "$prog - end"

# ok
exit 0