ecearth3/sources/tm5mp/bin/gss

#! /usr/bin/env sh

#ProTeX: 1.14-AJS
#
#BOI
#
# !TITLE:        GSS -  General Storage System
# !AUTHORS:      Arjo Segers
# !AFFILIATION:  KNMI, The Netherlands
# !DATE:         \today
#
# !INTRODUCTION: Usage
#
# \bv
#   gss <task> [<options>] [<location> ...]
# \ev
#
#
# !INTRODUCTION: Description
#
#   Shell script to access file systems, tape systems, ftp servers,
#   and other file archives in a uniform way.
#
#
# !INTRODUCTION: TASKS
#
# \bv
#   list [-l,--long] [<location> ...]
#
#       File or directory listing.
#       If no location is specified, the current directory is listed.
#       Use the '-l' option for long listing.
#
#   exist [-e,--echo] <location>
#
#       Returns with zero exit status (succeed) if the location exists,
#       and non-zero (fail) if not.
#       With the '-e' option, the csh status '1' (succeed) is echo'd
#       to standard output if the location exists and '0' (fail) if not;
#       the script always succeeds.  
#
#   copy [-f,--force] [-m,--mkdir] <source-location> <dest-location>
#
#       Copy source file to destionation.
#         -f, --force   : overwrite existing destination files if necessary
#         -m, --mkdir   : create destination directories if necessary
#
#
#   link [-s,--soft] [-f,--force] [-n,--new]  <remote-location>  <localfile>
#   unlink  <localfile>
#
#       Create a symbolic link with name <localfile> to a so-called 'linkfile'
#       with name '.*.gsslink'.
#       The 'linkfile' is copy from or a symbolic link to the file specified
#       by <remote-location> .
#       If the <localfile> is unlinked, both the <localfile> and the
#       'linkfile' are removed.
#       In case of a 'hard' link (default), the original location is written
#       to a file named '.*.gsslinksrc'; if a hard link is removed,
#       and the linkfile is a copy of a remote file,
#       the remote file is replaced by the (changed) local copy.
#
#          local                                   remote
#          --------------------------------------  ---------------------
#
#          data-a.dat  ->  ._data_a_dat.gsslink
#          ._data_a.dat.gsslink  ->  /data/a.dat   /data/a.dat
#
#          data-b.dat -> .tape_b_dat.gsslink
#          .tape_b_dat.gsslink                     tape:b.dat
#          .tape_b_dat.gsslinksrc
#
#       Options:
#         -s, --soft    : create soft link
#         -f, --force   : overwrite existing destination files if necessary
#         -n, --new     : create new empty file if necessary
#
#
#   tar-extract  <tarfile>  [<tarred-file>]
#
#       Extract the <tarred-file> from a tarfile or untar the tarfile completely.
#
#   tar-update   <tarfile>  <input-file>
#
#       Update the <input-file> in a (linked) tar file.
#       An old version of <input-file> is removed from the tarfile.
#
#
#   limit <max-usage> <directory>
#
#       If the disk usage of the <directory-location> exceeds the <max-usage>,
#       files that have not been touched recently are removed until the
#       disk usage is small enought.
#       A valid <max-usage> is a number followed by a character
#       to specify kilo (default), mega, or giga bytes,
#       or 'Inf' to not have any limitation (case independent):
#         100, 100k, 2m, 4g,  Inf
# \ev
#
#
# !INTRODUCTION: Locations
#
# \bv
#   #
#   # standard files
#   #
#   file:path/file
#
#   #
#   # KNMI's MOS tape system
#   #
#   mos:/fa/ks/file
#   mos:umask=002%/fa/ks/data/a.txt
#
#   #
#   # CINECA cartridge
#   #
#   # NOTE: no subdirectories allowed, only a single volume
#   cart:myvolume/myfile.txt
#
#   #
#   # ECMWF's ecfs
#   #
#   ecfs:/nl5/data/a.txt
#   ecfs:/TMP/nl5/dump.txt
#   ecfs:umask=002%/nlh/shared/a.txt
#
#   #
#   # ECMWF file system via EcAccess gateway
#   #
#   ec:echome[nl5]:path/file
#   ec:ecscratch[nl5]:path/file
#   ec:ecfs[nl5]:path/file
#   ec:ectmp[nl5]:path/file
#
#   #
#   # ftp server
#   #
#   ftp:ftp.somewhere.int:path/file
# \ev
#
# !INTRODUCTION: General options
#
# \bv
#   -v, --verbose   : display info on progress  
# \ev
#
#EOI

# ----------------------------------------------------------
# --- init
# ----------------------------------------------------------

# leave on error
set -e

# set program name and location:
call="$0 $*"
case $0 in
  /* ) script=$0 ;;
  *  ) script="`pwd`/$0" ;;
esac
bindir=`dirname ${script}`
prog=`basename ${script}`


# ----------------------------------------------------------
# --- help
# ----------------------------------------------------------

DisplayUsage ()
{
${PAGER:-less} << EOF
NAME
  ${prog} -  General Storage System
  
USAGE
  ${prog} <task> [<options>] [<location> ...]
  

DESCRIPTION

  Shell script to access file systems, tape systems, ftp servers,
  and other file archives in a uniform way.
  
  
TASKS
  
  list [-l,--long] [<location> ...]

      File or directory listing.
      If no location is specified, the current directory is listed.
      Use the '-l' option for long listing.
      
  exist [-e,--echo] <location>

      Returns with zero exit status (succeed) if the location exists,
      and non-zero (fail) if not.
      With the '-e' option, the csh status '1' (succeed) is echo'd
      to standard output if the location exists and '0' (fail) if not;
      the script always succeeds.  
  
  copy [-f,--force] [-m,--mkdir] <source-location> <dest-location>

      Copy source file to destionation.
        -f, --force   : overwrite existing destination files if necessary
        -m, --mkdir   : create destination directories if necessary
  

  link [-s,--soft] [-f,--force] [-n,--new]  <remote-location>  <localfile>
  unlink  <localfile>

      Create a symbolic link with name <localfile> to a so-called 'linkfile'
      with name '.*.gsslink'.
      The 'linkfile' is copy from or a symbolic link to the file specified
      by <remote-location> .
      If the <localfile> is unlinked, both the <localfile> and the
      'linkfile' are removed.
      In case of a 'hard' link (default), the original location is written
      to a file named '.*.gsslinksrc'; if a hard link is removed,
      and the linkfile is a copy of a remote file,
      the remote file is replaced by the (changed) local copy.
      
         local                                   remote
         --------------------------------------  ---------------------

         data-a.dat  ->  ._data_a_dat.gsslink
         ._data_a.dat.gsslink  ->  /data/a.dat   /data/a.dat
                                                
         data-b.dat -> .tape_b_dat.gsslink
         .tape_b_dat.gsslink                     tape:b.dat
         .tape_b_dat.gsslinksrc

      Options:
        -s, --soft    : create soft link
        -f, --force   : overwrite existing destination files if necessary
        -n, --new     : create new empty file if necessary
      

  tar-extract  <tarfile>  [<tarred-file>]

      Extract the <tarred-file> from a tarfile or untar the tarfile completely.

  tar-update  [-k,--keep-old-files]  <tarfile>  <input-file>

      Update the <input-file> in a (linked) tar file.
      If '--keep-old-files' is not specified, an old version of <input-file> 
      is removed from the tarfile.


  limit <max-usage> <directory>
  
      If the disk usage of the <directory-location> exceeds the <max-usage>,
      files that have not been touched recently are removed until the
      disk usage is small enought.
      A valid <max-usage> is a number followed by a character
      to specify kilo (default), mega, or giga bytes,
      or 'Inf' to not have any limitation (case independent):
        100, 100k, 2m, 4g,  Inf
      
  
LOCATIONS

  #
  # standard files
  #
  file:path/file

  #
  # KNMI's MOS tape system
  #
  mos:/fa/ks/file
  mos:umask=002%/fa/ks/data/a.txt
  
  #
  # CINECA cartridge
  #
  # NOTE: no subdirectories allowed, only a single volume
  cart:myvolume/myfile.txt

  #
  # ECMWF's ecfs
  #
  ecfs:/nl5/data/a.txt
  ecfs:/TMP/nl5/dump.txt
  ecfs:umask=002%/nlh/shared/a.txt
  
  #
  # ECMWF file system via EcAccess gateway
  #
  ec:echome[nl5]:path/file
  ec:ecscratch[nl5]:path/file
  ec:ecfs[nl5]:path/file
  ec:ectmp[nl5]:path/file
  
  #
  # ftp server
  #
  ftp:ftp.somewhere.int:path/file

  
GENERAL OPTIONS

  -v, --verbose   : display info on progress  
  
EOF
exit 0
}


# err 'help text'
err ()
{
  echo "$1" 1>&2
}


# errit <exit-status>
errit ()
{
  err "$prog - ERROR - from : ${call}"
  err "$prog - ERROR - Use '$prog --help' for more info."
  exit $1
}


# ----------------------------------------------------------
# --- arguments
# ----------------------------------------------------------

options=''
task=''
locations=''

verbose=''
keep_old_files='false'

# extract settings
for arg in "$@" ; do
  case ${arg} in
    -h | --help    ) DisplayUsage  ;;
    -* ) # option
      # add to option list:
      options="${options} ${arg}" 
      # special options:
      case ${arg} in
        -v | --verbose        ) verbose=${arg} ;;
        -k | --keep-old-files ) keep_old_files='true' ;;
      esac
      ;;
    * ) # task or (one of the) location(s)
      if [ -z "${task}" ]; then
        task=${arg}
      else
        # default protocol is 'file:'
        if [[ ${arg} != *:* ]] ; then
          location="file:${arg}"
        else
          location=${arg}
        fi
        # add to location list:
        locations="${locations} ${location}"
      fi
      ;;
  esac
done

# not complete ?
if [ -z "${task}" ]; then
  err "$prog - ERROR - no task specified"
  errit 1
fi


# ----------------------------------------------------------
# --- settings
# ----------------------------------------------------------

# search gnu tar or use another recent tar version:
gtar='/no/tar'
test ! -x ${gtar}  &&  gtar='/usr/local/bin/gtar'               # linux
test ! -x ${gtar}  &&  gtar='/usr/freeware/bin/tar'             # teras
test ! -x ${gtar}  &&  gtar='/usr/local/bin/tar-1.15'           # hpcf
test ! -x ${gtar}  &&  gtar='tar'                               # default


# ----------------------------------------------------------
# --- begin
# ----------------------------------------------------------

test ${verbose}  &&  echo "$prog - $prog $*"

# test task:
case ${task} in


  # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  # list files
  # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

  'list' )
  
    # empty ? then list current directory
    test -z "${locations}"  &&  locations='file:.'
  
    # loop over locations
    for loc in ${locations} ; do
    
      # split in location protocol and location name:
      protocol=`echo ${loc} | cut -d ':' -f 1`
      filespec=`echo ${loc} | cut -d ':' -f 2-`
  
      # call protocol specific script:
      case ${protocol} in
        file ) ${bindir}/gss_file list ${options} ${filespec} ;;
        mos  ) ${bindir}/gss_mos  list ${options} ${filespec} ;;
        cart ) ${bindir}/gss_cart list ${options} ${filespec} ;;
        ecfs ) ${bindir}/gss_ecfs list ${options} ${filespec} ;;
        ec   ) ${bindir}/gss_ec   list ${options} ${filespec} ;;
        ftp  ) ${bindir}/gss_ftp  list ${options} ${filespec} ;;
        * ) err "$prog - ERROR - unsupported location protocol '${protocol}' for task '${task}'."
            errit 1;;
      esac  # protocols
    
    done  # loop over locations

    ;;  


  # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  # file exists ?
  # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

  'exist' )
  
    # only one file specification:
    loc=''
    for aloc in ${locations} ; do 
      if [ -z "${loc}" ]; then
        loc=${aloc}
      else
        err "$prog - ERROR - task '${task}' requires single location only."
        errit 1
      fi
    done
    if [ -z "${loc}" ]; then
      err "$prog - ERROR - task '${task}' requires location."
      errit 1
    fi
    
    # split in protocol and file specification:
    protocol=`echo ${loc} | cut -d ':' -f 1`
    filespec=`echo ${loc} | cut -d ':' -f 2-`
    
    # call protocol specific script:
    case ${protocol} in

      file | mos | cart | ecfs | ec | ftp )
      
        # extract options
        echo_status=''
        for option in ${options} ; do
          case ${option} in
            -e | --echo  ) echo_status='true' ;;
            -v | --verbose ) ;;
            -* ) err "$prog - unknown option '${option}' for task '${task}'"
                 errit 1 ;;
          esac
        done

        # list file; return status 0 if no error, non zero otherwise
        if ( ${script} list ${loc} > /dev/null 2>&1 ) ; then
          # found !
          if [ ${echo_status} ]; then echo '1' ; else exit 0 ; fi
        else
          # not found ...
          if [ ${echo_status} ]; then echo '0' ; else exit 1 ; fi
        fi
        
        ;;

      * ) err "$prog - ERROR - unsupported protocol '${protocol}' for task '${task}'."
          errit 1 ;;
    esac  # protocols

    ;;

  
  # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  # copy files
  # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

  'copy' )
  
    # extract source and destination location:
    source_loc=''
    dest_loc=''
    for loc in ${locations} ; do 
      if [ -z "${source_loc}" ]; then
        source_loc=${loc}
      elif [ -z "${dest_loc}" ]; then
        dest_loc=${loc}
      else
        err "$prog - ERROR - task '${task}' requires source and destination locations only."
        errit 1
      fi
    done
    if [ -z "${dest_loc}" ]; then
      err "$prog - ERROR - task '${task}' requires source and destination locations."
      errit 1
    fi
    
    # split in protocol and file specification:
    source_protocol=`echo ${source_loc} | cut -d ':' -f 1`
    source_filespec=`echo ${source_loc} | cut -d ':' -f 2-`
    dest_protocol=`echo ${dest_loc} | cut -d ':' -f 1`
    dest_filespec=`echo ${dest_loc} | cut -d ':' -f 2-`
    
    # destination '.' ? set to filename
    test "${dest_filespec}" = "."  &&  dest_filespec=`basename ${source_filespec}`
    
    # if either source or destination is a local file or directory,
    # the task specific scripts will handle it:
    # a copy between two non-file locations requires intermediate storage:
    if [ "${source_protocol}" = "file" ] ; then

      # call destination protocol specific script:
      case ${dest_protocol} in
        file ) ${bindir}/gss_file put ${options} ${source_filespec} ${dest_filespec} ;;
        mos  ) ${bindir}/gss_mos  put ${options} ${source_filespec} ${dest_filespec} ;;
        cart ) ${bindir}/gss_cart put ${options} ${source_filespec} ${dest_filespec} ;;
        ecfs ) ${bindir}/gss_ecfs put ${options} ${source_filespec} ${dest_filespec} ;;
        ec   ) ${bindir}/gss_ec   put ${options} ${source_filespec} ${dest_filespec} ;;
        ftp  ) ${bindir}/gss_ftp  put ${options} ${source_filespec} ${dest_filespec} ;;
        * ) 
          err "$prog - ERROR - unsupported protocol '${protocol}' for task '${task}'."
          errit 1
          ;;
      esac  # protocols

    elif [ "${dest_protocol}" = "file" ] ; then

      # call source protocol specific script:
      case ${source_protocol} in
        file ) ${bindir}/gss_file get ${options} ${source_filespec} ${dest_filespec} ;;
        mos  ) ${bindir}/gss_mos  get ${options} ${source_filespec} ${dest_filespec} ;;
        cart ) ${bindir}/gss_cart get ${options} ${source_filespec} ${dest_filespec} ;;
        ecfs ) ${bindir}/gss_ecfs get ${options} ${source_filespec} ${dest_filespec} ;;
        ec   ) ${bindir}/gss_ec   get ${options} ${source_filespec} ${dest_filespec} ;;
        ftp  ) ${bindir}/gss_ftp  get ${options} ${source_filespec} ${dest_filespec} ;;
        * ) 
          err "$prog - ERROR - unsupported protocol '${protocol}' for task '${task}'."
          errit 1
          ;;
      esac  # protocols

    else
    
      # get file from source under temporary name, put to destination, remove ...
      
      # name of temporary file, almost similar to source; remove if existing:
      tempfile=".`echo ${source_loc} | tr ':/' '__'`"
      rm -f ${tempfile}
      
      # get from source:
      ${script} copy ${options} ${source_loc} ${tempfile}
      
      # put to destination:
      ${script} copy ${options} ${tempfile} ${dest_loc}
      
      # remove temporary file:
      rm -f ${tempfile}

    fi
    
    ;;  


  # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  # link files
  # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

  'link' )
 
    # extract options
    hardlink='true'
    force=''
    new=''
    for option in ${options} ; do
      case ${option} in
        -v | --verbose ) ;;
        -s | --soft  ) hardlink='' ;;
        -f | --force ) force=${option} ;;
        -n | --new   ) new='true' ;;
        -* )
          err "$prog - unknown option '${option}' for task '${task}'"
          errit 1
          ;;
      esac
    done

    # extract source and destination location:
    source_loc=''
    dest_loc=''
    for loc in ${locations} ; do 
      if [ -z "${source_loc}" ]; then
        source_loc=${loc}
      elif [ -z "${dest_loc}" ]; then
        dest_loc=${loc}
      else
        err "$prog - ERROR - task '${task}' requires source and destination locations only."
        errit 1
      fi
    done
    if [ -z "${dest_loc}" ]; then
      err "$prog - ERROR - task '${task}' requires source and destination locations."
      errit 1
    fi
    
    # split in protocol and file specification:
    source_protocol=`echo ${source_loc} | cut -d ':' -f 1`
    source_filespec=`echo ${source_loc} | cut -d ':' -f 2-`
    
    # name of linkfile is same as source with strange characters replaced:
    linkfile=`echo ${source_loc} | tr ':/.@%=\[\]' '________'`
    linkfile=".${linkfile}.gsslink"
    
    # call protocol specific script:
    case ${source_protocol} in

      #
      # create symbolic link
      #

      file ) 

        if [ ! -f ${source_filespec} ]; then
          # file not present; remove existing link if neccessary:
          test -f ${linkfile}  &&  rm -f ${linkfile}
          # create empty source file if necessary ?
          if [ ${new} ]; then
            # create empty target :
            mkdir -p `dirname ${source_filespec}`
            touch ${source_filespec}
          else
            err "$prog - source file not found:"
            err "$prog -   ${filespec}"
            errit 1
          fi
        fi

        # target exists or new empty file; link:
        ln -s ${force} ${source_filespec} ${linkfile} 

        ;;

      #
      # retrieve local copy
      #

      mos | cart | ecfs | ec | ftp )

        # source file present ?
        if ${bindir}/gss exist ${source_loc} ; then
          # file present on storage system; get local copy:
          ${bindir}/gss ${verbose} copy ${force} ${source_loc} ${linkfile}
        else
          # file not present on storage system; remove existing link if neccessary:
          test -f ${linkfile}  &&  rm -f ${linkfile}
          # create empty source file if necessary ?
          if [ ${new} ]; then
            # create empty local 'copy' :
            touch ${linkfile}
          else
            err "$prog - source file not found:"
            err "$prog -   ${source_loc}"
            errit 1
          fi
        fi

        # hard link ? then store original location:
        test "${hardlink}"  &&  echo ${source_loc} > "${linkfile}source"

        ;;

      #
      # error ...
      #

      * ) err "$prog - ERROR - unsupported protocol '${protocol}' for task '${task}'."
          errit 1 ;;

    esac  # protocols
      
    # split in protocol and file specification:
    dest_protocol=`echo ${dest_loc} | cut -d ':' -f 1`
    dest_filespec=`echo ${dest_loc} | cut -d ':' -f 2-`

    # destination should be a file ...
    if [ "${dest_protocol}" != "file" ]; then
      err "$prog - ERROR - destination for task '${task}' should be a local file"
      errit 1
    fi

    # create symbolic link:    
    ln -s -f ${linkfile} ${dest_filespec}

    ;;  


  # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  

  'unlink' )
  
    # extract location:
    loc=`echo ${locations} | cut -d ' ' -f 1`
    if [ -z "${loc}" ]; then
      err "$prog - ERROR - task '${task}' requires location."
      err "$prog -   $0 $*"
      errit 1
    fi
    
    # split in protocol and file specification:
    protocol=`echo ${loc} | cut -d ':' -f 1`
    filespec=`echo ${loc} | cut -d ':' -f 2-`
    
    # local file required ...
    if [ "${protocol}" != "file" ]; then
      err "$prog - ERROR - destination for task '${task}' should be a local file"
      errit 1
    fi
    
    # symbolic link required ...
    if [ ! -h ${filespec} ]; then
      err "$prog - ERROR - destination for task '${task}' should be a symbolic link"
      errit 1
    fi
    
    # linked file:
    #linkfile=`readlink ${filespec}`
    linkfile=`ls -l ${filespec} | cut -d '>' -f 2 | tr -d ' '`
    
    # linkfile should have special name ...
    case ${linkfile} in
      .*.gsslink ) ;;
      * ) err "$prog - ERROR - linked file should be named '.*.gsslink', not:"
          err "$prog - ERROR -   ${linkfile}"
          errit 1 ;;
    esac
    
    # hard link ? then restore
    linkfile_source="${linkfile}source"
    if [ -f ${linkfile_source} ]; then
      # original location:
      linkfile_loc=`cat ${linkfile_source}`
      # restore:
      ${bindir}/gss ${verbose} copy --force ${linkfile} ${linkfile_loc}
      # remove source info:
      rm -f ${linkfile_source}
    fi
    
    # remove linkfile:
    rm -f ${linkfile}
    
    # remove local file:
    rm -f ${filespec}
    
    ;;  


  # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  # operations on linked tar files
  # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

  'tar-extract' )
  
    # extract source and destination location:
    tarfile_loc=''
    outfile_loc=''
    for loc in ${locations} ; do 
      if [ -z "${tarfile_loc}" ]; then
        tarfile_loc=${loc}
      elif [ -z "${outfile_loc}" ]; then
        outfile_loc=${loc}
      else
        err "$prog - ERROR - task '${task}' requires tarfile and tarred file locations only."
        errit 1
      fi
    done
    if [ -z "${tarfile_loc}" ]; then
      err "$prog - ERROR - task '${task}' requires tarfile and optional tarred file locations."
      errit 1
    fi
    
    # split tarfile location in protocol and file specification:
    tarfileprot=`echo ${tarfile_loc} | cut -d ':' -f 1`
    tarfilespec=`echo ${tarfile_loc} | cut -d ':' -f 2-`
    
    # local file required ...
    if [ "${tarfileprot}" != "file" ]; then
      err "$prog - ERROR - tarfile for task '${task}' should be a local file"
      errit 1
    fi
    
    # tarred file ?
    if [ -z "${outfile_loc}" ]; then 
      outfile_filespec=''
    else
      # split output file location in protocol and specification:
      outfile_protocol=`echo ${outfile_loc} | cut -d ':' -f 1`
      outfile_filespec=`echo ${outfile_loc} | cut -d ':' -f 2-`
      # output file should have 'file:' protocol ...
      if [ "${outfile_protocol}" != "file" ]; then
        err "$prog - ERROR - task '${task}' requires 'file:' protocol for outfile"
        errit 1
      fi
    fi
    
    test ${verbose}  &&  echo "$prog -   extract ..."
    #
    # extract from tarfile:
    #     x, --extract            : extract files from an archive
    #    -f, --file=F             : use archive file or device F
    # by default, an extracted file remains the archived time;
    # prevent that the times of previously touched files are overwritten:
    #    -k, --keep-old-files     : keep existing files
    # (if a file already exists, an error message is issued)
    # ensure that new files are touched:
    #    -m, --touch              : don't extract file modified time
    #
    # redirection for standard output and error:
    stdout=".tmp.out.$$"

    # untar; trap error status:
    # pls: Remove the -k option => was crashing on c2a 
    if ( ${gtar} xmf ${tarfilespec} ${outfile_filespec} > ${stdout} 2>&1 ); then
      # ok
      rm -f ${stdout}
    else
      # some errors; probably about overwriting existing files
      echo "$prog -     WARNING : errors from tar (overwriting existing files?); written to: ${stdout}"
      echo "$prog-      WARNING from command : ${gtar} xkmf ${tarfilespec} ${outfile_filespec}"
    fi
    
    ;;  


  # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  

  'tar-update' )
  
    # extract source and destination location:
    tarfile_loc=''
    infile_loc=''
    for loc in ${locations} ; do 
      if [ -z "${tarfile_loc}" ]; then
        tarfile_loc=${loc}
      elif [ -z "${infile_loc}" ]; then
        infile_loc=${loc}
      else
        err "$prog - ERROR - task '${task}' requires tarfile and input file locations only."
        errit 1
      fi
    done
    if [ -z "${infile_loc}" ]; then
      err "$prog - ERROR - task '${task}' requires tarfile and input file locations."
      errit 1
    fi
    
    # split tarfile location in protocol and file specification:
    tarfileprot=`echo ${tarfile_loc} | cut -d ':' -f 1`
    tarfilespec=`echo ${tarfile_loc} | cut -d ':' -f 2-`
    
    # local file required ...
    if [ "${tarfileprot}" != "file" ]; then
      err "$prog - ERROR - tarfile for task '${task}' should be a local file"
      errit 1
    fi
    
    # tarfile should be present:
    if [ ! -f ${tarfilespec} ]; then
      err "$prog - ERROR - tar file not found :"
      err "$prog - ERROR -   ${tarfilespec}"
      errit 1
    fi
        
    # split input file in protocol and specification:
    inprotocol=`echo ${infile_loc} | cut -d ':' -f 1`
    infilespec=`echo ${infile_loc} | cut -d ':' -f 2-`
    
    # input file should have 'file:' protocol ...
    if [ "${inprotocol}" != "file" ]; then
      err "$prog - ERROR - task '${task}' requires 'file:' protocol for infile"
      errit 1
    fi
    
    # old version present in tarfile ? 
    if ( ${gtar} tf ${tarfilespec} ${infilespec} > /dev/null 2>&1 ) ; then
      # keep old version in tarfile ? 
      if [ "${keep_old_files}" = "true" ]; then
        # keep old; do not add new file
        test ${verbose}  &&  echo "$prog -   keep old ..."
      else
        # sometimes long options not available; trap error status:
        if ! (
          # remove old file:
          test ${verbose}  &&  echo "$prog -   remove old ..."
          ${gtar} --delete --file=${tarfilespec} ${infilespec}
          # add to tarfile:
          test ${verbose}  &&  echo "$prog -   add ..."
          ${gtar} rf ${tarfilespec} ${infilespec} \
        ); then
          # something crashed, now simply replace with update:
          test ${verbose}  &&  echo "$prog -   update ..."
          ${gtar} u -v -f ${tarfilespec} ${infilespec}
        fi
      fi # keep old files?
    else
      # add to (eventually empty) tarfile:
      #${gtar} --append --file=${tarfilespec} ${infilespec}
      #PLS: tar does not work (although it was working before) with not valid
      # archive. So now check that the file has non-zero size (I assume that zero
      # size files have not been created by a tar command basically)
      if [ -s ${tarfilespec} ]; then 
        test ${verbose}  &&  echo "$prog -   replace ..."
        ${gtar} rf ${tarfilespec} ${infilespec}
      else
        test ${verbose}  &&  echo "$prog -   create ..."
        ${gtar} cf ${tarfilespec} ${infilespec}
      fi
    fi
    
    ;;  


  # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  # limit directory usage
  # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

  
  'limit')
  
    # extract source and destination location:
    max_usage=''
    bufdir_loc=''
    for loc in ${locations} ; do 
      if [ -z "${max_usage}" ]; then
        max_usage=`echo ${loc} | cut -d ':' -f 2-`
      elif [ -z "${bufdir_loc}" ]; then
        bufdir_loc=${loc}
      else
        err "$prog - ERROR - task '${task}' requires directory and max usage only."
        errit 1
      fi
    done
    if [ -z "${bufdir_loc}" ]; then
      err "$prog - ERROR - task '${task}' requires directory and max usage."
      errit 1
    fi
    
    # no infinite usage ? then remove files if necessary:
    if [[ ${max_usage} != [Ii][Nn][Ff] ]]; then

      # convert max usage to kB :
      mu=`echo ${max_usage} | tr -d 'kKmMgG'`
      case ${max_usage} in
        *g | *G ) max_usage_kB=`expr ${mu} \* 1000000` ;;
        *m | *M ) max_usage_kB=`expr ${mu} \* 1000` ;;
        * ) max_usage_kB=${mu} ;;
      esac
      test ${verbose}  &&  echo "$prog - max usage  : ${max_usage_kB} kB"

      # split in protocol and file specification:
      protocol=`echo ${bufdir_loc} | cut -d ':' -f 1`
      bufdir=`echo ${bufdir_loc} | cut -d ':' -f 2-`

      # local directory required ...
      if [ "${protocol}" != "file" ]; then
        err "$prog - ERROR - buffer directory for task '${task}' should be local"
        errit 1
      fi

      # buffer exists ?
      if [ ! -d ${bufdir} ]; then
        err "$prog - ERROR - directory to limit not found:"
        err "$prog - ERROR -   ${bufdir_loc}"
        errit 1
      fi

      # current usage:
      curr_usage_kB=`du -sk ${bufdir} | cut -f 1`

      test ${verbose}  &&  echo "$prog - curr usage : ${curr_usage_kB} kB"

      # disk usage exceeds maximum ? then remove some files:
      if [ ${curr_usage_kB} -gt ${max_usage_kB} ]; then

        # loop over files in buffer;
        # sort on time (-t), oldest first (-r);
        # for symbolic links, show times for linked files (-L)
        #   (if a symbolic link is touched, only time of linked file changes)
        for file in `ls -t -r -L ${bufdir}` ; do
        
          # tmp directory ? must be old scratch ...
          test -d ${file}  &&  rm -r -f ${file}

          # symbolic link to gsslink ? then remove gsslink too
          if [ -h ${file} ]; then
            #linkfile=`readlink ${file}`
            linkfile=`ls -l ${file} | cut -d '>' -f 2 | tr -d ' '`
            # gss link ?
            if [[ ${linkfile} = .*.gsslink ]]; then
              # update disk usages for linkfile:
              file_usage_kB=`du -k ${bufdir}/${linkfile} | cut -f 1`
              curr_usage_kB=`expr ${curr_usage_kB} - ${file_usage_kB}`
              # source file present ? update disk usage for this file:
              linkfile_source="${linkfile}src"
              if [ -f ${linkfile_source} ]; then
                file_usage_kB=`du -k ${bufdir}/${linkfile_source} | cut -f 1`
                curr_usage_kB=`expr ${curr_usage_kB} - ${file_usage_kB}`
              fi
              # unlink, eventually the file is restored:
              #${bindir}/gss ${verbose} unlink ${file}
              ${bindir}/gss unlink ${file}
            fi
          fi

          # remove file; update disk usage:
          if [ -f ${file} ]; then
            file_usage_kB=`du -k ${bufdir}/${file} | cut -f 1`
            curr_usage_kB=`expr ${curr_usage_kB} - ${file_usage_kB}`
            rm -f ${bufdir}/${file}
          fi

          test ${verbose}  &&  echo "$prog -  ${curr_usage_kB}  ${file_usage_kB}  ${file}"

          # current usage smaller than maximum ? then leave
          test ${curr_usage_kB} -le ${max_usage_kB}  &&  break

        done  # loop over old files

      fi   # usage > max
          
    fi  # max usage < Inf
        
    ;;


  # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
  # task ???
  # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

  * )
    err "$prog - ERROR - unsupported task '${task}'"
    errit 1
    ;;
  

esac  # tasks


# ----------------------------------------------------------
# --- end
# ----------------------------------------------------------

test ${verbose}  &&  echo "$prog - end"

# ok
exit 0