forcing-perturbation/transfer/data_transfer.sh

#!/bin/bash
#
# nohup ./data_transfer.sh >& expid.log &
#

##################################
####  User Defined Variables  #### 
##################################

# User must fill-in
arch="" # choose platform (ithaca, mn-*, cfs-aux-4, ecmwf, ht-*, ar-*)
model="" # choose model (ecearth, nemo)
expid="" # supply expid: e.g. xxxx
sdates="" # supply list of start dates: e.g. 19601101 19651101 ...
members="" # supply list of members: e.g. fc0 fc1 fc2 ...
remove=TRUE # remove output from source by default (if user want to kee the model output at HPC make it FALSE)

# User may change
types="logfiles MM PP diags REST restart ICM CMOR"
dst1=/esnas/exp/$model # store logfiles, MM*
dst2=/esnas/exp/$model/restartfiles # store REST*
dst3=/esnas/exp/$model/rawfiles # store ICM*, ORCA, EXPID_* (raw data)
dst4=/esnas/exp/$model/cmorfiles # store CMOR*

#####  End of User Defined Variables  ####

#################################
####  User Defined Funtions  #### 
#################################

function transfer1(){
 set -e
 dst=$1/$expid/$s/$m/$2
 if [[ $remove == TRUE ]]; then
  RSYNC="rsync -acv --remove-sent-files"
 else
  RSYNC="rsync -acv"
 fi
 for l in $list; do
  echo "tranfering file ... ${arch}:$l $dst"
  $RSYNC ${arch}:${l} $dst
  echo "done."
  echo
  echo
 done
 set +e
}

function transfer2(){
 dst=$1/$expid/$s/$m/$2
 for l in $list; do
  echo "tranfering file ... $src/$expid/$s/$m/$2/$l $dst/$l"
  ecaccess-file-get $src/$expid/$s/$m/$2/$l $dst/$l
  if [[ $? -eq 0 && $remove == TRUE ]]; then
   ecaccess-file-delete -force $src/$expid/$s/$m/$2/$l
  fi
  echo "done."
  echo
  echo
 done
}

####  End of the User Defined Functions  #### 

###################################
####  Main Part of the Script  ####
###################################

date
echo "platform is: $arch"
echo "model is: $model"
echo "expid is: $expid"
echo "start dates: $sdates"
echo "members per start date: $members"
echo "remove output from HPC disks: $remove"

for s in $sdates; do
 for m in $members; do

  rmtlist="ssh $arch ls -1"
  transfer="transfer1"

  case $arch in
   ithaca)
    src=/share/data/cfu/exp
   ;;
   mn-*)
    src=/gpfs/scratch/ecm86/ecm86010/exp
   ;;
   cfs-aux-4)
    src=/cfs/klemming/nobackup/a/asifsami/exp
   ;;
   ecmwf*)
    rmtlist="ecaccess-file-dir"
    transfer="transfer2"
    src=ec:/c3m/exp
   ;;
   ht-*)
    src=/work/pr1u1011/pr1u1011/pr1e1001/exp
   ;;
   ar-*)
    src=/work/pr1u1011/pr1u1011/shared/exp
   ;;
   *)
    echo "$arch is not a valid machine"
   ;;
  esac

  for typ in $types; do
   case $typ in
    logfiles)
     lswc=`$rmtlist $src/$expid/$s/$m/logfiles* | wc -l`
     if [[ $lswc -gt 0 ]]; then
      mkdir -p $dst1/$expid/$s/$m
      list=`$rmtlist $src/$expid/$s/$m/logfiles*`
      $transfer $dst1 ""
     fi
    ;;
    MM|PP|diags)
     lswc=`$rmtlist $src/$expid/$s/$m/outputs/${typ}* | wc -l`
     if [[ $lswc -gt 0 ]]; then
      mkdir -p $dst1/$expid/$s/$m/outputs
      list=`$rmtlist $src/$expid/$s/$m/outputs/${typ}*`
      $transfer $dst1 outputs
     fi
    ;;
    REST|restart)
     lswc=`$rmtlist $src/$expid/$s/$m/restarts/${typ}* | wc -l`
     if [[ $lswc -gt 0 ]]; then
      mkdir -p $dst2/$expid/$s/$m/restarts
      list=`$rmtlist $src/$expid/$s/$m/restarts/${typ}*`
      $transfer $dst2 restarts
     fi
    ;;
    ICM|ORCA|$(eval echo ${expid}_))
     lswc=`$rmtlist $src/$expid/$s/$m/outputs/${typ}* | wc -l`
     if [[ $lswc -gt 0 ]]; then
      mkdir -p $dst3/$expid/$s/$m/outputs
      list=`$rmtlist $src/$expid/$s/$m/outputs/${typ}*`
      $transfer $dst3 outputs
     fi
    ;;
    CMOR)
     lswc=`$rmtlist $src/$expid/$s/$m/outputs/${typ}* | wc -l`
     if [[ $lswc -gt 0 ]]; then
      mkdir -p $dst4/$expid/$s/$m/outputs
      list=`$rmtlist $src/$expid/$s/$m/outputs/${typ}*`
      $transfer $dst4 outputs
     fi
    ;;
    *)
     echo "$typ is not a valid data type"
    ;;
   esac
  done

 done
done
date

####  End of the Main Part of Script  ####