forcing-perturbation/transfer/download_hsm

#!/usr/bin/env bash
set -exf
###########################################
# get USER ID on BSC for differnet CFU users #
# You have to be able to login in to BSC with the command "ssh mn" without using password
###########################################
case $USER in
 fdoblasreyes)
 MN_USER='ecm86998'
 ;;
 huidu)
 MN_USER='ecm86133'
 ;;
 jgarcia)
 MN_USER='ecm86734'
 ;;
 masif)
 MN_USER='ecm86010'
 ;;
 omula)
 MN_USER='ecm86843'
 ;;
 vguemas)
 MN_USER='ecm86859'
 ;;
 acarrassi)
 MN_USER='ecm86014'
 ;;
esac

#######################
## get arguments
######################


while getopts e:s:m:t:f:g: option
do
  case $option in
    e) exp_id=$OPTARG;;
    s) sdate=$OPTARG;;
    m) mem=$OPTARG;;
    t) data_type=$OPTARG;;
    f) file=$OPTARG;;
    g) filetype=$OPTARG;; 
    \?) exit 1;;
  esac
done

######################
## functions 
######################

get_sdate(){
sdate_lst=`ssh mn ssh login8 ls $BASE_DIR/$1`
}

get_member(){
member_lst=`ssh mn ssh login8 ls $BASE_DIR/$1/$2`
}

grab_data(){  # this will download all the data for one specific member

sdate=$1
mem=$2
file_type=$3

HSM_DIR=/HSM/ecm86/exp/${exp_id}/${sdate}/${mem}/outputs
TARGET_DIR=/cfunas/exp/ecearth/${exp_id}/${sdate}/${mem}/outputs

mkdir -p ${TARGET_DIR}

file_list=`ssh mn ssh login8 find $HSM_DIR -type f -iname "ICM${file_type}*.grb"`

if [[ ${#file_list[@]} == 0 ]]; then #IV
 echo "There are no elements in the given PATH: $1"
 return 1
fi  #IV

for f in ${file_list[@]};do
 f1=`basename $f`
 ssh mn rsync login7:$f $MN_CONTAINER/ # HSM to SCRATCH of BSC
 rsync mn:$MN_CONTAINER/$f1 ${TARGET_DIR}/ # SCRATCH of BSC to cfunas
 ssh mn rm $MN_CONTAINER/$f1  # clean BSC
done #loop for files

}

grab_one_file(){

sdate=$1
mem=$2
f=$3
HSM_DIR=/HSM/ecm86/exp/${exp_id}/${sdate}/${mem}/outputs
TARGET_DIR=/cfunas/exp/ecearth/${exp_id}/${sdate}/${mem}/outputs
mkdir -p ${TARGET_DIR}
ssh mn rsync login7:$HSM_DIR/$f $MN_CONTAINER/ # HSM to SCRATCH of BSC
rsync mn:$MN_CONTAINER/$f ${TARGET_DIR}/ # SCRATCH of BSC to cfunas
ssh mn rm $MN_CONTAINER/$f  # clean BSC

}


print_help(){
echo 
echo "File type (netcdf or grb) have to be specified: -g <filetype>, where filetype is either netcdf or grb"
echo "USAGE: $(basename $0) -g <filetype> -e <exp_id> -s <startdate> -m <mem> -t <data_type>"
echo "If <filetype> is grb file, <data_type> has to be either sh or gg"
echo "OR: $(basename $0) -f <filename>"
echo "if no filename (which have to including the information of sdates and member) is specify, <exp_id> is required, in this case, all outputs, restart and log files will be downloaded"
echo "<data_type> can be mm/mmo/mma/rest/resta/resto/rest/log/diags/sh/gg"
echo

}

if [[ $filetype == '' ]];then
  print_help
  exit 1
else
  case $filetype in 
   nc)
    if [[ $file == '' ]] && [[ $exp_id == '' ]];then
    print_help
    exit 1
    fi 
   ;;
   grb)
    if [[ $file != '' ]]; then
      if [[ $sdate == '' ]] || [[ $mem == '' ]];then 
       echo 
       echo "You have to specify both the starting date and member for this grb file"
       echo 
      fi  

    fi
   ;;
  esac

fi

##################
# Constants      #
##################
BASE_DIR='/HSM/ecm86/exp'
MN_CONTAINER="/gpfs/scratch/ecm86/${MN_USER}/container/"
HSM_NODE='login8'
HSM_DIR=/${BASE_DIR}/${exp_id}/${sdate}/${mem}/

#####################
# create directoris #yy
#####################

ssh mn "mkdir -p ${MN_CONTAINER}"

##################
# main scripts 
##################

case $filetype in 

 grb)

if [[ $file == '' ]]; then

 if [[ $sdate == '' ]];then  #I

   get_sdate ${exp_id}

   if [[ $mem == '' ]];then #II
     for sdate in ${sdate_lst[@]};do 

      get_member ${exp_id} ${sdate}
      for mem in ${member_lst};do
       grab_data ${sdate} ${mem} ${data_type}
      done # loop for member

     done #loop for starting dates

   else

    for sdate in ${sdate_lst[@]};do
      grab_data ${sdate} ${mem} ${data_type}
    done #loop for starting dates

  fi #II

 else #I
  
  if [[ $mem == '' ]];then
    get_member ${exp_id} $sdate
     for mem in ${member_lst};do
      grab_data ${sdate} ${mem} ${data_type}
     done # loop for member
  else
    grab_data ${sdate} ${mem} ${data_type}

  fi

 
 fi #I


else 

 if [[ $sdate == '' ]] || [[ $mem == '' ]];then
  echo "You have to specify both sdate and mem in this casei"
  return 1
 else
  
  grab_one_file $sdate $mem $file

 fi


fi


;;

 nc)  

 if [ -z "$file" ]; then 
 file_list=`ssh mn ssh login8 find $HSM_DIR -type f -iname "$data_type*.tar"`
 else
 exp_id=`echo $file|cut -f2 -d'_'`
 file_list=`ssh mn ssh login8 find $BASE_DIR/${expid} -type f -iname "$file"`
 fi

# file_list=`ssh mn ssh login8 find $HSM_DIR -type f -iname "$data_type*"`
 if [[ ${#file_list[@]} == 0 ]]; then
  echo "here are no elements in the given MMO PATH: $1"
  return 1
 fi
 

 for f in ${file_list[@]};do
  
  f1=`basename $f`
  type=`echo $f1|cut -f1 -d'_'`
  sdate=`echo $f1|cut -f3 -d'_'`
  mem=`echo $f1|cut -f4 -d'_'`
  echo $type $sdate $mem
  case $type in 
   'RESTA'|'RESTO')
    TARGET_DIR=/cfunas/exp/ecearth/${exp_id}/${sdate}/${mem}/restarts
    ;;
   'MMA'|'MMO'|'diags')
    TARGET_DIR=/cfunas/exp/ecearth/${exp_id}/${sdate}/${mem}/outputs 
    ;;
    'logfile')
    TARGET_DIR=/cfunas/exp/ecearth/${exp_id}/${sdate}/${mem}/log
    ;;
  esac

  if [[ ! -d "$TARGET_DIR" ]]; then
   mkdir -p $TARGET_DIR
  fi
 
  ssh mn rsync login7:$f $MN_CONTAINER/ # HSM to SCRATCH of BSC
  rsync mn:$MN_CONTAINER/$f1 ${TARGET_DIR}/ # SCRATCH of BSC to cfunas
  ssh mn rm $MN_CONTAINER/$f1  # clean BSC
 done

 ;;

esac