#!/usr/bin/env bash set -exf ########################################### # get USER ID on BSC for differnet CFU users # # You have to be able to login in to BSC with the command "ssh mn" without using password ########################################### case $USER in fdoblasreyes) MN_USER='ecm86998' ;; huidu) MN_USER='ecm86133' ;; jgarcia) MN_USER='ecm86734' ;; masif) MN_USER='ecm86010' ;; omula) MN_USER='ecm86843' ;; vguemas) MN_USER='ecm86859' ;; acarrassi) MN_USER='ecm86014' ;; esac ####################### ## get arguments ###################### while getopts e:s:m:t:f:g: option do case $option in e) exp_id=$OPTARG;; s) sdate=$OPTARG;; m) mem=$OPTARG;; t) data_type=$OPTARG;; f) file=$OPTARG;; g) filetype=$OPTARG;; \?) exit 1;; esac done ###################### ## functions ###################### get_sdate(){ sdate_lst=`ssh mn ssh login8 ls $BASE_DIR/$1` } get_member(){ member_lst=`ssh mn ssh login8 ls $BASE_DIR/$1/$2` } grab_data(){ # this will download all the data for one specific member sdate=$1 mem=$2 file_type=$3 HSM_DIR=/HSM/ecm86/exp/${exp_id}/${sdate}/${mem}/outputs TARGET_DIR=/cfunas/exp/ecearth/${exp_id}/${sdate}/${mem}/outputs mkdir -p ${TARGET_DIR} file_list=`ssh mn ssh login8 find $HSM_DIR -type f -iname "ICM${file_type}*.grb"` if [[ ${#file_list[@]} == 0 ]]; then #IV echo "There are no elements in the given PATH: $1" return 1 fi #IV for f in ${file_list[@]};do f1=`basename $f` ssh mn rsync login7:$f $MN_CONTAINER/ # HSM to SCRATCH of BSC rsync mn:$MN_CONTAINER/$f1 ${TARGET_DIR}/ # SCRATCH of BSC to cfunas ssh mn rm $MN_CONTAINER/$f1 # clean BSC done #loop for files } grab_one_file(){ sdate=$1 mem=$2 f=$3 HSM_DIR=/HSM/ecm86/exp/${exp_id}/${sdate}/${mem}/outputs TARGET_DIR=/cfunas/exp/ecearth/${exp_id}/${sdate}/${mem}/outputs mkdir -p ${TARGET_DIR} ssh mn rsync login7:$HSM_DIR/$f $MN_CONTAINER/ # HSM to SCRATCH of BSC rsync mn:$MN_CONTAINER/$f ${TARGET_DIR}/ # SCRATCH of BSC to cfunas ssh mn rm $MN_CONTAINER/$f # clean BSC } print_help(){ echo echo "File type (netcdf or grb) have to be specified: -g , where filetype is either netcdf or grb" echo "USAGE: $(basename $0) -g -e -s -m -t " echo "If is grb file, has to be either sh or gg" echo "OR: $(basename $0) -f " echo "if no filename (which have to including the information of sdates and member) is specify, is required, in this case, all outputs, restart and log files will be downloaded" echo " can be mm/mmo/mma/rest/resta/resto/rest/log/diags/sh/gg" echo } if [[ $filetype == '' ]];then print_help exit 1 else case $filetype in nc) if [[ $file == '' ]] && [[ $exp_id == '' ]];then print_help exit 1 fi ;; grb) if [[ $file != '' ]]; then if [[ $sdate == '' ]] || [[ $mem == '' ]];then echo echo "You have to specify both the starting date and member for this grb file" echo fi fi ;; esac fi ################## # Constants # ################## BASE_DIR='/HSM/ecm86/exp' MN_CONTAINER="/gpfs/scratch/ecm86/${MN_USER}/container/" HSM_NODE='login8' HSM_DIR=/${BASE_DIR}/${exp_id}/${sdate}/${mem}/ ##################### # create directoris #yy ##################### ssh mn "mkdir -p ${MN_CONTAINER}" ################## # main scripts ################## case $filetype in grb) if [[ $file == '' ]]; then if [[ $sdate == '' ]];then #I get_sdate ${exp_id} if [[ $mem == '' ]];then #II for sdate in ${sdate_lst[@]};do get_member ${exp_id} ${sdate} for mem in ${member_lst};do grab_data ${sdate} ${mem} ${data_type} done # loop for member done #loop for starting dates else for sdate in ${sdate_lst[@]};do grab_data ${sdate} ${mem} ${data_type} done #loop for starting dates fi #II else #I if [[ $mem == '' ]];then get_member ${exp_id} $sdate for mem in ${member_lst};do grab_data ${sdate} ${mem} ${data_type} done # loop for member else grab_data ${sdate} ${mem} ${data_type} fi fi #I else if [[ $sdate == '' ]] || [[ $mem == '' ]];then echo "You have to specify both sdate and mem in this casei" return 1 else grab_one_file $sdate $mem $file fi fi ;; nc) if [ -z "$file" ]; then file_list=`ssh mn ssh login8 find $HSM_DIR -type f -iname "$data_type*.tar"` else exp_id=`echo $file|cut -f2 -d'_'` file_list=`ssh mn ssh login8 find $BASE_DIR/${expid} -type f -iname "$file"` fi # file_list=`ssh mn ssh login8 find $HSM_DIR -type f -iname "$data_type*"` if [[ ${#file_list[@]} == 0 ]]; then echo "here are no elements in the given MMO PATH: $1" return 1 fi for f in ${file_list[@]};do f1=`basename $f` type=`echo $f1|cut -f1 -d'_'` sdate=`echo $f1|cut -f3 -d'_'` mem=`echo $f1|cut -f4 -d'_'` echo $type $sdate $mem case $type in 'RESTA'|'RESTO') TARGET_DIR=/cfunas/exp/ecearth/${exp_id}/${sdate}/${mem}/restarts ;; 'MMA'|'MMO'|'diags') TARGET_DIR=/cfunas/exp/ecearth/${exp_id}/${sdate}/${mem}/outputs ;; 'logfile') TARGET_DIR=/cfunas/exp/ecearth/${exp_id}/${sdate}/${mem}/log ;; esac if [[ ! -d "$TARGET_DIR" ]]; then mkdir -p $TARGET_DIR fi ssh mn rsync login7:$f $MN_CONTAINER/ # HSM to SCRATCH of BSC rsync mn:$MN_CONTAINER/$f1 ${TARGET_DIR}/ # SCRATCH of BSC to cfunas ssh mn rm $MN_CONTAINER/$f1 # clean BSC done ;; esac