#!/bin/bash
#
# submit.sh
#
# Portable bash script to run LPJ-GUESS version:
# BINARY
# as a parallel job on a multicore system.
#
# Created automatically on DATE
# 
# Usage: 
#
#   1. Copy script to the directory where you want output written.
#      This will be called the RUN DIRECTORY.
#   2. In an editor, set appropriate values for the variables NPROCESS,
#      INSFILE, GRIDLIST and OUTFILES (NB: no space after the = sign):

NPROCESS=15
INSFILE=guess.ins
INPUT_MODULE=cru_ncep
GRIDLIST=gridlist.txt
OUTFILES='*.out'

#      Where:
#      NPROCESS     = number of processes in parallel job
#      INSFILE      = path to ins file from run directory
#      INPUT_MODULE = input module to use
#      GRIDLIST     = path to gridlist file from run directory
#      OUTFILES     = list of LPJ-GUESS output files in single quotes,
#                     and separated by spaces (filenames only, including
#                     extension, no directory.) Shell wildcards are allowed.
#
#   3. Run the script using the command:
#        ./submit.sh
#      or:
#        ./submit.sh [-s <file>] [-i <ins-file>]
#
#      All arguments are optional and interpreted as:
#      file     = filename of a file which can override the variables
#                 above
#      ins-file = instruction file to use, overrides the INSFILE
#                 variable above
#
# Nothing to change past here
########################################################################

# Handle the command line arguments
while getopts ":s:i:" opt; do
    case $opt in
	s ) submit_vars_file=$OPTARG ;;
	i ) ins=$OPTARG ;;
    esac
done

# Override the submit variables with the contents of a file, if given
if [ -n "$submit_vars_file" ]; then
    source $submit_vars_file
fi

# Override INSFILE with the ins-file parameter, if given
if [ -n "$ins" ]; then
    INSFILE=$ins
fi

# Convert INSFILE to an absolute path since we will be starting the
# guess instances from different directories.
# Please note when porting this script: readlink may not be available
# on non-Linux systems. Also, using absolute path names means the
# instruction file needs to be in a place accessible from the nodes.
INSFILE=$(readlink -f "$INSFILE")

GRIDLIST_FILENAME=$(basename $GRIDLIST)

# This function creates the gridlist files for each run by splitting
# the original gridlist file into approximately equal parts.
function split_gridlist {
    # Create empty gridlists first to make sure each run gets one
    for ((a=1; a <= NPROCESS ; a++)) 
    do
      echo > run$a/$GRIDLIST_FILENAME
    done

    # Figure out suitable number of lines per gridlist, get the number of
    # lines in original gridlist file, divide by NPROCESS and round up.
    local lines_per_run=$(wc -l $GRIDLIST | \
	awk '{ x = $1/'$NPROCESS'; d = (x == int(x)) ? x : int(x)+1; print d}')

    # Use the split command to split the files into temporary files
    split --suffix-length=4 --lines $lines_per_run $GRIDLIST tmpSPLITGRID_

    # Move the temporary files into the runX-directories
    local files=$(ls tmpSPLITGRID_*)
    local i=1
    for file in $files
    do
      mv $file run$i/$GRIDLIST_FILENAME
      i=$((i+1))
    done
}

# Create header of progress.sh script

echo "##############################################################" > progress.sh
echo "# PROGRESS.SH" >> progress.sh
echo "# Upload current guess.log files from local nodes and check" >> progress.sh
echo "# Usage: sh progress.sh" >> progress.sh
echo >> progress.sh

# Create a run subdirectory for each process and clean up

for ((a=1; a <= NPROCESS ; a++))
do
  mkdir -p run$a
  cd run$a ; rm -f guess.log ; rm -f $GRIDLIST_FILENAME ; cd ..
  echo "echo '********** Last few lines of ./run${a}/guess.log: **********'" >> progress.sh
  echo "tail ./run${a}/guess.log" >> progress.sh
done

split_gridlist

# The failed.txt file lists the processes that didn't finish properly.
# If it exists, remove it before running again.
rm -f failed.txt

# Start all processes in the background
for ((a=1; a <= NPROCESS ; a++))
do
  echo "Starting process $a..."
  cd run$a
  ( if ! BINARY -input $INPUT_MODULE $INSFILE 1>> ../stdout.txt 2>> ../stderr.txt ; then
      echo -n $a" " >> ../failed.txt
  fi ) &
  cd ..
done

# Install a clean up trap to handle Ctrl-C from the user.
# kill 0 kills all sub processes (so all GUESS instances).
trap 'kill 0; exit 1' INT

# Wait for them to finish
echo "Done!"
echo
echo "Check progress with the progress script, either in a different shell, or"
echo "by placing this job in the background."
echo
echo "Waiting for all processes to finish..."

wait

echo "Done!"
echo

# Don't append if one of the sub processes failed
if [ -f failed.txt ]; then
    echo "The following processes failed: " $(cat failed.txt)
    echo "Check the log files for these processes for more information."

    exit 1
fi

# Append the results
function append_files {
    local number_of_jobs=$1
    local file=$2

    cp run1/$file $file

    local i=""
    for ((i=2; i <= number_of_jobs; i++))
    do
      if [ -f run$i/$file ]; then
        cat run$i/$file | awk 'NR!=1 || NF==0 || $1 == $1+0 { print $0 }' >> $file
      fi
    done
}

pushd run1 &> /dev/null
outfiles_unexpanded=$OUTFILES
outfiles_expanded=$(echo $outfiles_unexpanded)
popd &> /dev/null

echo "Appending results..."

for file in $outfiles_expanded
do
  append_files $NPROCESS $file
done

echo "Done!"