ecearth3/sources/tm5mp/bin/submit_tm5

#! /usr/bin/env python3

#
# job chain:
#
#   myrun_001_init.jb
#     build/bin/submit_tm5_step_init rcfile
#   myrun_001_run.jb
#     build/bin/submit_tm5_step_run './tm5.x rcfile'
#   myrun_001_done.jb
#     build/bin/submit_tm5_step_done rcfile
#
#   myrun_002_init.jb
#     ...
#
# or for a loadleveler job:
#
#   myrun_001.jb
#     build/bin/submit_tm5_step_init rcfile
#     build/bin/submit_tm5_step_run './tm5.x rcfile'
#     build/bin/submit_tm5_step_done rcfile
#
#   myrun_002.jb
#     ...
#

# -----------------------------------------------
# external
# -----------------------------------------------

# standard modules:
import sys
import os
import optparse
import logging
import datetime


# -----------------------------------------------
# defaults
# -----------------------------------------------

# display info messages ?
verbose_default = False

# where to search for additional scripts ?
# NOTE: this line will be (has been?) replaced by 'pycasso.py/Main'
# with the actual path to the build directory:
pypath_default = os.path.join( os.pardir, 'build', 'bin' )


# -----------------------------------------------
# arguments (1)
# -----------------------------------------------

# extract arguments from sys.argv array:
#   0 = name of calling script, 1: = actual arguments
script = sys.argv[0]
args = sys.argv[1:]

# set text for 'usage' help line:
usage = "%prog [options] [-e|--exec=]<executable>] [-r|--rcfile=]<rcfile> ..."

# extra help text
epilog = """
Arguments that are no flags are supposed to be file names
and clasified according to their extension as either
an executable (.x) or rc file (.rc) .
"""


# initialise the option parser:
#parser = optparse.OptionParser(usage=usage,epilog=epilog)
parser = optparse.OptionParser(usage=usage)  # not all python version accept epilog yet ...

# define verbose option:
parser.add_option( "-v", "--verbose", 
                     help="print extra logging messages to standard output (%s)" % str(verbose_default),
                     dest="verbose", action="store_true", default=verbose_default )
parser.add_option( "-p", "--py-path", 
                     help="path to additional submit scripts (%s)" % pypath_default,
                     dest="pypath", action="store", default=pypath_default )
parser.add_option( "-b", "--background", 
                     help="""Run executable in background. Standard output and error are redirected.""",
                     dest="submit_to", action="store_const", const='background' )
parser.add_option( "-q", "--queue", 
                     help="submit executable to a queue system",
                     dest="submit_to", action="store_const", const='queue' )
parser.add_option( "-s", "--submit-option", 
                     help="""Extra submit option(s) passed to queue scheduler. 
                     Example for BSUB:                               
                       -K : (keep) wait for end of run before return.""",
                     dest="submit_options", action="store" )
parser.add_option( "-d", "--debugger", 
                     help="run executable in debuger",
                     dest="submit_debugger", action="store_true" )
parser.add_option( "-e", "--exec", 
                     help="name of executable",
                     dest="exe", type="string" )
parser.add_option( "-r", "--rcfile", 
                     help="""Rc file with runtime settings.
                     Multiple rcfiles could be specified, in which case
                     a job is submitted for each rcfile.""",
                     dest="rcfiles", action="append", default=[] )

# now parse the actual arguments;
# return an object 'opts' with fields 'verbose' etc,
# and the unnamed arguments in the list 'args' :
opts,args = parser.parse_args( args=args )


# -----------------------------------------------
# logging
# -----------------------------------------------

# setup messages:
logging.basicConfig( format='%(lineno)-4s:%(filename)-30s [%(levelname)-8s] %(message)s', level=logging.INFO, stream=sys.stdout )

# info ...
logging.info( '' )
tnow = datetime.datetime.now().isoformat(' ').split('.',1)[0]
logging.info( 'Started script at %s ...' % tnow )
logging.info( '' )

# info ...
logging.info( '  parsed options  : %s' % opts )
logging.info( '  parsed argument : %s' % args )

# debug messages to standard output ?
if opts.verbose : 
    logging.info( '  verbose mode for standard output; print all messages ...' )
    stdout_handler.setLevel(logging.DEBUG)
else :
    logging.info( '  quiet mode for standard output; print info messages only ...' )
#endif


# -----------------------------------------------
# arguments (2)
# -----------------------------------------------

# info ...
logging.info( 'store arguments ...' )

# copy from options into less cryptic variables:
rcfiles = opts.rcfiles
exe = opts.exe

# arguments are either rcfiles or an executable;
# copy from arguments into new variables;
# loop over arguments:
for arg in args :
    # executable ?
    if arg.endswith('.x') :
        # already specified ?
        if exe != None :
            logging.error( 'found argument "%s", while already executable defined : %s' % (arg,exe) )
            raise Exception
        #endif
        # info ...
        logging.info( '  found executable : %s' % arg )
        # store:
        exe = arg
        # next argument:
        continue
    #endif
    # ends with '.rc' ? then add to list with rcfiles:
    if arg.endswith('.rc') :
        # info ...
        logging.info( '  found rcfile : %s' % arg )
        # add extra rcfile:
        rcfiles.append(arg)
        # next argument:
        continue
    #endif
    # unknown ...
    logging.error( 'argument is not an executable or rcfile : %s' % arg )
    raise Exception
#endfor


# -----------------------------------------------
# toolboxes
# -----------------------------------------------

# prepend location of python modules to search path;
# either passed via arguments or the default defined above:
sys.path.insert( 0, opts.pypath )

# local modules:
import rc
import submit_tm5_tools


# -----------------------------------------------
# begin
# -----------------------------------------------

# executable should have been specified:
if exe == None :
    logging.error( 'no executable specified ...' )
    sys.exit(1)
#endif
# executable exist ?
if not os.path.exists(exe) :
    logging.error( 'executable not found : %s' % exe )
    sys.exit(1)
#endif
# info ...
logging.info( 'executable : %s' % exe )

# rcfile(s) should have been specified:
if len(rcfiles) == 0 :
    logging.error( 'no rcfile specified ...' )
    sys.exit(1)
#endif
# check if rcfiles exists ...
for rcfile in rcfiles :
    if not os.path.exists(rcfile) :
        logging.error( 'rcfile not found : %s' % rcfile )
        sys.exit(1)
    #endif
#endfor


# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# loop over rcfiles
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

# loop over rcfiles:
for rcfile in rcfiles :

    # info ...
    logging.info( 'master rcfile : %s' % rcfile )
    
    # read settings:
    rcf = rc.RcFile( rcfile )

    # replace rc keys given options passed to the script:
    if opts.submit_to       != None : rcf.replace( 'submit.to'      , opts.submit_to       )
    if opts.submit_options  != None : rcf.replace( 'submit.options' , opts.submit_options  )
    if opts.submit_debugger != None : rcf.replace( 'submit.debugger', opts.submit_debugger )
    
    ## save sources if requested:
    #submit_tm5_tools.Archive_Sources( rcf )
    
    # create first rcfile and job file(s) for the chain,
    # submit the result:
    try :
        submit_tm5_tools.WriteAndSubmitNewJob( rcf, opts.pypath )
    except :
        logging.error( sys.exc_info()[1] )
        logging.error( 'exception from WriteAndSubmitNewJob( rcf )' )
        sys.exit(1)
    #endtry

#endfor  # rcfiles


# -----------------------------------------------
# end
# -----------------------------------------------

logging.info( '' )
tnow = datetime.datetime.now().isoformat(' ').split('.',1)[0]
logging.info( 'End script at %s ...' % tnow )
logging.info( '' )