#! /usr/bin/env python3 # # job chain: # # myrun_001_init.jb # build/bin/submit_tm5_step_init rcfile # myrun_001_run.jb # build/bin/submit_tm5_step_run './tm5.x rcfile' # myrun_001_done.jb # build/bin/submit_tm5_step_done rcfile # # myrun_002_init.jb # ... # # or for a loadleveler job: # # myrun_001.jb # build/bin/submit_tm5_step_init rcfile # build/bin/submit_tm5_step_run './tm5.x rcfile' # build/bin/submit_tm5_step_done rcfile # # myrun_002.jb # ... # # ----------------------------------------------- # external # ----------------------------------------------- # standard modules: import sys import os import optparse import logging import datetime # ----------------------------------------------- # defaults # ----------------------------------------------- # display info messages ? verbose_default = False # where to search for additional scripts ? # NOTE: this line will be (has been?) replaced by 'pycasso.py/Main' # with the actual path to the build directory: pypath_default = os.path.join( os.pardir, 'build', 'bin' ) # ----------------------------------------------- # arguments (1) # ----------------------------------------------- # extract arguments from sys.argv array: # 0 = name of calling script, 1: = actual arguments script = sys.argv[0] args = sys.argv[1:] # set text for 'usage' help line: usage = "%prog [options] [-e|--exec=]] [-r|--rcfile=] ..." # extra help text epilog = """ Arguments that are no flags are supposed to be file names and clasified according to their extension as either an executable (.x) or rc file (.rc) . """ # initialise the option parser: #parser = optparse.OptionParser(usage=usage,epilog=epilog) parser = optparse.OptionParser(usage=usage) # not all python version accept epilog yet ... # define verbose option: parser.add_option( "-v", "--verbose", help="print extra logging messages to standard output (%s)" % str(verbose_default), dest="verbose", action="store_true", default=verbose_default ) parser.add_option( "-p", "--py-path", help="path to additional submit scripts (%s)" % pypath_default, dest="pypath", action="store", default=pypath_default ) parser.add_option( "-b", "--background", help="""Run executable in background. Standard output and error are redirected.""", dest="submit_to", action="store_const", const='background' ) parser.add_option( "-q", "--queue", help="submit executable to a queue system", dest="submit_to", action="store_const", const='queue' ) parser.add_option( "-s", "--submit-option", help="""Extra submit option(s) passed to queue scheduler. Example for BSUB: -K : (keep) wait for end of run before return.""", dest="submit_options", action="store" ) parser.add_option( "-d", "--debugger", help="run executable in debuger", dest="submit_debugger", action="store_true" ) parser.add_option( "-e", "--exec", help="name of executable", dest="exe", type="string" ) parser.add_option( "-r", "--rcfile", help="""Rc file with runtime settings. Multiple rcfiles could be specified, in which case a job is submitted for each rcfile.""", dest="rcfiles", action="append", default=[] ) # now parse the actual arguments; # return an object 'opts' with fields 'verbose' etc, # and the unnamed arguments in the list 'args' : opts,args = parser.parse_args( args=args ) # ----------------------------------------------- # logging # ----------------------------------------------- # setup messages: logging.basicConfig( format='%(lineno)-4s:%(filename)-30s [%(levelname)-8s] %(message)s', level=logging.INFO, stream=sys.stdout ) # info ... logging.info( '' ) tnow = datetime.datetime.now().isoformat(' ').split('.',1)[0] logging.info( 'Started script at %s ...' % tnow ) logging.info( '' ) # info ... logging.info( ' parsed options : %s' % opts ) logging.info( ' parsed argument : %s' % args ) # debug messages to standard output ? if opts.verbose : logging.info( ' verbose mode for standard output; print all messages ...' ) stdout_handler.setLevel(logging.DEBUG) else : logging.info( ' quiet mode for standard output; print info messages only ...' ) #endif # ----------------------------------------------- # arguments (2) # ----------------------------------------------- # info ... logging.info( 'store arguments ...' ) # copy from options into less cryptic variables: rcfiles = opts.rcfiles exe = opts.exe # arguments are either rcfiles or an executable; # copy from arguments into new variables; # loop over arguments: for arg in args : # executable ? if arg.endswith('.x') : # already specified ? if exe != None : logging.error( 'found argument "%s", while already executable defined : %s' % (arg,exe) ) raise Exception #endif # info ... logging.info( ' found executable : %s' % arg ) # store: exe = arg # next argument: continue #endif # ends with '.rc' ? then add to list with rcfiles: if arg.endswith('.rc') : # info ... logging.info( ' found rcfile : %s' % arg ) # add extra rcfile: rcfiles.append(arg) # next argument: continue #endif # unknown ... logging.error( 'argument is not an executable or rcfile : %s' % arg ) raise Exception #endfor # ----------------------------------------------- # toolboxes # ----------------------------------------------- # prepend location of python modules to search path; # either passed via arguments or the default defined above: sys.path.insert( 0, opts.pypath ) # local modules: import rc import submit_tm5_tools # ----------------------------------------------- # begin # ----------------------------------------------- # executable should have been specified: if exe == None : logging.error( 'no executable specified ...' ) sys.exit(1) #endif # executable exist ? if not os.path.exists(exe) : logging.error( 'executable not found : %s' % exe ) sys.exit(1) #endif # info ... logging.info( 'executable : %s' % exe ) # rcfile(s) should have been specified: if len(rcfiles) == 0 : logging.error( 'no rcfile specified ...' ) sys.exit(1) #endif # check if rcfiles exists ... for rcfile in rcfiles : if not os.path.exists(rcfile) : logging.error( 'rcfile not found : %s' % rcfile ) sys.exit(1) #endif #endfor # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # loop over rcfiles # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ # loop over rcfiles: for rcfile in rcfiles : # info ... logging.info( 'master rcfile : %s' % rcfile ) # read settings: rcf = rc.RcFile( rcfile ) # replace rc keys given options passed to the script: if opts.submit_to != None : rcf.replace( 'submit.to' , opts.submit_to ) if opts.submit_options != None : rcf.replace( 'submit.options' , opts.submit_options ) if opts.submit_debugger != None : rcf.replace( 'submit.debugger', opts.submit_debugger ) ## save sources if requested: #submit_tm5_tools.Archive_Sources( rcf ) # create first rcfile and job file(s) for the chain, # submit the result: try : submit_tm5_tools.WriteAndSubmitNewJob( rcf, opts.pypath ) except : logging.error( sys.exc_info()[1] ) logging.error( 'exception from WriteAndSubmitNewJob( rcf )' ) sys.exit(1) #endtry #endfor # rcfiles # ----------------------------------------------- # end # ----------------------------------------------- logging.info( '' ) tnow = datetime.datetime.now().isoformat(' ').split('.',1)[0] logging.info( 'End script at %s ...' % tnow ) logging.info( '' )