submit_tm5_tools.py 44 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458
  1. #
  2. # TM5 run tools
  3. #
  4. # ***
  5. def Command_Line( rcf, exe, args, in_debugger ) :
  6. """
  7. Return command line that runs executable.
  8. ARGUMENTS
  9. rcf
  10. Rcfile with settings.
  11. exe
  12. Name of executable.
  13. args
  14. Arguments to be passed to executable.
  15. in_debugger
  16. Set to 'True' if the job should be run in a debugger.
  17. RETURN VALUES
  18. cmndline
  19. Command line to be executed.
  20. """
  21. # external
  22. import socket
  23. import logging
  24. # mpi run ?
  25. if rcf.get('par.mpi','bool') :
  26. # number of mpi tasks:
  27. ntask = rcf.get('par.ntask','int')
  28. # get command line:
  29. cmnd_exec = rcf.get('mpirun.command')
  30. cmnd_args = rcf.get('mpirun.args' )
  31. # write command file ?
  32. cmdfile = rcf.get('mpirun.cmdfile',default='')
  33. if len(cmdfile) > 0 :
  34. # write command line for each task:
  35. f = open(cmdfile,'w')
  36. for i in range(ntask) : f.write( '%s %s\n' % (exe,args) )
  37. f.close()
  38. else :
  39. # otherwise, add the executable and its arguments:
  40. cmnd_args = '%s %s %s' % (cmnd_args,exe,args)
  41. #endif
  42. # write host file ? PLS: This is done too early, and should be done
  43. # inside the *_run.jb script : that will let you specify nodes
  44. # scattered on different hosts (eg, linux cluster), and also different
  45. # from the current host (which probably is a login node)!! See
  46. # WriteJob below for cases of Loadleveler and Slurm.
  47. #
  48. # Leave it here for other cases not handled in WriteJob yet.
  49. hostfile = rcf.get('mpirun.hostfile',default='')
  50. if len(hostfile) > 0 :
  51. # get hostname:
  52. hname = socket.gethostname()
  53. # write hostname for each task:
  54. f = open(hostfile,'w')
  55. for i in range(ntask) : f.write( '%s\n' % hname )
  56. f.close()
  57. else :
  58. # standard run:
  59. cmnd_exec = exe
  60. cmnd_args = args
  61. # run in debugger ?
  62. if in_debugger :
  63. # debugger type:
  64. debugger = rcf.get( 'debugger' )
  65. # get debugger command:
  66. debugger_call = rcf.get( 'debugger.command' )
  67. # large differences ...
  68. if debugger == 'totalview' :
  69. # syntaxis: totalview <executable> [-a <arguments>]
  70. # pass executable:
  71. cmndline = '%s %s' % (debugger_call,cmnd_exec)
  72. # add arguments ?
  73. if len(cmnd_args) > 0 :
  74. cmndline = '%s -a %s' % (cmndline,cmnd_args)
  75. #endif
  76. elif debugger == 'idb' :
  77. # syntaxis: idb [-args <executable> <arguments>]
  78. # fill executable and arguments:
  79. cmndline = '%s -args %s %s' % (debugger_call,cmnd_exec,cmnd_args)
  80. else :
  81. logging.error('unsupported debugger : %s' % debugger )
  82. raise Exception
  83. #endif
  84. else :
  85. # standard line:
  86. cmndline = '%s %s' % (cmnd_exec,cmnd_args)
  87. # ok
  88. return cmndline
  89. # ***
  90. def WriteAndSubmitNewJob( rcfile, bindir ) :
  91. """
  92. Write first or next rcfile and job files(s) in the job chain;
  93. if chain is not finished yet, submit a new job.
  94. The argument could be the name of an rcfile or an rcfile object itself,
  95. since the submit script might have changed some values given
  96. the provided command line arguments.
  97. The following function is used:
  98. submit_tm5_setup_rcfile.WriteRcfile # writes the new rcfile
  99. This is placed in a seperate file since users might need to
  100. change these routines for their specific projects.
  101. """
  102. # external:
  103. import sys
  104. import logging
  105. import rc
  106. # import setup module:
  107. import submit_tm5_setup_rcfile
  108. # name provided ?
  109. if type(rcfile) == str :
  110. # load:
  111. rcf = rc.RcFile( rcfile )
  112. else :
  113. # just copy ..
  114. rcf = rcfile
  115. #endif
  116. # write next rfile, return name:
  117. try :
  118. rcfile_next = submit_tm5_setup_rcfile.WriteRcfile( rcf )
  119. except :
  120. logging.error( sys.exc_info()[1] )
  121. logging.error( 'exception from WriteRcfile' )
  122. raise Exception
  123. # finished ?
  124. if rcfile_next == None :
  125. logging.info( ' end of job chain !' )
  126. else :
  127. # write job file(s) for this period and return the (first) name;
  128. # last command in a file should submit the next job if necessary:
  129. logging.info( ' write jobfile for %s ...' % rcfile_next )
  130. try :
  131. jobfile_next = WriteJob( rcfile_next, bindir )
  132. except :
  133. logging.error( sys.exc_info()[1] )
  134. logging.error( 'exception from WriteJob' )
  135. raise Exception
  136. logging.info( ' submit next job : %s' % jobfile_next )
  137. try :
  138. jobid = SubmitJob( jobfile_next, rcfile_next )
  139. except :
  140. logging.error( sys.exc_info()[1] )
  141. logging.error( 'exception from SubmitJob' )
  142. raise Exception
  143. return
  144. # ***
  145. def WriteJob( rcfile, bindir ) :
  146. """
  147. jobfile = WriteJob(rcfile)
  148. Write job file given the settings in rcfile.
  149. The name of the jobfile is based on the name of the rcfile.
  150. The last command in the job should submit the next job,
  151. and the script is therefore written in python.
  152. """
  153. # external:
  154. import os
  155. import rc
  156. # load settings:
  157. rcf = rc.RcFile( rcfile )
  158. # basename for scripts etc is name of rcfile minus extension:
  159. bname,ext = os.path.splitext(rcfile)
  160. # loadleveler supports master job with multiple steps:
  161. with_master_job = (rcf.get('submit.to') == 'queue') and (rcf.get('queue') == 'loadleveler')
  162. # which shell ?
  163. job_shell = '/usr/bin/env python'
  164. # start master job ?
  165. if with_master_job :
  166. ntasks = rcf.get('par.ntask')
  167. # name of jobfile:
  168. jobfile = '%s.jb' % bname
  169. # set header:
  170. header = []
  171. header.append( '#! %s\n' % job_shell )
  172. header.append( '\n' )
  173. # init queue options:
  174. qopt = QueueOptions( bname, rcf, 'default' )
  175. # init job file:
  176. job = []
  177. job.append( '# external:\n' )
  178. job.append( 'import os\n' )
  179. job.append( 'import sys\n' )
  180. job.append( 'import socket\n')
  181. job.append( 'import subprocess\n' )
  182. job.append( 'import logging\n' )
  183. job.append( '\n' )
  184. job.append( '# setup messages:\n' )
  185. job.append( "logging.basicConfig( format='%(lineno)-4s:%(filename)-30s [%(levelname)-8s] %(message)s', level=logging.INFO, stream=sys.stdout )\n" )
  186. job.append( '\n' )
  187. job.append( '# prepend locations of python modules to search path:\n' )
  188. job.append( "sys.path.insert( 0, '%s' )\n" % bindir )
  189. job.append( '\n' )
  190. job.append( '# tools:\n' )
  191. job.append( 'import submit_tm5_tools\n' )
  192. job.append( '\n' )
  193. job.append( '# current loadleveler steps:\n' )
  194. job.append( 'step_name = os.getenv("LOADL_STEP_NAME")\n' )
  195. job.append( '\n' )
  196. # HOSTFILE - Moved here from Command_Line for loadleveler.
  197. hostfile = rcf.get('mpirun.hostfile',default='') # calling script will create hostfile
  198. if (len(hostfile) > 0) and rcf.get('par.mpi','bool') :
  199. job.append( 'f = open("%s",\'w\') \n' % hostfile )
  200. job.append( 'hname = socket.gethostname() \n')
  201. job.append( "for i in range( %s ) : \n" % ntasks )
  202. job.append( "\tf.write( '%s\\n' % hname)\n" )
  203. job.append( 'f.close() \n')
  204. job.append( '\n' )
  205. # job step names:
  206. steps = rcf.get('job.steps').split(' ')
  207. # number of steps:
  208. nstep = len(steps)
  209. # loop over job steps:
  210. for istep in range(nstep) :
  211. # current:
  212. step = steps[istep]
  213. # next:
  214. if istep < nstep-1 : step_next = steps[istep+1]
  215. # list with queue option lines for this step:
  216. qopt_step = QueueOptions( bname, rcf, step )
  217. # call to actual script:
  218. if step == 'run' :
  219. # get command line:
  220. exe = os.path.join( os.curdir, rcf.get('job.step.%s.exe' % step) )
  221. args = rcfile
  222. indb = rcf.get('submit.debugger','bool')
  223. cmndline = Command_Line( rcf, exe, args, indb )
  224. # <script> <commandline>
  225. step_command = '["%s/submit_tm5_step_%s","%s"]' % (bindir,step,cmndline)
  226. else :
  227. # <script> <rcfile>
  228. step_command = '["%s/submit_tm5_step_%s","%s","--bindir=%s"]' % (bindir,step,rcfile,bindir)
  229. #endif
  230. # add queue options to destintation:
  231. if with_master_job :
  232. # add to queue options for master job:
  233. qopt = qopt + qopt_step
  234. # add lines to run the step:
  235. job.append( '# which step ?\n' )
  236. job.append( 'if step_name == "%s" :\n' % step )
  237. job.append( ' \n' )
  238. job.append( ' # run:\n' )
  239. job.append( ' retcode = subprocess.call( %s )\n' % step_command )
  240. job.append( ' if retcode != 0 :\n' )
  241. job.append( ' logging.error( sys.exc_info()[1] )\n' )
  242. job.append( ' logging.error( \'exception from subprocess call to : %s\' )\n' % step_command )
  243. job.append( ' sys.exit(1)\n' )
  244. job.append( ' #endif\n' )
  245. job.append( ' \n' )
  246. # last step ? then add lines to submit next job:
  247. if istep == nstep-1 :
  248. job.append( ' # write and submit next job if necessary:\n' )
  249. job.append( ' submit_tm5_tools.WriteAndSubmitNewJob( "%s", "%s" )\n' % (rcfile,bindir) )
  250. job.append( ' \n' )
  251. #endif
  252. # close step:
  253. job.append( '#endif\n' )
  254. job.append( '\n' )
  255. else : # no master job, but seperate files
  256. # name of step job to be written:
  257. step_job_template = bname+'_%s.jb'
  258. # actual name:
  259. step_job = step_job_template % step
  260. # open:
  261. f = open( step_job, 'w' )
  262. # write header:
  263. f.write( '#! %s\n' % job_shell )
  264. f.write( '\n' )
  265. # add queue options:
  266. for line in qopt_step : f.write(line)
  267. # add lines to call the actual script:
  268. f.write( '# external:\n' )
  269. f.write( 'import sys\n' )
  270. f.write( 'import os\n' )
  271. f.write( 'import logging\n' )
  272. f.write( 'import socket\n' )
  273. f.write( 'import subprocess\n' )
  274. f.write( '\n' )
  275. f.write( '# go to run directory:\n' )
  276. f.write( 'os.chdir("%s")\n' % os.getcwd() )
  277. f.write( '\n' )
  278. f.write( '# prepend locations of python modules to search path:\n' )
  279. f.write( "sys.path.insert( 0, '%s' )\n" % bindir )
  280. f.write( '\n' )
  281. f.write( '# tools:\n' )
  282. f.write( 'import rc\n' )
  283. f.write( 'import submit_tm5_tools\n' )
  284. f.write( '\n' )
  285. f.write( '# setup messages:\n' )
  286. f.write( "logging.basicConfig( format='%(lineno)-4s:%(filename)-30s [%(levelname)-8s] %(message)s', level=logging.INFO, stream=sys.stdout )\n" )
  287. f.write( '\n' )
  288. f.write( '# info ...\n' )
  289. f.write( 'logging.info( "start" )\n' )
  290. f.write( '\n' )
  291. # module command if needed
  292. module_cmd = rcf.get('module.cmd',default='')
  293. if (module_cmd != '') and (step == 'run'):
  294. f.write( 'mod_cmd="%s".split()\n' % module_cmd )
  295. f.write( 'retcode = subprocess.call( mod_cmd )\n' )
  296. f.write( 'if retcode != 0:\n' )
  297. f.write( ' logging.error( sys.exc_info()[1] )\n' )
  298. f.write( ' logging.error( \'exception from subprocess call to : %s\' )\n' % module_cmd )
  299. f.write( ' sys.exit(1)\n' )
  300. f.write( '\n' )
  301. # openMP
  302. if ( (rcf.get('queue') == 'slurm') or (rcf.get('queue') == 'pbs')) and \
  303. rcf.get('par.openmp','bool') and (step == 'run'):
  304. nthread = rcf.get( 'par.nthread', 'int' )
  305. f.write( "os.putenv( \'OMP_NUM_THREADS\', '%s')\n" % nthread )
  306. # HOSTFILE - Moved here (and adapted) from Command_Line for case of
  307. # "SLURM with a host file" (case of NEURON@KNMI, and not CARTESIUS@SARA).
  308. # Needed only if MPI, step run.
  309. hostfile = rcf.get('mpirun.hostfile',default='')
  310. if (len(hostfile) > 0) and (rcf.get('queue') == 'slurm') and rcf.get('par.mpi','bool') and (step == 'run'):
  311. f.write( 'f = open("%s",\'w\') \n' % hostfile)
  312. # PREVIOUS:
  313. #f.write( 'hname = socket.gethostname() \n')
  314. #f.write( 'for i in range(ntask) : f.write( '%s' % hname ) \n')
  315. # NOW: for SLURM on NEURON, need nodelist and number of
  316. # tasks-per-node effectively allocated.
  317. # --Idee #1
  318. # -- Use "output environment variables" to gather effective values:
  319. #
  320. #f.write( 'for node in os.getenv('SLURM_NODELIST'): \n')
  321. #f.write( ' for i in range(os.getenv('SLURM_NTASKS_PER_NODE'): f.write( '%s\n' % node ) \n')
  322. #
  323. # -- BUT that would not work in all imaginable cases... so we
  324. # -- should use SLURM_TASKS_PER_NODE instead, but its format, for eg:
  325. # -- 2(x3),1,5,2(x5)
  326. # -- is a bit annoying to process. Then idee #2... good old unix command line:
  327. # --Idee #2 - Use srun | sort | awk to get ordered list
  328. hcommand = '[ "srun -l /bin/hostname | sort -n | awk \'{print $2}\' > %s"]' % hostfile
  329. f.write( 'retcode = subprocess.call( %s, shell=True )\n' % hcommand )
  330. f.write( 'if retcode != 0:\n' )
  331. f.write( ' logging.error( sys.exc_info()[1] )\n' )
  332. f.write( ' logging.error( \'exception from creating host file\' )\n' )
  333. f.write( ' sys.exit(1)\n' )
  334. f.write( 'f.close() \n')
  335. f.write( '\n' )
  336. f.write( '# call user script:\n' )
  337. f.write( 'retcode = subprocess.call( %s )\n' % step_command )
  338. f.write( 'if retcode != 0:\n' )
  339. f.write( ' logging.error( sys.exc_info()[1] )\n' )
  340. f.write( ' logging.error( \'exception from subprocess call to : %s\' )\n' % step_command )
  341. f.write( ' sys.exit(1)\n' )
  342. f.write( '#endif\n' )
  343. f.write( '\n' )
  344. # add submission of next step?
  345. if istep < nstep-1 :
  346. # job script of next step:
  347. step_job_next = step_job_template % step_next
  348. # add submission command:
  349. f.write( '# submit next step:\n' )
  350. f.write( 'try :\n' )
  351. f.write( ' submit_tm5_tools.SubmitJob( "%s", "%s" )\n' % (step_job_next,rcfile) )
  352. f.write( 'except:\n' )
  353. f.write( ' logging.error( sys.exc_info()[1] )\n' )
  354. f.write( ' logging.error( \'exception from SubmitJob( "%s", "%s" )\' )\n' % (step_job_next,rcfile) )
  355. f.write( ' sys.exit(1)\n' )
  356. f.write( '#endtry\n' )
  357. f.write( '\n' )
  358. else :
  359. # last step; might be necessary to submit a new job:
  360. f.write( '# write and submit next job if necessary:\n' )
  361. f.write( 'submit_tm5_tools.WriteAndSubmitNewJob( "%s", "%s" )\n' % (rcfile,bindir) )
  362. f.write( '\n' )
  363. f.write( '# info ...\n' )
  364. f.write( 'logging.info( "end" )\n' )
  365. f.write( '\n' )
  366. f.close()
  367. # make it executable and readible for all, writable for user only:
  368. # u+r u+w u+x g+r g-w g+x o+r o-w o+x
  369. os.chmod( step_job, 2**8 + 2**7 + 2**6 + 2**5 + 0 + 2**3 + 2**2 + 0 + 2**0 )
  370. # fill return value:
  371. if istep == 0 : jobfile = step_job
  372. # write master job:
  373. if with_master_job :
  374. # combine:
  375. job = header + qopt + job
  376. # write:
  377. f = open(jobfile,'w')
  378. f.writelines(job)
  379. f.close()
  380. # make it executable and readible for all, writable for user only:
  381. # u+r u+w u+x g+r g-w g+x o+r o-w o+x
  382. os.chmod( jobfile, 2**8 + 2**7 + 2**6 + 2**5 + 0 + 2**3 + 2**2 + 0 + 2**0 )
  383. return jobfile
  384. # ***
  385. def WriteAndSubmitBuildJob( rcf, command ):
  386. """
  387. Write bash script that run 'command' after being submitted to the
  388. queue. Written to compile TM5 in the queue.
  389. """
  390. import os
  391. orig_val = rcf.get('submit.to')
  392. dummy = rcf.replace('submit.to','queue')
  393. source_dir = os.getcwd() # assume that it is called from
  394. # source directory
  395. jobfile = os.path.join(source_dir,'build.jb')
  396. f = open( jobfile, 'w' )
  397. f.write( '#! /bin/bash\n' )
  398. f.write( '\n' )
  399. qopt = QueueOptions( 'buildTM', rcf, 'build' )
  400. for line in qopt : f.write(line)
  401. f.write( "cd %s \n" % source_dir )
  402. f.write( '%s' % ' '.join(command) )
  403. f.write( '\n' )
  404. f.close()
  405. id=SubmitJob( jobfile, rcf )
  406. dummy = rcf.replace('submit.to', orig_val)
  407. return
  408. # ***
  409. def QueueOptions( bname, rcf, step ) :
  410. """
  411. Return list with queue option lines.
  412. """
  413. # modules:
  414. import logging
  415. # submit to queue ?
  416. if rcf.get('submit.to') == 'queue' :
  417. # queue type:
  418. queue = rcf.get('queue')
  419. # different options and commands:
  420. if queue == 'loadleveler' :
  421. qopt = QueueOptions_LoadLeveler( bname, rcf, step )
  422. elif queue == 'bsub' :
  423. qopt = QueueOptions_BSub( bname, rcf, step )
  424. elif queue == 'qsub' :
  425. qopt = QueueOptions_QSub( bname, rcf, step )
  426. elif queue == 'pbs' :
  427. qopt = QueueOptions_PBS( bname, rcf, 'all' ) # init result with the "all" step
  428. qopt = qopt + QueueOptions_PBS( bname, rcf, step )
  429. elif queue == 'slurm' :
  430. qopt = QueueOptions_Slurm( bname, rcf, step )
  431. else :
  432. # not supported ...
  433. logging.error( 'unsupported queue : %s' % queue )
  434. raise Exception
  435. else:
  436. qopt = []
  437. return qopt
  438. # ***
  439. def SubmitJob( job_script, rcfile ) :
  440. """
  441. Submit jobscript. Where to submit to (foreground,background, queue) is
  442. read from rcfile settings. Returns jobid if submitting to queue (for now
  443. dummy for all cases except PBS).
  444. """
  445. import sys
  446. import logging
  447. import rc
  448. jobid='not-a-real-jobid-yet' # default
  449. # settings
  450. if type(rcfile) == str :
  451. rcf = rc.RcFile( rcfile )
  452. else :
  453. rcf = rcfile
  454. # where to ?
  455. submit_to = rcf.get('submit.to')
  456. # info ...
  457. logging.info( 'submit %s to %s ...' % (job_script,submit_to) )
  458. # call specific submit routines:
  459. if submit_to == 'foreground' :
  460. # call run script, catch errors:
  461. try :
  462. Run_Job_In_Foreground( job_script )
  463. except :
  464. logging.error( sys.exc_info()[1] )
  465. logging.error( 'from Run_Job_In_Foreground for %s' % job_script )
  466. raise Exception
  467. elif submit_to == 'background' :
  468. # call run script, catch errors:
  469. try :
  470. Submit_Job_To_Background( job_script, rcf )
  471. except :
  472. logging.error( 'from Submit_Job_To_Background for %s' % job_script )
  473. raise Exception
  474. elif submit_to == 'queue' :
  475. # queue type:
  476. queue = rcf.get('queue')
  477. # different options and commands:
  478. if queue == 'loadleveler' :
  479. Submit_Job_To_LoadLeveler( job_script, rcf )
  480. elif queue == 'bsub' :
  481. Submit_Job_To_BSub( job_script, rcf )
  482. elif queue == 'qsub' :
  483. Submit_Job_To_QSub( job_script, rcf )
  484. elif queue == 'pbs' :
  485. jobid = Submit_Job_To_PBS( job_script, rcf )
  486. elif queue == 'slurm' :
  487. Submit_Job_To_Slurm( job_script, rcf )
  488. else :
  489. # not supported ...
  490. logging.error( 'unsupported queue : %s' % queue )
  491. raise Exception
  492. else :
  493. # not supported ...
  494. logging.error( 'unsupported run environment : %s' % submit_to )
  495. sys.exit(1)
  496. return jobid
  497. # ======================================================================
  498. # ===
  499. # === foreground
  500. # ===
  501. # ======================================================================
  502. def Run_Job_In_Foreground( job_script ) :
  503. """
  504. Run job script in foreground.
  505. """
  506. # external:
  507. import sys
  508. import os
  509. import logging
  510. import subprocess
  511. # setup command line, e.g. './myscript.jb' :
  512. command = os.path.join(os.curdir,job_script)
  513. # execute:
  514. retcode = subprocess.call( command )
  515. if retcode != 0 :
  516. logging.error( sys.exc_info()[1] )
  517. logging.error( 'from subprocess call to : %s' % command )
  518. raise Exception
  519. return
  520. # ======================================================================
  521. # ===
  522. # === background
  523. # ===
  524. # ======================================================================
  525. def Submit_Job_To_Background( job_script, rcf ) :
  526. """
  527. Submit job to background.
  528. """
  529. # external:
  530. import sys
  531. import os
  532. import logging
  533. import subprocess
  534. # basename for scripts etc is name of rcfile minus extension:
  535. bname,ext = os.path.splitext(job_script)
  536. # output files:
  537. job_stdout = bname+'.out'
  538. job_stderr = bname+'.err'
  539. job_info = bname+'.info'
  540. # setup command line, e.g. './myscript.jb' :
  541. command = os.path.join(os.curdir,job_script)
  542. # re-direct standard output:
  543. command = command+' > %s' % job_stdout
  544. # write error messages to seperate file:
  545. #command = command+' 2> %s' % job_stderr
  546. command = command+' 2>&1'
  547. # run in background, return process id:
  548. logging.info( 'run shell command : "%s" ...' % command )
  549. p = subprocess.Popen( command, shell=True )
  550. # info ...
  551. infotext = []
  552. infotext.append( '\n' )
  553. infotext.append( 'Summary:\n' )
  554. infotext.append( '\n' )
  555. infotext.append( ' job script : %s\n' % job_script )
  556. infotext.append( ' standard output : %s\n' % job_stdout )
  557. infotext.append( ' standard error : %s\n' % job_stderr )
  558. infotext.append( '\n' )
  559. infotext.append( 'Process snapshot:\n')
  560. infotext.append( '\n')
  561. p2 = subprocess.Popen( '/bin/ps -f -p %i' % p.pid, shell=True,
  562. stdout=subprocess.PIPE, stderr=subprocess.STDOUT )
  563. for line in p2.stdout.readlines() : infotext.append( line )
  564. infotext.append( '\n')
  565. infotext.append( 'To manage this process:\n' )
  566. infotext.append( '\n' )
  567. infotext.append( ' # show process snapshot:\n' )
  568. infotext.append( ' ps -f -p %i\n' % p.pid )
  569. infotext.append( ' \n' )
  570. infotext.append( ' # kill process:\n' )
  571. infotext.append( ' kill %i\n' % p.pid )
  572. infotext.append( ' \n' )
  573. infotext.append( ' # follow standard output:\n' )
  574. infotext.append( ' tail -f %s\n' % job_stdout )
  575. infotext.append( '\n' )
  576. # write to log:
  577. for line in infotext : logging.info( line.strip() )
  578. # write to file:
  579. f = open( job_info, 'w' )
  580. f.writelines(infotext)
  581. f.close()
  582. return
  583. # ======================================================================
  584. # ===
  585. # === LoadLeveler queue
  586. # ===
  587. # ======================================================================
  588. def QueueOptions_LoadLeveler( bname, rcf, step ) :
  589. """
  590. Return list with queue options.
  591. """
  592. # external:
  593. import math
  594. # init result:
  595. qopt = []
  596. # which step ?
  597. if step == 'default' :
  598. # list with options:
  599. opts = rcf.get( 'queue.ll.options.%s' % step ).split()
  600. # default options:
  601. for opt in opts :
  602. # get value:
  603. val = rcf.get( 'queue.ll.option.%s.%s' % (step,opt) )
  604. # write:
  605. qopt.append( '#@ %-20s = %s\n' % (opt,val) )
  606. #endfor
  607. # layout ...
  608. qopt.append( '\n' )
  609. else :
  610. # list with options:
  611. opts = rcf.get( 'queue.ll.options.%s' % step ).split()
  612. # default options:
  613. for opt in opts :
  614. # get value:
  615. val = rcf.get( 'queue.ll.option.%s.%s' % (step,opt) )
  616. # to be set ?
  617. if val == '<auto>' :
  618. # differs per option ...
  619. if opt == 'output' :
  620. val = '%s_%s.out' % (bname,step)
  621. elif opt == 'error' :
  622. val = '%s_%s.err' % (bname,step)
  623. elif opt == 'tasks_per_node' :
  624. t_mpi = rcf.get( 'par.ntask', 'int' )
  625. t_omp = rcf.get( 'par.nthread', 'int' )
  626. val = str(t_omp*t_mpi)
  627. #endif
  628. #endif
  629. # no, empty, or normal value ?
  630. if val == '<none>' :
  631. # skip this keyword:
  632. continue
  633. elif val == '' :
  634. # just the keyword:
  635. qopt.append( '#@ %s\n' % opt )
  636. else :
  637. # keyword and value:
  638. qopt.append( '#@ %-20s = %s\n' % (opt,val) )
  639. #endif
  640. #endfor
  641. # layout ...
  642. qopt.append( '\n' )
  643. return qopt
  644. # ***
  645. def Submit_Job_To_LoadLeveler( job_script, rcf ) :
  646. """
  647. Submit job to LoadLeveler queue.
  648. """
  649. # external:
  650. import sys
  651. import os
  652. import logging
  653. import subprocess
  654. # basename for scripts etc is name of rcfile minus extension:
  655. bname,ext = os.path.splitext(job_script)
  656. # output files:
  657. job_info = bname+'.info'
  658. # options passed directly to submission command:
  659. qopts = rcf.get( 'queue.ll.submit.options' )
  660. # add options passed to submit script:
  661. qopts = qopts+' '+rcf.get('submit.options')
  662. # info ...
  663. logging.info( ' launch ...' )
  664. # setup command line:
  665. command = 'llsubmit '+qopts
  666. # last argument is script:
  667. command = command+' '+job_script
  668. # info ...
  669. logging.info( ' command: %s' % command )
  670. # init submission info file:
  671. infotext = []
  672. infotext.append( '\n' )
  673. # call submit command, trap errors:
  674. try:
  675. # submit; redirect errors to standard output:
  676. p = subprocess.Popen( command.split(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT )
  677. except :
  678. logging.error( sys.exc_info()[1] )
  679. logging.error( 'from subprocess.Popen( %s )' % command.split() )
  680. raise Exception
  681. #endtry
  682. # extract:
  683. outlines = p.stdout.readlines()
  684. # add to help info message:
  685. infotext = infotext + outlines
  686. # extract job id from last line:
  687. # llsubmit: The job "c1a0303.4290133" with 3 job steps has been submitted.
  688. firstwords = 'llsubmit: The job'
  689. lastline = outlines[-1]
  690. if lastline.startswith(firstwords) :
  691. job_id = lastline.lstrip(firstwords).split()[0].replace('"','')
  692. else :
  693. job_id = '<job-id>'
  694. #endif
  695. # add help text to submission info:
  696. infotext.append( '\n' )
  697. infotext.append( 'To manage LoadLeveler jobs:\n' )
  698. infotext.append( '\n' )
  699. infotext.append( ' llq [-u ${USER}] # list [your] current jobs\n' )
  700. infotext.append( ' llq %s # list this job\n' % job_id )
  701. infotext.append( ' llcancel %s # kill this job\n' % job_id )
  702. infotext.append( '\n' )
  703. # write to log:
  704. for line in infotext : logging.info( line.rstrip() )
  705. # write to file:
  706. f = open( job_info, 'w' )
  707. f.writelines(infotext)
  708. f.close()
  709. # ok
  710. return
  711. #enddef
  712. # ======================================================================
  713. # ===
  714. # === BSUB queue
  715. # ===
  716. # ======================================================================
  717. def QueueOptions_BSub( bname, rcf, step ) :
  718. """
  719. Return list with queue options.
  720. """
  721. # init result:
  722. qopt = []
  723. # List of options for specified step:
  724. opts = rcf.get( 'queue.bsub.options.%s' % step ).split()
  725. # loop over options:
  726. for opt in opts :
  727. # get value:
  728. val = rcf.get( 'queue.bsub.option.%s.%s' % (step,opt) )
  729. # default options:
  730. if val == '<auto>' :
  731. if opt in ['o','oo'] :
  732. val = '%s_%s.out' % (bname,step)
  733. elif opt in ['e','eo'] :
  734. val = '%s_%s.err' % (bname,step)
  735. #endif
  736. #endif # <auto> value
  737. # define option key:
  738. # queue.bsub.options.R : 20 -> -R 20
  739. # queue.bsub.options.Rx : -R 20 -> -R 20
  740. if val.startswith('-') :
  741. qopt.append( '#BSUB %s\n' % (val) )
  742. else :
  743. qopt.append( '#BSUB -%s %s\n' % (opt,val) )
  744. #endif
  745. #endfor # opts
  746. # layout ...
  747. qopt.append( '\n' )
  748. # ok
  749. return qopt
  750. #enddef QueueOptions_BSub
  751. # ***
  752. def Submit_Job_To_BSub( job_script, rcf ) :
  753. """
  754. Submit job to BSUB queue.
  755. """
  756. # external:
  757. import os
  758. import logging
  759. import subprocess
  760. # basename for scripts etc is name of rcfile minus extension:
  761. bname,ext = os.path.splitext(job_script)
  762. # output files:
  763. job_info = bname+'.info'
  764. # options passed directly to submission command:
  765. qopts = rcf.get( 'queue.bsub.submit.options' )
  766. # add options passed to submit script:
  767. qopts = qopts+' '+rcf.get('submit.options')
  768. # info ...
  769. logging.info( ' launch ...' )
  770. # setup command line:
  771. command = 'bsub '+qopts
  772. # pass job script to std.input:
  773. command = command+' < '+job_script
  774. # info ...
  775. logging.info( ' command: %s' % command )
  776. # prepare for OS errors (file does not exist etc.)
  777. try:
  778. # submit; redirect errors to standard output:
  779. p = subprocess.Popen( command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT )
  780. except OSError, err :
  781. logging.error( 'OSError: '+err.strerror )
  782. logging.error( 'from call : %s' % command )
  783. raise Exception
  784. #endtry
  785. # extract:
  786. outlines = p.stdout.readlines()
  787. # display:
  788. for line in outlines : logging.info( ' %s' % line.rstrip() )
  789. # standard output is:
  790. # <jobname> <jobnr>
  791. # extract job nr:
  792. job_nr = outlines[0].split()[1].strip('<>')
  793. # info ...
  794. infotext = []
  795. infotext.append( '\n' )
  796. infotext.append( 'Summary:\n' )
  797. infotext.append( '\n' )
  798. infotext.append( ' current dir : %s\n' % os.getcwd() )
  799. infotext.append( ' job script : %s\n' % job_script )
  800. infotext.append( '\n' )
  801. infotext.append( 'To manage this job:\n' )
  802. infotext.append( ' \n' )
  803. infotext.append( ' # kill job:\n' )
  804. infotext.append( ' bkill %s\n' % job_nr )
  805. infotext.append( ' \n' )
  806. infotext.append( 'To show all your running jobs:\n' )
  807. infotext.append( ' \n' )
  808. infotext.append( ' bjobs\n' )
  809. infotext.append( ' \n' )
  810. # write to log:
  811. for line in infotext : logging.info( line.rstrip() )
  812. # write to file:
  813. f = open( job_info, 'w' )
  814. f.writelines(infotext)
  815. f.close()
  816. # ok
  817. return
  818. #enddef
  819. # ======================================================================
  820. # ===
  821. # === QSUB queue
  822. # ===
  823. # ======================================================================
  824. def QueueOptions_QSub( bname, rcf, step ) :
  825. """
  826. Return list with queue options.
  827. """
  828. # init result:
  829. qopt = []
  830. # list with options:
  831. opts = rcf.get( 'queue.qsub.options' ).split()
  832. append_jobstep = rcf.get('queue.qsub.name.append.jobstep',default=False)
  833. # default options:
  834. for opt in opts :
  835. # look first for a jobstep-specific qsub option
  836. val = rcf.get( 'queue.qsub.option.%s.%s' % (opt,step), default='STEP_SPECIFIC_KEY_MISSING' )
  837. # if jobstep-specific option is missing, look for one without jobstep
  838. if val == 'STEP_SPECIFIC_KEY_MISSING':
  839. val = rcf.get( 'queue.qsub.option.%s' % opt)
  840. if (opt == 'N') and append_jobstep:
  841. jobstep = rcf.get('jobstep')
  842. val = val + '_%03d' % (int(jobstep))
  843. # fill option line:
  844. qopt.append( '#PBS -%s %s\n' % (opt,val) )
  845. #endfor
  846. # layout ...
  847. qopt.append( '\n' )
  848. # ok
  849. return qopt
  850. #enddef
  851. # ***
  852. def Submit_Job_To_QSub( job_script, rcf ) :
  853. """
  854. Submit job to QSUB queue.
  855. """
  856. # external:
  857. import os
  858. import logging
  859. import subprocess
  860. # basename for scripts etc is name of rcfile minus extension:
  861. bname,ext = os.path.splitext(job_script)
  862. # output files:
  863. job_info = bname+'.info'
  864. # options passed directly to submission command:
  865. qopts = rcf.get( 'queue.qsub.submit.options' )
  866. # add options passed to submit script:
  867. qopts = qopts+' '+rcf.get('submit.options')
  868. # info ...
  869. logging.info( ' launch ...' )
  870. # setup command line:
  871. command = 'qsub '+qopts
  872. # last argument is script:
  873. command = command+' '+job_script
  874. # info ...
  875. logging.info( ' command: %s' % command )
  876. # prepare for OS errors (file does not exist etc.)
  877. try:
  878. # submit; redirect errors to standard output:
  879. p = subprocess.Popen( command.split(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT )
  880. except OSError, err :
  881. logging.error( 'OSError: '+err.strerror )
  882. logging.error( 'from call : %s' % command )
  883. raise Exception
  884. #endtry
  885. # extract:
  886. outlines = p.stdout.readlines()
  887. # display:
  888. for line in outlines : logging.info( ' %s' % line.rstrip() )
  889. # standard output is:
  890. # jobnr
  891. # extract job nr:
  892. job_nr = outlines[0].split()[0]
  893. # info ...
  894. infotext = []
  895. infotext.append( '\n' )
  896. infotext.append( 'Summary:\n' )
  897. infotext.append( '\n' )
  898. infotext.append( ' current dir : %s\n' % os.getcwd() )
  899. infotext.append( ' job script : %s\n' % job_script )
  900. infotext.append( '\n' )
  901. infotext.append( 'To manage this job:\n' )
  902. infotext.append( ' \n' )
  903. infotext.append( ' # kill job:\n' )
  904. infotext.append( ' qdel %s\n' % job_nr )
  905. infotext.append( ' \n' )
  906. infotext.append( 'To show all your running jobs:\n' )
  907. infotext.append( ' \n' )
  908. infotext.append( ' qstat [-u ${USER}]\n' )
  909. infotext.append( ' \n' )
  910. # write to log:
  911. for line in infotext : logging.info( line.rstrip() )
  912. # write to file:
  913. f = open( job_info, 'w' )
  914. f.writelines(infotext)
  915. f.close()
  916. # ok
  917. return
  918. # ======================================================================
  919. # ===
  920. # === PBS queue (PBSpro)
  921. # ===
  922. # === Note that there are several implementations of PBS and the "pbs queue"
  923. # === here, although it also uses the qsub command, differs from the "qsub
  924. # === queue" defined above. The later uses #$ directives, while the pbsPro
  925. # === directives start with #PBS.
  926. # ===
  927. # ======================================================================
  928. def QueueOptions_PBS( bname, rcf, step ) :
  929. """
  930. Return list with queue options. To call twice, first for pseudo-step
  931. 'all'. See QueueOptions above.
  932. """
  933. import os
  934. qopt = [] # init result
  935. opts = rcf.get( 'queue.pbs.options.%s' % step ).split() # list with options
  936. # use fully qualified filename for log files
  937. if step != 'build':
  938. fqpath = rcf.get('rundir')
  939. else:
  940. fqpath = rcf.get('build.sourcedir')
  941. fqname = os.path.join( fqpath, bname )
  942. # build list of job directives
  943. for opt in opts :
  944. val = rcf.get( 'queue.pbs.option.%s.%s' % (step,opt) )
  945. # deal w/ multiple options with -l
  946. if opt == 'l':
  947. l_vals = val.split()
  948. for lval in l_vals: qopt.append( '#PBS -l %s\n' % lval )
  949. continue
  950. # options still to be set
  951. if val == '<auto>' :
  952. if opt == 'o' :
  953. val = '%s_%s.out' % (fqname,step)
  954. elif opt == 'e' :
  955. val = '%s_%s.err' % (fqname,step)
  956. # none, empty or plain normal value ?
  957. if val == '<none>' :
  958. continue # skip this option
  959. elif val == '' :
  960. qopt.append( '#PBS -%s\n' % opt ) # just the keyword
  961. else :
  962. qopt.append( '#PBS -%s %s\n' % (opt,val) ) # keyword and value
  963. qopt.append( '\n' )
  964. return qopt
  965. def Submit_Job_To_PBS( job_script, rcf ) :
  966. """
  967. Submit job to PBS queue.
  968. """
  969. import os
  970. import logging
  971. import subprocess
  972. # basename for scripts etc is name of rcfile minus extension:
  973. bname,ext = os.path.splitext(job_script)
  974. # output files:
  975. job_info = bname+'.info'
  976. # Two set of options to pass directly unmodified at the submission command :
  977. qopts = rcf.get( 'queue.pbs.submit.options' ) # (1) from machine or queue (if separated from machine) rcfile
  978. qopts = qopts+' '+rcf.get('submit.options') # (2) from expert rcfile
  979. # info ...
  980. logging.info( ' launch ...' )
  981. # setup command line:
  982. command = 'qsub '+qopts
  983. # last argument is script:
  984. command = command+' '+job_script
  985. # info ...
  986. logging.info( ' command: %s' % command )
  987. # prepare for OS errors (file does not exist etc.)
  988. try:
  989. # submit; redirect errors to standard output:
  990. p = subprocess.Popen( command.split(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT )
  991. except OSError, err :
  992. logging.error( 'OSError: '+err.strerror )
  993. logging.error( 'from call : %s' % command )
  994. raise Exception
  995. # log output
  996. outlines = p.stdout.readlines()
  997. for line in outlines : logging.info( ' %s' % line.rstrip() )
  998. # standard output is:
  999. # jobnr
  1000. # extract job nr:
  1001. job_nr = outlines[0].split()[0]
  1002. # info ...
  1003. infotext = []
  1004. infotext.append( '\n' )
  1005. infotext.append( 'Summary:\n' )
  1006. infotext.append( '\n' )
  1007. infotext.append( ' current dir : %s\n' % os.getcwd() )
  1008. infotext.append( ' job script : %s\n' % job_script )
  1009. infotext.append( '\n' )
  1010. infotext.append( 'To manage this job:\n' )
  1011. infotext.append( ' \n' )
  1012. infotext.append( ' # kill job:\n' )
  1013. infotext.append( ' qdel %s\n' % job_nr )
  1014. infotext.append( ' \n' )
  1015. infotext.append( 'To show all your running jobs:\n' )
  1016. infotext.append( ' \n' )
  1017. infotext.append( ' qstat [-u ${USER}]\n' )
  1018. infotext.append( ' \n' )
  1019. infotext.append( 'To monitor this running job (ECMWF only!):\n' )
  1020. infotext.append( ' \n' )
  1021. infotext.append( ' qcat %s\n' % job_nr )
  1022. infotext.append( ' \n' )
  1023. # write to log:
  1024. for line in infotext : logging.info( line.rstrip() )
  1025. # write to file:
  1026. f = open( job_info, 'w' )
  1027. f.writelines(infotext)
  1028. f.close()
  1029. return job_nr
  1030. # ======================================================================
  1031. # ===
  1032. # === SLURM queue
  1033. # ===
  1034. # ======================================================================
  1035. def QueueOptions_Slurm( bname, rcf, step ) :
  1036. """
  1037. Return list with queue options directives, that will be at the top of
  1038. script.
  1039. Script for run step (NEURON@KNMI) should contains:
  1040. #SBATCH -n ${par.nthread}*${ntask}
  1041. export OMP_NUM_THREADS=${par.nthread}
  1042. mpiexec.hydra -machinefile ./mpd.hosts -np ${ntask} ./$bin
  1043. """
  1044. this_jobname=""
  1045. ntasks=0
  1046. qopt = [] # init result
  1047. # List of options for specified step:
  1048. if rcf.has_key(( 'queue.slurm.options.%s' % step )) :
  1049. opts = rcf.get( 'queue.slurm.options.%s' % step ).split()
  1050. else:
  1051. opts = rcf.get( 'queue.slurm.options').split()
  1052. # NOAA-specific option
  1053. append_jobstep = rcf.get('queue.slurm.name.append.jobstep',default=False)
  1054. for opt in opts :
  1055. # look first for a step-specific slurm option
  1056. val = rcf.get( 'queue.slurm.option.%s.%s' % (step,opt), default='STEP_SPECIFIC_KEY_MISSING' )
  1057. # if step-specific option is missing, look for one without step
  1058. if (val == 'STEP_SPECIFIC_KEY_MISSING') :
  1059. # get value:
  1060. val = rcf.get( 'queue.slurm.option.%s' % opt, default='NOT_PRESENT' )
  1061. # default options:
  1062. if val == '<auto>' :
  1063. if opt == 'o' :
  1064. val = '%s_%s.out' % (bname,step)
  1065. elif opt == 'e' :
  1066. val = '%s_%s.err' % (bname,step)
  1067. if (opt == 'J' or opt == 'job-name'):
  1068. if append_jobstep:
  1069. jobstep = rcf.get('jobstep')
  1070. val = val + '_' + ( '%03d' % int(jobstep) )
  1071. this_jobname=val
  1072. if (opt == 'ntasks'):
  1073. ntasks=int(val)
  1074. # skip unset (empty) directives
  1075. if opt == 'p' or 'w':
  1076. if len(val)==0 : continue
  1077. # Some options are represented byonly a single-letter
  1078. # (e.g. J), and these require a single dash
  1079. # (e.g. -J <jobname>) but others have only a multi-letter
  1080. # invocation, and need two dashes (e.g. --qos batch).
  1081. dashes='-'
  1082. if(len(opt)>1):
  1083. dashes='--'
  1084. qopt.append( '#SBATCH %s%s %s\n' % (dashes,opt,val) )
  1085. # layout ...
  1086. qopt.append( '\n' )
  1087. # ok
  1088. return qopt
  1089. def Submit_Job_To_Slurm( job_script, rcf ) :
  1090. """
  1091. Submit job to SLURM queue.
  1092. """
  1093. # external:
  1094. import os
  1095. import logging
  1096. import subprocess
  1097. # basename for scripts etc is name of rcfile minus extension:
  1098. bname,ext = os.path.splitext(job_script)
  1099. # output files:
  1100. job_info = bname+'.info'
  1101. # Two set of options to pass directly unmodified at the submission command :
  1102. qopts = rcf.get( 'queue.slurm.submit.options' ) # (1) from pycassso-queue-slurm.rc
  1103. qopts = qopts+' '+rcf.get('submit.options') # (2) from pycasso-tm5-expert.rc
  1104. # info ...
  1105. logging.info( ' launch ...' )
  1106. # setup command line:
  1107. command = 'sbatch '+qopts
  1108. # last argument is script:
  1109. command = command+' '+job_script
  1110. # info ...
  1111. logging.info( ' command: %s' % command )
  1112. # prepare for OS errors (file does not exist etc.)
  1113. try:
  1114. # submit; redirect errors to standard output:
  1115. p = subprocess.Popen( command.split(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT )
  1116. except OSError, err :
  1117. logging.error( 'OSError: '+err.strerror )
  1118. logging.error( 'from call : %s' % command )
  1119. raise Exception
  1120. # extract:
  1121. outlines = p.stdout.readlines()
  1122. # display:
  1123. for line in outlines : logging.info( ' %s' % line.rstrip() )
  1124. # standard output is:
  1125. # jobnr
  1126. # extract job nr:
  1127. job_nr = outlines[0].split()[3]
  1128. # info ...
  1129. infotext = []
  1130. infotext.append( '\n' )
  1131. infotext.append( 'Summary:\n' )
  1132. infotext.append( '\n' )
  1133. infotext.append( ' current dir : %s\n' % os.getcwd() )
  1134. infotext.append( ' job script : %s\n' % job_script )
  1135. infotext.append( '\n' )
  1136. infotext.append( 'To manage this job:\n' )
  1137. infotext.append( ' \n' )
  1138. infotext.append( ' # kill job:\n' )
  1139. infotext.append( ' scancel %s\n' % job_nr )
  1140. infotext.append( ' \n' )
  1141. infotext.append( 'To show all your running jobs:\n' )
  1142. infotext.append( ' \n' )
  1143. infotext.append( ' squeue [-u ${USER}]\n' )
  1144. infotext.append( ' \n' )
  1145. # write to log:
  1146. for line in infotext : logging.info( line.rstrip() )
  1147. # write to file:
  1148. f = open( job_info, 'w' )
  1149. f.writelines(infotext)
  1150. f.close()
  1151. return
  1152. # ======================================================================
  1153. # ===
  1154. # === end
  1155. # ===
  1156. # ======================================================================