ecearth3/sources/tm5mp/tools/ttb/bin/ttb_change.py

"""
TTB change - test if a change to a code provides exactly the same results

Provide a list with runid's for all runs to be compared:
 o the first is the id for the reference run,
   output of other runs is compared to this one;
 o proper settings for each run should be set in
   the template rcfile using '#if' keywords etc:

    ttb.change.runids             :  ref test

If it is not necessary to run some of these, for example
because their code did not change, include their id's in
a second list; for example, to skip the "ref" run:

    ttb.change.runids.no-run      :  ref

To define what of the output should be compared,
provide a list with one or more keywords:
  'restart'            : restart files (best choice!)
  'save'               : save files
  'save-cy1'           : save files in cy1 format
  'budget_global'      : global budget file
  'budget_global-cy1'  : global budget file in cy1 format (dataset for each region)
For example, to just compare the restart files:

    ttb.change.compare            :  restart

"""


# ***


def MyLogger( name ) :
    # external:
    import logging
    import sys
    # setup logger:
    logger = logging.getLogger( name )
    # no handlers yet ? then print to standard output:
    if len(logger.handlers) == 0 : logger.addHandler(logging.StreamHandler(sys.stdout))
    # no level set yet ? then set for info and higher:
    if logger.level == 0 : logger.setLevel(logging.INFO)
    # ok:
    return logger
#enddef


# ***


def Test( rcf, build_jobs=None, build_new=False ) :

    # external:
    import sys
    import os
    import datetime
    import subprocess
    import traceback
    
    # tools:
    import rc
    import ttb_compare
    
    # get logger:
    logger = MyLogger('ttb')
    
    # test name:
    test_name = 'change'
    
    # info ...
    logger.info( '  "%s" test ...' % test_name )
    
    # name of TM5 rcfile:
    tm_rcfile = rcf.get( 'ttb.rcfile' )
    
    # load the tm5 rcfile:
    tm_rcf = rc.RcFile( tm_rcfile )
    
    # flags:
    no_run = rcf.get( 'ttb.no-run', 'bool' )

    # run id's:
    runids = rcf.get( 'ttb.%s.runids' % test_name ).split()
    # do not run some of them:
    runids_no_run = rcf.get( 'ttb.%s.runids.no-run' % test_name ).split()
    
    # keywords for what to compare: save, restart, ...
    to_compares = rcf.get( 'ttb.%s.compare' % test_name ).split()
    
    # is failue an error ? otherwise messages are displayed and testing is continued:
    error_on_failure = rcf.get( 'ttb.%s.error_on_failure' % test_name, 'bool' )

    # init lists:
    test_rcfs = {}
    
    # loop over runs:
    for irun in range(len(runids)) :

        # current value:
        runid = runids[irun]
    
        #
        # run model
        #
    
        # read template rcfile in raw format:
        test_rcf_raw = rc.RcFile( tm_rcfile, raw=True )
        
        # replace runid:
        test_rcf_raw.replace( 'runid', runid )

        # target filename: tmp-<origingal_basename>-<runid>.rc
        bname,ext = os.path.splitext(tm_rcfile)
        test_rcfile_test = 'tmp-ttb-%s-%s.rc' % (test_name,runid)
        # write:
        test_rcf_raw.WriteFile( test_rcfile_test )
        
        # read again:
        test_rcf = rc.RcFile( test_rcfile_test )

        # store name:
        test_rcfs[runid] = test_rcf
        
        # run if necessary:
        if (not no_run) and (runid not in runids_no_run) :
        
            logger.info( '' )
            logger.info( '******************************************************************' )
            logger.info( '' )
            logger.info( '%s' % runid )
            logger.info( '' )
            logger.info( '******************************************************************' )
            logger.info( '' )

            # setup and submit command:
            command = [os.path.join(os.curdir,'setup_tm5'), test_rcfile_test, '--submit']
            if build_new : command = command + ['--new']
            if build_jobs != None : command = command + ['--jobs',build_jobs]
            retcode = subprocess.call( command )
            if retcode != 0 :
                logger.error( 'from call:' )
                logger.error( '  %s' % command )
                raise Exception
            #endif
            
        #endif  # run model

        #
        # check results
        #

        # can be compared to reference ?
        if irun > 0 : 

            # runid of reference is the first:
            runid_ref = runids[0]

            # flag ...
            found_diffs = False

            # loop over comparisions:
            for to_compare in to_compares :

                # select:
                if to_compare == 'restart' :

                    # which regions ?
                    regions = test_rcf.get( 'regions' ).split()

                    # end time:    
                    tfmt = '%Y-%m-%d %H:%M:%S'
                    t = datetime.datetime.strptime( tm_rcf.get('timerange.end'), tfmt )

                    # info ...
                    logger.info( '' )
                    logger.info( 'compare output files ...' )

                    # loop over regions
                    for region in regions :

                        # files to be compared:
                        fnames = []
                        for runid2 in [runid_ref,runid] :
                            # directory with restart files:
                            fdir = test_rcfs[runid2].get('restart.store.dir')
                            # target file:
                            fnam = 'TM5_restart_%s_%s.nc' % (t.strftime('%Y%m%d_%H%M'),region)
                            # add:
                            fnames.append( os.path.join(fdir,fnam) )
                        #endfor

                        # compare ...
                        try :
                            ttb_compare.df_files( fnames[0], fnames[1] )
                        except ValueError :
                            found_diffs = True
                        except :
                            for line in traceback.format_exc().split('\n') : logger.error(line)
                            raise Exception
                        #endtry

                    #endfor  # regions

                elif to_compare == 'save-cy1' :

                    # which regions ?
                    regions = test_rcf.get( 'regions' ).split()

                    # info ...
                    logger.info( '' )
                    logger.info( 'compare save files ...' )

                    # loop over regions
                    for region in regions :

                        # files to be compared:
                        fnames = []
                        for runid2 in [runid_ref,runid] :
                            # target file:
                            fnam = 'save_%s.hdf' % region
                            # directory with save files:
                            output_dir = test_rcfs[runid2].get( 'output.dir' )
                            subdir = test_rcf.get( 'save.output.subdir', default='' )
                            fdir = os.path.join( output_dir, subdir )
                            # add full path:
                            fnames.append( os.path.join(fdir,fnam) )
                        #endfor

                        # compare ...
                        try :
                            ttb_compare.df_files( fnames[0], fnames[1] )
                        except ValueError :
                            found_diffs = True
                        except :
                            for line in traceback.format_exc().split('\n') : logger.error(line)
                            raise Exception
                        #endtry

                    #endfor  # regions

                elif to_compare in ['save'] :

                    # which regions ?
                    regions = test_rcf.get( 'regions' ).split()

                    # end time:    
                    tfmt = '%Y-%m-%d %H:%M:%S'
                    t = datetime.datetime.strptime( tm_rcf.get('timerange.end'  ), tfmt )

                    # info ...
                    logger.info( '' )
                    logger.info( 'compare %s files ...' % to_compare )

                    # loop over regions
                    for region in regions :

                        # files to be compared:
                        fnames = []
                        for runid2 in [runid_ref,runid] :
                            # target file:
                            fnam = '%s_%s_%s.hdf' % (to_compare,t.strftime('%Y%m%d%H%M'),region)
                            # directory with save files:
                            output_dir = test_rcfs[runid2].get( 'output.dir' )
                            subdir = test_rcf.get( 'save.output.subdir', default='' )
                            fdir = os.path.join( output_dir, subdir )
                            # add full path:
                            fnames.append( os.path.join(fdir,fnam) )
                        #endfor

                        # compare ...
                        try :
                            ttb_compare.df_files( fnames[0], fnames[1] )
                        except ValueError :
                            found_diffs = True
                        except :
                            for line in traceback.format_exc().split('\n') : logger.error(line)
                            raise Exception
                        #endtry
                            

                    #endfor  # regions

                elif to_compare in ['adj_save'] :

                    # which regions ?
                    regions = test_rcf.get( 'regions' ).split()

                    # end time:    
                    tfmt = '%Y-%m-%d %H:%M:%S'
                    t = datetime.datetime.strptime( tm_rcf.get('timerange.start'  ), tfmt )

                    # info ...
                    logger.info( '' )
                    logger.info( 'compare %s files ...' % to_compare )

                    # loop over regions
                    for region in regions :

                        # files to be compared:
                        fnames = []
                        for runid2 in [runid_ref,runid] :
                            # target file:
                            fnam = '%s_%s_%s.hdf' % (to_compare,t.strftime('%Y%m%d%H%M'),region)
                            # directory with save files:
                            output_dir = test_rcfs[runid2].get( 'output.dir' )
                            subdir = test_rcf.get( 'save.output.subdir', default='' )
                            fdir = os.path.join( output_dir, subdir )
                            # add full path:
                            fnames.append( os.path.join(fdir,fnam) )
                        #endfor

                        # compare ...
                        try :
                            ttb_compare.df_files( fnames[0], fnames[1] )
                        except ValueError :
                            found_diffs = True
                        except :
                            for line in traceback.format_exc().split('\n') : logger.error(line)
                            raise Exception
                        #endtry
                            

                    #endfor  # regions

                elif to_compare == 'budget_global-cy1' :

                    # info ...
                    logger.info( '' )
                    logger.info( 'compare budget_global files ...' )

                    # files to be compared:
                    fnames = []
                    for runid2 in [runid_ref,runid] :
                        # target file:
                        fnam = 'budget_global.hdf'
                        # directory with save files:
                        output_dir = test_rcfs[runid2].get( 'output.dir' )
                        subdir = test_rcf.get( 'budget.output.subdir', default='' )
                        fdir = os.path.join( output_dir, subdir )
                        # add full path:
                        fnames.append( os.path.join(fdir,fnam) )
                    #endfor

                    # budget_global.hdf files contain datasets with the same name
                    # for different regions:
                    for iregion in range(len(regions)) :
                        # compare ...
                        try :
                            ttb_compare.df_files( fnames[0], fnames[1], sample=iregion+1 )
                        except ValueError :
                            found_diffs = True
                        except :
                            for line in traceback.format_exc().split('\n') : logger.error(line)
                            raise Exception
                        #endtry
                    #endfor

                elif to_compare == 'budget_global' :

                    # info ...
                    logger.info( '' )
                    logger.info( 'compare budget_global files ...' )

                    # files to be compared:
                    fnames = []
                    for runid2 in [runid_ref,runid] :
                        # target file:
                        fnam = 'budget_global.hdf'
                        # directory with save files:
                        output_dir = test_rcfs[runid2].get( 'output.dir' )
                        subdir = test_rcf.get( 'budget.output.subdir', default='' )
                        fdir = os.path.join( output_dir, subdir )
                        # add full path:
                        fnames.append( os.path.join(fdir,fnam) )
                    #endfor

                    # compare ...
                    try :
                        ttb_compare.df_files( fnames[0], fnames[1] )
                    except ValueError :
                        found_diffs = True
                    except :
                        for line in traceback.format_exc().split('\n') : logger.error(line)
                        raise Exception
                    #endtry

                else :

                    logger.error( 'unsupported comparision : %s' % to_compare )
                    raise ValueError

                #endif

            #endfor   # comparisons

            # any differences ?
            if found_diffs and error_on_failure :
                logger.error( 'found some differences; quit' )
                return
            #endif

        #endif   $ irun > 0

        #
        # *
        #

    #endfor  # runids

#enddef