Skip to content
Snippets Groups Projects
Select Git revision
  • 03845695bb7bb3edc13744a8b1b15549cd9d15f0
  • master default protected
  • 72-improve-docs-for_optimal_setup
  • os-path-join
  • develop-GA
  • add-higher-spindown-components
  • v1.3
  • v1.2
  • v1.1.2
  • v1.1.0
  • v1.0.1
11 results

generate_data.py

Blame
  • test.py 19.53 KiB
    #!/bin/python
    import gzip
    from threading import Thread, Lock
    from time import sleep
    from optparse import OptionParser
    import os
    import multiprocessing
    from multiprocessing import Pool, Queue
    import subprocess as sub
    import numpy as np
    import difflib
    from StringIO import StringIO
    import shutil
    import smtplib
    import string
    import time
    import pickle
    from datetime import datetime
    from pykat.testing import utils
    import sys, traceback
    import stat
    import math
    
    def initProcess(dkats):
        #print "init!!!", dkats
        global done_kats
        done_kats = dkats
    
    def run_kat_file(item):
        kat = item["kat"]
        suite = item["suite"]
        runtime = 0.0;
        try:
            #print os.getpid(),"getting kat...",item["kat"]
            global done_kats
            
            FINESSE_EXE = item["FINESSE_EXE"]
            SUITE_PATH = item["SUITE_PATH"]
            SUITE_OUTPUT_DIR  = item["SUITE_OUTPUT_DIR"]
            basename = os.path.splitext(kat)[0]
            exp = None
            
            if item["run_fast"] and ('map ' in open(kat).read()):
                print "skipping " + kat			
            else:
                
                #try:
                start = time.time()
                
                out,err = utils.runcmd([FINESSE_EXE, "--noheader", kat], cwd=SUITE_PATH)
                runtime = time.time()-start
                
                OUT_FILE = os.path.join(SUITE_PATH,basename + ".out")
                LOG_FILE = os.path.join(SUITE_PATH,basename + ".log")
                
                f_in = open(LOG_FILE, 'rb')
                f_out = gzip.open(LOG_FILE + ".gz", 'wb')
                f_out.writelines(f_in)
                f_out.close()
                f_in.close()
                
                shutil.move(OUT_FILE, SUITE_OUTPUT_DIR)
                shutil.move(LOG_FILE + ".gz", SUITE_OUTPUT_DIR)
                
                #except utils.RunException as e:
                #
                 #   print "STDERR: " + e.out
                  #  print "STDOUT: " + e.err
                   # 
                   # print "Error running " + kat
                    
                  #  exp = e
                #finally:
                done_kats.value += 1
                #    return [time.time()-start, suite, kat, exp]
                        
        except Exception as e:
            print "main error in kat call"
            exp = None
        finally:
            # Annoyingly it seems that exceptions can't be returned
    	# from a process that has been mapped because exceptions
    	# or some at least, can't be pickled, so they will appear as 
    	# nan in the tests
            return [runtime, suite, kat, None] #[runtime,suite,kat,e]
            
    
    class DiffException(Exception):
    	def __init__(self, msg, outfile):
    		self.msg = msg
    		self.outfile = outfile
    
    class FinesseTestProcess(Thread):
            
        def __init__(self, TEST_DIR, BASE_DIR, test_commit, 
                     run_fast=False, kats={}, test_id="0",
                     git_bin="",emails="", nobuild=False, pool_size=int(multiprocessing.cpu_count()*3.5/4.0),*args, **kqwargs):
                     
             
            self.queue_time = None
            self.status = ""
            self.built = False
            self.total_kats = 0
            self.done_kats = multiprocessing.Value('i', 0)
            self.git_commit = ""
            self.test_id = -1
            self.finished_test = False
            self.diff_rel_eps = 1e-12
            self.running_kat = ""
            self.running_suite = ""
            self.cancelling = False
            self.errorOccurred = None
            self.diffFound = False
            self.diffing = False
            
            if pool_size < 1:
                self.pool_size = 1
            else:
                self.pool_size = pool_size
            
            Thread.__init__(self)
            self.git_commit = test_commit
            
            if test_commit is None:
                raise Exception("A git commit ID must be provided for the test")
            
            self.kat_run_exceptions = {}
            self.output_differences = {}
            self.run_times = {}
            
            self.queue_time = datetime.now()
            self.test_id = test_id
            self.TEST_DIR = TEST_DIR
            self.BASE_DIR = BASE_DIR
            
            self.emails = ""
            
            if type(nobuild) is str:
            
                if nobuild.lower() == "true":
                    self.nobuild = True
                elif nobuild.lower() == "false":
                    self.nobuild = False
                else:
                    raise Exception("nobuild is not a boolean value")
                    
            elif type(nobuild) is bool:
                self.nobuild = nobuild
            else:
                raise Exception("nobuild is not a boolean value")
            
            if type(run_fast) is str:
                if run_fast.lower() == "true":
                    self.run_fast = True
                elif run_fast.lower() == "false":
                    self.run_fast = False
                else:
                    raise Exception("run_fast is not a boolean value")
                    
            elif type(run_fast) is bool:
                self.run_fast = run_fast
            else:
                raise Exception("nobuild is not a boolean value")
                
            if not os.path.isdir(self.TEST_DIR):
                raise Exception("TEST_DIR was not a valid directory, should point to a clone of the FINESSE test repository")
                
            self.kats_to_run = kats
            self.GIT_BIN = git_bin
        
        def cancelCheck(self):
            if self.cancelling:
                raise SystemExit()
        
        def percent_done(self):
            if self.total_kats == 0:
                return 0.0
            else:
                return 100.0*float(self.done_kats.value)/float(self.total_kats)
            
        def get_version(self):
            return self.git_commit
            
        def get_progress(self):
            if self.diffing:
                return 'Diffing {0} out of {1} ({2} in {3})'.format(self.done_kats.value, self.total_kats/2, self.running_kat, self.running_suite)
            if self.built:
                return 'Running {0} out of {1} ({2} in {3})'.format(self.done_kats.value, self.total_kats/2, self.running_kat, self.running_suite)
            else:
                return 'Building FINESSE executable'
                
        def startFinesseTest(self):
            self.done_kats.value = 0
            
            if sys.platform == "win32":
                EXE = ".exe"
            else:
                EXE = ""
            
            print "Using", self.pool_size, "processes..."
            
            self.built = False
    
            BUILD_PATH = os.path.join(self.BASE_DIR, "build")
            
            # Firstly we need to build the latest version of finesse
            if not self.nobuild:
                
                if os.path.exists(self.BASE_DIR):
                    print "Deleting previous base_dir " + self.BASE_DIR
                    shutil.rmtree(self.BASE_DIR)
                
                os.mkdir(self.BASE_DIR)
                
                print "deleting build dir..." + BUILD_PATH
                if os.path.exists(BUILD_PATH):
                    shutil.rmtree(BUILD_PATH)
    
                print "Checking out finesse base..."
                utils.git(["clone","https://gitlab.aei.uni-hannover.de/finesse/finesse.git", BUILD_PATH])
    
                print "Checking out and building develop version of finesse " + self.git_commit
                
                SRC_PATH = os.path.join(BUILD_PATH,"src")
                
                if sys.platform == "win32":                
                    utils.git(["checkout",self.git_commit],cwd=SRC_PATH)
                    self.cancelCheck()
                    
                    utils.runcmd(["bash","./finesse.sh","--build"],cwd=BUILD_PATH)
                    self.cancelCheck()
                else:                
                    utils.git(["checkout",self.git_commit],cwd=SRC_PATH)
                    self.cancelCheck()
                    
                    utils.runcmd(["./finesse.sh","--build"],cwd=BUILD_PATH)
                    self.cancelCheck()
                              
            FINESSE_EXE = os.path.join(self.BASE_DIR,"build","kat" + EXE)
            print "NOBUILD",self.nobuild
            # check if kat runs
            if not os.path.exists(FINESSE_EXE):
                raise Exception("Kat file was not found in " + FINESSE_EXE)
            
            
            if not os.access(FINESSE_EXE, os.X_OK):
                if sys.platform != "win32":
                    print "Trying to chmod " + FINESSE_EXE
                    os.chmod(FINESSE_EXE, stat.S_IRWXU)
            
            out = None
            
            # check version numbers match upkat
            out = utils.runcmd([FINESSE_EXE,"-v"])
                
            # I am sure there is a regex expression that could make this
            # easier but I am being lazy
            out = out[0]
            out = out.split("\n")
            out = out[0].split("(")[1].split(")")
            out = out[0].split("-")[-1]
            shortid = out.lstrip("g")
            
            if shortid != self.git_commit[0:len(shortid)]:
                raise Exception("Version of kat {0} did not match the version that it was requested to build {1}.".format(shortid, self.git_commit[0:len(shortid)]))
            
            self.built = True
            
            print "kat file found in " + FINESSE_EXE
            
            OUTPUTS_DIR = os.path.join(self.BASE_DIR,"outputs")
            
            if os.path.isdir(OUTPUTS_DIR):
                print "deleting outputs dir..."
                shutil.rmtree(OUTPUTS_DIR)
                
            os.mkdir(OUTPUTS_DIR)
            
            os.environ["KATINI"] = os.path.join(BUILD_PATH,"kat.ini")
            
            self.cancelCheck()
            # Clean up and pull latest test repository
            print "Cleaning test repository..."
            utils.git(["clean","-xdf"], cwd=self.TEST_DIR)
            self.cancelCheck()
            utils.git(["reset","--hard"], cwd=self.TEST_DIR)
            self.cancelCheck()
            print "Pulling latest test..."
            utils.git(["pull"],cwd=self.TEST_DIR)
            self.cancelCheck()
        
            self.total_kats = 0
            
            # create dictionary structures
            # and count up total number of files to process
            for suite in self.kats_to_run.keys():
                print "RUNNING SUITES", suite
                
                self.kat_run_exceptions[suite] = {}
                self.output_differences[suite] = {}
                self.run_times[suite] = {}
                
                self.total_kats += len(self.kats_to_run[suite])
            
            # multiply as we include the diffining in the percentage
            # done
            self.total_kats *= 2
            runs = []
            
            for suite in self.kats_to_run.keys():
                self.cancelCheck()
                print "Queuing up suite: " + suite + "..."
                kats = self.kats_to_run[suite]
                SUITE_PATH = os.path.join(self.TEST_DIR,"kat_test",suite)
    
                SUITE_OUTPUT_DIR = os.path.join(OUTPUTS_DIR,suite)
                os.mkdir(SUITE_OUTPUT_DIR)
                
                for kat in kats:
                    runs.append({'SUITE_OUTPUT_DIR':SUITE_OUTPUT_DIR,'suite':suite, 'run_fast':self.run_fast, 'kat':kat, 'FINESSE_EXE':FINESSE_EXE, 'SUITE_PATH':SUITE_PATH})
            
            self.pool = Pool(initializer=initProcess,initargs=(self.done_kats,) ,processes = self.pool_size)    
            results = self.pool.imap_unordered(run_kat_file, runs, 1)
            print "Finsihed running pool of kats"
            self.pool.close()
            
            for result in results:
                
                if result[3] is not None:
                    self.kat_run_exceptions[result[1]][result[2]] = result[3]
                    self.diffFound = True
                
                self.run_times[result[1]][result[2]] = result[0]
            
            self.cancelCheck()
            
            for suite in self.kats_to_run.keys():
                if len(self.kat_run_exceptions[suite].keys()) > 0:
                    print "Could not run the following kats:\n" + "\n".join(self.kat_run_exceptions[suite].keys()) + " in " + suite
                else:
                    print "No exceptions whilst running: " + suite
    
            self.diffing = True
            
            # Now we have generated the output files compare them to the references
            for suite in self.kats_to_run.keys():
                self.cancelCheck()
                print "Diffing suite: " + suite + "..."
    
                outs = []
                SUITE_PATH = os.path.join(OUTPUTS_DIR, suite)
                
                for files in os.listdir(SUITE_PATH):
                    if files.endswith(".out"):
                        outs.append(files)
    
                REF_DIR = os.path.join(self.TEST_DIR,"kat_test",suite,"reference")
                
                if not os.path.exists(REF_DIR):
                    raise Exception("Suite reference directory doesn't exist: " + REF_DIR)
                    
                for out in outs:
                    self.cancelCheck()
                    self.running_kat = out
                    
                    try:        
                        ref_file = os.path.join(REF_DIR,out)
                        out_file = os.path.join(SUITE_PATH,out)
                    
                        if not os.path.exists(ref_file):
                            raise DiffException("Reference file doesn't exist for " + out, out)
                        
                        ref_arr = np.loadtxt(ref_file, dtype=np.float64)
                        out_arr = np.loadtxt(out_file, dtype=np.float64)
                    
                        if ref_arr.shape != out_arr.shape:
                            raise DiffException("Reference and output are different shapes", out)
    
                        # for computing relative errors we need to make sure we
                        # have no zeros in the data
                        nzix = (ref_arr != 0)
                    
                        rel_diff = np.abs(out_arr-ref_arr)
                        # only compute rel diff for non zero values
                        rel_diff[nzix] = np.divide(rel_diff[nzix], np.abs(ref_arr[nzix]))
                    
                        diff = np.any(rel_diff >= self.diff_rel_eps)
                                    
                        if diff:
                            self.diffFound = True
                        
                            # store the rows which are different
                            ix = np.where(rel_diff >= self.diff_rel_eps)[0][0]
                    
                            self.output_differences[suite][out] = (True,
                                                                   ref_arr[ix],
                                                                   out_arr[ix],
                                                                   np.max(rel_diff))
                        else:
                            max = np.max(rel_diff) 
                        
                            self.output_differences[suite][out] = (False, max)
                    
                    
                            
                        # compress the data
                        f_in = open(out_file, 'rb')
                        f_out = gzip.open(out_file + ".gz", 'wb')
                        f_out.writelines(f_in)
                        f_out.close()
                        f_in.close()
                    
                        print "removing out file ", out_file
                        os.remove(out_file)
                        
                    except DiffException as ex:
                        print(str(ex), "output =", out)
                    except:
                        print("Error processing output = ", out)
                    
                    self.done_kats.value += 1
            
            print "Finished diffing..."
            
            REPORT_PATH = os.path.join(self.BASE_DIR,"reports")
            
            if not os.path.exists(REPORT_PATH):
                os.mkdir(REPORT_PATH)
    
            today = datetime.utcnow()
            reportname = os.path.join(REPORT_PATH,"report.log") #today.strftime('%d%m%y')
            print "Writing report to " + reportname
            
            f = open(reportname,'w')
            f.write("Python Nightly Test\n")
            f.write(today.strftime('%A, %d. %B %Y %I:%M%p') + "\n")
    
            # add kat file header
            p = sub.Popen([FINESSE_EXE], stdout=sub.PIPE, stderr=sub.PIPE)
            out, err = p.communicate()
            f.write(out)
            
            # Now time to generate a report...
            np.set_printoptions(precision=16)
            
            isError = False
    
            for suite in self.kats_to_run.keys():
                f.write("\n\n" + str(len(self.output_differences[suite].keys())) + " differences in suite " + suite)
                for k in self.output_differences[suite].keys():
                    isError = True
                    f.write(k + ":\n")
                    if self.output_differences[suite][k][0]:
                        f.write("     Differences larger than " + str(self.diff_rel_eps) + "\n")
                        f.write("     ref: " + str(self.output_differences[suite][k][1]) + "\n")
                        f.write("     out: " + str(self.output_differences[suite][k][2]) + "\n")
                        f.write("     Max relative difference: " + str(self.output_differences[suite][k][3]) + "\n")
                    else:
                        f.write("     Differences smaller than " + str(self.diff_rel_eps) + "\n")
                        f.write("     Max relative difference: " + str(self.output_differences[suite][k][1]) + "\n")
                        
                f.write("\n\n" + str(len(self.output_differences[suite].keys())) + " errors in suite " + suite)
                for k in self.kat_run_exceptions[suite].keys():
                    isError = True
                    f.write(k + ":\n")
                    f.write("err: " + self.kat_run_exceptions[suite][k].err + "\n")
    
            f.close()
            
            #if self.emails:
            #    
            #    if isError:
            #        subject = "Finesse test ERROR"
            #    else:
            #        subject = "Finesse test OK"
            #    emails = self.emails
    
            #    args = ["mailx", "-s", subject, emails]
            #    p = sub.Popen(args, stdout=sub.PIPE, stderr=sub.PIPE, stdin=sub.PIPE)
            #    r = open(reportname,"r")
            #    out, err = p.communicate(r.read())
            #else:
            #    print "No emails specified"
    
                
                
        def run(self):
            
            try:
                self.startFinesseTest()
            except Exception as ex:
                
                exc_type, exc_value, exc_traceback = sys.exc_info()
                
                self.errorOccurred = dict(value=str(exc_value), traceback=str(traceback.format_exc(5)))
                
                if exc_type is utils.RunException:
                    self.errorOccurred["stdout"] = ex.out
                    self.errorOccurred["stderr"] = ex.err
                    
                print "*** Exception for test_id = " + str(self.test_id)
                traceback.print_exception(exc_type, exc_value, exc_traceback,
                                          limit=5, file=sys.stdout)
            finally:
                self.finished_test = True
            
            
    
    if __name__ == "__main__":
        
        parser = OptionParser()
        
        parser.add_option("-t","--test-dir",dest="test_dir",help="")
        parser.add_option("-b","--base-dir",dest="base_dir",help="")
        parser.add_option("-c","--test-commit",dest="test_commit",help="")
        parser.add_option("-s","--suites",dest="suites",help="comma delimited list of each suite to run")
        parser.add_option("-g","--git-bin",dest="git_bin", default="git",help="")
        parser.add_option("-e","--emails",dest="emails", help="")
        parser.add_option("-n","--no-build",default="False",dest="nobuild",action="store_true")
        parser.add_option("-f","--fast",default="True",dest="fast",action="store_true")
    
        options, args = parser.parse_args()
    
        if options.test_dir is None:
            print "--test-dir argument is missing"
            exit()
            
        if options.test_commit is None:
            print "--test-commit argument is missing"
            exit()
        
        if options.base_dir is None:
            options.base_dir = os.getcwd()
        
        if options.suites is None:
            suites = []
        else:
            suites = options.suites.split(",")
            
        test = FinesseTestProcess(options.test_dir,
                                  options.base_dir,
                                  options.test_commit,
                                  run_fast=options.fast,
                                  suites=suites,
                                  git_bin=options.git_bin,
                                  emails=options.emails,
                                  nobuild=options.nobuild)
        test.run()