diff --git a/pykat/testing/__init__.py b/pykat/testing/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/pykat/testing/test.py b/pykat/testing/test.py index b8939b84d4ada300bb6f4533abdbc2ab3970a016..875e1fb794be98337b6ed4e5ef0d1f839dc5ba81 100644 --- a/pykat/testing/test.py +++ b/pykat/testing/test.py @@ -1,5 +1,6 @@ - #!/bin/python - +#!/bin/python +from threading import Thread, Lock +from time import sleep from optparse import OptionParser import os import subprocess as sub @@ -11,11 +12,9 @@ import smtplib import string import time import pickle -import datetime - -options = None -diff_rel_eps = 1e-13 -GIT_BIN = "" +from datetime import datetime +from pykat.testing import utils +import sys class RunException(Exception): def __init__(self, returncode, args, err, out): @@ -39,226 +38,364 @@ def runcmd(args): return [out,err] -def git(args): - p = sub.Popen([GIT_BIN] + args, stdout=sub.PIPE, stderr=sub.PIPE) - out, err = p.communicate() - - if p.returncode != 0: - print err - raise RunException(p.returncode, args, err, out) - - return [out, err] - - - BASE_DIR = os.getcwd() - - if not options.suites: - suites = ["physics","random"] - else: - suites = [] - suites.extend(options.suites.split(",")) - - - if not options.git: - GIT_BIN = "/usr/git/bin" - else: - GIT_BIN = options.git - - if not options.fast: - run_fast = False - else: - run_fast = True - print "Running fast test" - - # Firstly we need to build the latest version of finesse - if os.path.isdir("build") and not options.nobuild: - print "deleting build dir..." - shutil.rmtree("build") - - print "Checking out finesse base..." - git(["clone","git://gitmaster.atlas.aei.uni-hannover.de/finesse/base.git","build"]) - - os.chdir("build") - print "Checking out develop version of finesse..." - runcmd(["./finesse.sh","--checkout","develop"]) - print "Building finesse..." - runcmd(["./finesse.sh","--build"]) - os.chdir(BASE_DIR) - - # check if kat runs - if not os.path.exists("./build/kat"): - raise Exception("Kat file was not found") - - FINESSE_EXE = os.path.join(os.getcwd(),"build","kat") - - print "kat file found in " + FINESSE_EXE - - - OUTPUTS_DIR = os.path.join(BASE_DIR,"outputs") - if os.path.isdir(OUTPUTS_DIR): - print "deleting outputs dir..." - shutil.rmtree(OUTPUTS_DIR) - - os.mkdir(OUTPUTS_DIR) - - os.environ["KATINI"]=os.path.join(BASE_DIR,"build","kat.ini") - - # Clean up and pull latest test repository - os.chdir(os.path.join(options.test_git)) - print "Cleaning test repository...." - git(["clean","-xdf"]) - git(["reset","--hard"]) - print "Pulling latest test..." - git(["pull"]) - - # Define storage structures for generating report later - - kat_run_exceptions = {} - output_differences = {} - run_times = {} - - # create dictionary structures - for suite in suites: - kat_run_exceptions[suite] = {} - output_differences[suite] = {} - run_times[suite] = {} - - - for suite in suites: - print "Running suite: " + suite + "..." - kats = [] - os.chdir(os.path.join(options.test_git,"kat_test",suite)) - - for files in os.listdir("."): - if files.endswith(".kat"): - kats.append(files) - - SUITE_OUTPUT_DIR = os.path.join(OUTPUTS_DIR,suite) - os.mkdir(SUITE_OUTPUT_DIR) - - for kat in kats: - print "Running kat: " + kat - basename = os.path.splitext(kat)[0] - - if run_fast and ('map ' in open(kat).read()): - print "skipping " + kat - else: - try: - start = time.time() - out,err = runcmd([FINESSE_EXE, "--noheader", kat]) - finish = time.time()-start - run_times[suite][kat] = finish - shutil.move(basename + ".out", SUITE_OUTPUT_DIR) - except RunException as e: - print "Error running " + kat + ": " + e.err - kat_run_exceptions[suite][kat] = e - - for suite in suites: - if len(kat_run_exceptions[suite].keys()) > 0: - print "Could not run the following kats:\n" + "\n".join(kat_run_exceptions.keys()) + " in " + suite - else: - print "No errors whilst running" + suite - - - # Now we have generated the output files compare them to the references - for suite in suites: - print "Diffing suite: " + suite + "..." - - outs = [] - os.chdir(os.path.join(OUTPUTS_DIR,suite)) - - for files in os.listdir("."): - if files.endswith(".out"): - outs.append(files) - - REF_DIR = os.path.join(options.test_git,"kat_test",suite,"reference") - - if not os.path.exists(REF_DIR): - raise Exception("Suite reference directory doesn't exist: " + REF_DIR) - for out in outs: - #print "Diffing " + out - ref_file = os.path.join(REF_DIR,out) - - if not os.path.exists(ref_file): - raise DiffException("Reference file doesn't exist for " + out, out) - ref_arr = np.loadtxt(ref_file) - out_arr = np.loadtxt(out) - - if ref_arr.shape != out_arr.shape: - raise DiffException("Reference and output are different shapes", out) - - # for computing relative errors we need to make sure we - # have no zeros in the data - ref_arr_c = np.where(ref_arr == 0, ref_arr, 1) - ref_arr_c[ref_arr_c==0] = 1 - - rel_diff = np.abs(out_arr-ref_arr)/np.abs(ref_arr_c) - - diff = np.any(rel_diff >= diff_rel_eps) - - if diff: - # store the rows which are different - ix = np.where(rel_diff >= diff_rel_eps)[0][0] - output_differences[suite][out] = (ref_arr[ix], out_arr[ix], np.max(rel_diff)) - - - os.chdir(BASE_DIR) - if not os.path.exists("reports"): - os.mkdir("reports") - - os.chdir("reports") - today = datetime.datetime.utcnow() - reportname = today.strftime('%d%m%y') - print "Writing report to " + reportname - - f = open(reportname,'w') - f.write("Python Nightly Test\n") - f.write(today.strftime('%A, %d. %B %Y %I:%M%p') + "\n") - - - # add kat file header - p = sub.Popen([FINESSE_EXE], stdout=sub.PIPE, stderr=sub.PIPE) - out, err = p.communicate() - f.write(out) - - # Now time to generate a report... - np.set_printoptions(precision=16) - - isError = False - - for suite in suites: - f.write("\n\n" + str(len(output_differences[suite].keys())) + " differences in suite " + suite) - for k in output_differences[suite].keys(): - isError = True - f.write(k + ":\n") - f.write(" ref: " + str(output_differences[suite][k][0]) + "\n") - f.write(" out: " + str(output_differences[suite][k][1]) + "\n") - f.write(" Max relative difference: " + str(output_differences[suite][k][2]) + "\n") - - f.write("\n\n" + str(len(output_differences[suite].keys())) + " errors in suite " + suite) - for k in kat_run_exceptions[suite].keys(): - isError = True - f.write(k + ":\n") - f.write("err: " + kat_run_exceptions[suite][k].err + "\n") - - - - f.close() - - if options.emails: - - if isError: - subject = "Finesse test ERROR" - else: - subject = "Finesse test OK" - - emails = options.emails - - args = ["mailx", "-s", subject, emails] - p = sub.Popen(args, stdout=sub.PIPE, stderr=sub.PIPE, stdin=sub.PIPE) - r = open(reportname,"r") - out, err = p.communicate(r.read()) - else: - print "No emails specified" - - +class FinesseTestProcess(Thread): + + queue_time = None + status = "Not started" + built = False + total_kats = 0 + done_kats = 0 + git_commit = "" + test_id = -1 + finished_test = False + diff_rel_eps = 1e-13 + running_kat = "" + running_suite = "" + + def __init__(self, TEST_DIR, BASE_DIR, git_commit, + run_fast=False, suites=[], test_id="0", + git_bin="",emails="", nobuild=False,*args, **kqwargs): + + Thread.__init__(self) + self.git_commit = git_commit + self.queue_time = datetime.now() + self.test_id = test_id + self.TEST_DIR = TEST_DIR + self.BASE_DIR = BASE_DIR + self.emails = "" + + if type(nobuild) is str: + if nobuild.lower() == "true": + self.nobuild = True + elif nobuild.lower() == "false": + self.nobuild = False + else: + raise Exception("nobuild is not a boolean value") + elif type(nobuild) is bool: + self.nobuild = nobuild + else: + raise Exception("nobuild is not a boolean value") + + if type(run_fast) is str: + if run_fast.lower() == "true": + self.run_fast = True + elif run_fast.lower() == "false": + self.run_fast = False + else: + raise Exception("run_fast is not a boolean value") + + elif type(run_fast) is bool: + self.run_fast = run_fast + else: + raise Exception("nobuild is not a boolean value") + + if not os.path.isdir(self.BASE_DIR): + raise Exception("BASE_DIR was not a valid directory") + + if not os.path.isdir(self.TEST_DIR): + raise Exception("TEST_DIR was not a valid directory, should point to a clone of the FINESSE test repository") + + if not suites: + self.suites = ["physics","random"] + else: + self.suites = [] + self.suites.extend(suites) + + self.GIT_BIN = git_bin + + def percent_done(self): + return 100.0*float(self.done_files)/float(self.total_files) + + def get_version(self): + return self.git_commit + + def get_progress(self): + if self.built: + return '{0} out of {1} ({2} in {3})'.format(self.done_kats, self.total_kats,self.running_kat, self.running_suite) + else: + return 'Building FINESSE executable' + + def startFinesseTest(self): + if sys.platform == "win32": + EXE = ".exe" + else: + EXE = "" + + self.built = False + + print type(self.nobuild), self.nobuild + + if not os.path.exists("build"): + os.mkdir("build") + + # Firstly we need to build the latest version of finesse + if os.path.isdir("build") and not self.nobuild: + print "deleting build dir..." + shutil.rmtree("build") + + print "Checking out finesse base..." + utils.git(["clone","git://gitmaster.atlas.aei.uni-hannover.de/finesse/base.git","build"]) + + os.chdir("build") + print "Checking out and building develop version of finesse..." + + if sys.platform == "win32": + runcmd(["bash","./finesse.sh","--checkout","develop"]) + runcmd(["bash","./finesse.sh","--build"]) + else: + EXE = "" + runcmd(["./finesse.sh","--checkout","develop"]) + runcmd(["./finesse.sh","--build"]) + + os.chdir(self.BASE_DIR) + + FINESSE_EXE = os.path.join(self.BASE_DIR,"build","kat" + EXE) + + # check if kat runs + if not os.path.exists(FINESSE_EXE): + raise Exception("Kat file was not found") + + self.built = True + + print "kat file found in " + FINESSE_EXE + + OUTPUTS_DIR = os.path.join(self.BASE_DIR,"outputs") + + if os.path.isdir(OUTPUTS_DIR): + print "deleting outputs dir..." + shutil.rmtree(OUTPUTS_DIR) + + os.mkdir(OUTPUTS_DIR) + + os.environ["KATINI"] = os.path.join(self.TEST_DIR,"kat.ini") + + # Clean up and pull latest test repository + print "Cleaning test repository..." + os.chdir(self.TEST_DIR) + utils.git(["clean","-xdf"]) + utils.git(["reset","--hard"]) + print "Pulling latest test..." + utils.git(["pull"]) + + # Define storage structures for generating report later + kat_run_exceptions = {} + output_differences = {} + run_times = {} + + self.total_kats = 0 + + # create dictionary structures + # and count up total number of files to process + for suite in self.suites: + kat_run_exceptions[suite] = {} + output_differences[suite] = {} + run_times[suite] = {} + + os.chdir(os.path.join(self.TEST_DIR,"kat_test",suite)) + print suite + + for files in os.listdir("."): + if files.endswith(".kat"): + self.total_kats += 1 + print self.total_kats + + for suite in self.suites: + print "Running suite: " + suite + "..." + kats = [] + os.chdir(os.path.join(self.TEST_DIR,"kat_test",suite)) + + for files in os.listdir("."): + if files.endswith(".kat"): + kats.append(files) + + SUITE_OUTPUT_DIR = os.path.join(OUTPUTS_DIR,suite) + os.mkdir(SUITE_OUTPUT_DIR) + + self.running_suite = suite + + for kat in kats: + self.running_kat = kat + + print self.get_progress() + basename = os.path.splitext(kat)[0] + + if self.run_fast and ('map ' in open(kat).read()): + print "skipping " + kat + else: + try: + start = time.time() + out,err = runcmd([FINESSE_EXE, "--noheader", kat]) + finish = time.time()-start + run_times[suite][kat] = finish + shutil.move(basename + ".out", SUITE_OUTPUT_DIR) + except RunException as e: + print "Error running " + kat + ": " + e.err + kat_run_exceptions[suite][kat] = e + finally: + self.done_kats += 1 + + for suite in self.suites: + if len(kat_run_exceptions[suite].keys()) > 0: + print "Could not run the following kats:\n" + "\n".join(kat_run_exceptions.keys()) + " in " + suite + else: + print "No errors whilst running" + suite + + + # Now we have generated the output files compare them to the references + for suite in self.suites: + print "Diffing suite: " + suite + "..." + + outs = [] + os.chdir(os.path.join(OUTPUTS_DIR,suite)) + + for files in os.listdir("."): + if files.endswith(".out"): + outs.append(files) + + REF_DIR = os.path.join(self.TEST_DIR,"kat_test",suite,"reference") + + if not os.path.exists(REF_DIR): + raise Exception("Suite reference directory doesn't exist: " + REF_DIR) + for out in outs: + #print "Diffing " + out + ref_file = os.path.join(REF_DIR,out) + + if not os.path.exists(ref_file): + raise DiffException("Reference file doesn't exist for " + out, out) + ref_arr = np.loadtxt(ref_file) + out_arr = np.loadtxt(out) + + if ref_arr.shape != out_arr.shape: + raise DiffException("Reference and output are different shapes", out) + + # for computing relative errors we need to make sure we + # have no zeros in the data + ref_arr_c = np.where(ref_arr == 0, ref_arr, 1) + ref_arr_c[ref_arr_c==0] = 1 + + rel_diff = np.abs(out_arr-ref_arr)/np.abs(ref_arr_c) + + diff = np.any(rel_diff >= self.diff_rel_eps) + + if diff: + # store the rows which are different + ix = np.where(rel_diff >= self.diff_rel_eps)[0][0] + output_differences[suite][out] = (ref_arr[ix], out_arr[ix], np.max(rel_diff)) + + os.chdir(BASE_DIR) + if not os.path.exists("reports"): + os.mkdir("reports") + + os.chdir("reports") + today = datetime.datetime.utcnow() + reportname = today.strftime('%d%m%y') + print "Writing report to " + reportname + + f = open(reportname,'w') + f.write("Python Nightly Test\n") + f.write(today.strftime('%A, %d. %B %Y %I:%M%p') + "\n") + + # add kat file header + p = sub.Popen([FINESSE_EXE], stdout=sub.PIPE, stderr=sub.PIPE) + out, err = p.communicate() + f.write(out) + + # Now time to generate a report... + np.set_printoptions(precision=16) + + isError = False + + for suite in suites: + f.write("\n\n" + str(len(output_differences[suite].keys())) + " differences in suite " + suite) + for k in output_differences[suite].keys(): + isError = True + f.write(k + ":\n") + f.write(" ref: " + str(output_differences[suite][k][0]) + "\n") + f.write(" out: " + str(output_differences[suite][k][1]) + "\n") + f.write(" Max relative difference: " + str(output_differences[suite][k][2]) + "\n") + + f.write("\n\n" + str(len(output_differences[suite].keys())) + " errors in suite " + suite) + for k in kat_run_exceptions[suite].keys(): + isError = True + f.write(k + ":\n") + f.write("err: " + kat_run_exceptions[suite][k].err + "\n") + + f.close() + + if self.emails: + + if isError: + subject = "Finesse test ERROR" + else: + subject = "Finesse test OK" + + emails = self.emails + + args = ["mailx", "-s", subject, emails] + p = sub.Popen(args, stdout=sub.PIPE, stderr=sub.PIPE, stdin=sub.PIPE) + r = open(reportname,"r") + out, err = p.communicate(r.read()) + else: + print "No emails specified" + + def run(self): + + try: + + self.startFinesseTest() + + finally: + finished_test = True + + # once done check if any other tests need to be ran + #schedule_lock.acquire() + + #if len(scheduled_tests) > 0: + # current_test = scheduled_tests.pop(0) + # current_test.start() + #else: + # current_test = None + + #schedule_lock.release() + + +if __name__ == "__main__": + + parser = OptionParser() + + parser.add_option("-t","--test-dir",dest="test_dir",help="") + parser.add_option("-b","--base-dir",dest="base_dir",help="") + parser.add_option("-c","--test-commit",dest="test_commit",help="") + parser.add_option("-s","--suites",dest="suites",help="comma delimited list of each suite to run") + parser.add_option("-g","--git-bin",dest="git_bin", default="git",help="") + parser.add_option("-e","--emails",dest="emails", help="") + parser.add_option("-n","--no-build",default="False",dest="nobuild",action="store_true") + parser.add_option("-f","--fast",default="True",dest="fast",action="store_true") + + options, args = parser.parse_args() + + if options.test_dir is None: + print "--test-dir argument is missing" + exit() + + if options.test_commit is None: + print "--test-commit argument is missing" + exit() + + if options.base_dir is None: + options.base_dir = os.getcwd() + + if options.suites is None: + suites = [] + else: + suites = options.suites.split(",") + + test = FinesseTestProcess(options.test_dir, + options.base_dir, + options.test_commit, + run_fast=options.fast, + suites=suites, + git_bin=options.git_bin, + emails=options.emails, + nobuild=options.nobuild) + test.run() \ No newline at end of file diff --git a/pykat/testing/utils.py b/pykat/testing/utils.py index f49a2fa0b948589444a11d9f5f9dc70a77d50fa7..2e1c6a95b676418ad139ee9777a452db87ac2612 100644 --- a/pykat/testing/utils.py +++ b/pykat/testing/utils.py @@ -18,9 +18,7 @@ def git(args): cmd = " ".join(args) else: cmd = GIT_BIN + " " + args - - print cmd - + p = sub.Popen(cmd, stdout=sub.PIPE, stderr=sub.PIPE) out, err = p.communicate() diff --git a/pykat/testing/web/__init__.py b/pykat/testing/web/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6b12d6e49d7f31ea13d3768be39104192ce12f49 --- /dev/null +++ b/pykat/testing/web/__init__.py @@ -0,0 +1,5 @@ +from flask import Flask +import os + +global app +app = Flask(__name__, instance_path=os.getcwd()) \ No newline at end of file diff --git a/pykat/testing/templates/finesse_test.html b/pykat/testing/web/templates/finesse_test.html similarity index 100% rename from pykat/testing/templates/finesse_test.html rename to pykat/testing/web/templates/finesse_test.html diff --git a/pykat/testing/web_interface.py b/pykat/testing/web/web_interface.py similarity index 65% rename from pykat/testing/web_interface.py rename to pykat/testing/web/web_interface.py index 121072162cfc2e50b1921fa2ede61db71c9fac09..70f7b0d5167778b974c446c816abd9cbeeb57a66 100644 --- a/pykat/testing/web_interface.py +++ b/pykat/testing/web/web_interface.py @@ -8,8 +8,10 @@ from flask import request from flask import render_template from datetime import datetime from collections import namedtuple +from pykat.testing import utils + +from pykat.testing.web import app -import utils import os global current_test, scheduled_tests, schedule_lock @@ -18,8 +20,8 @@ test_id = 0 current_test = None scheduled_tests = [] schedule_lock = Lock() - -app = Flask(__name__) + +print "loading web interface" @app.route('/') def hello_world(): @@ -109,59 +111,7 @@ def finesse_get_log(count): return jsonify(logs=log2send) - -class FinesseTestProcess(Thread): - - queue_time = None - status = "Not started" - built = False - total_files = 100 - done_files = 0 - suite = "" - git_commit = "" - test_id = -1 - - def __init__(self, git_commit, test_id, *args, **kqwargs): - Thread.__init__(self) - self.git_commit = git_commit - self.queue_time = datetime.now() - self.test_id = test_id - - def run(self): - global current_test, scheduled_tests, schedule_lock - - for x in range(0,self.total_files): - sleep(0.1) - self.done_files += 1 - - if x > 10: - self.built = True - - # once done check if any other tests need to be ran - schedule_lock.acquire() - - if len(scheduled_tests) > 0: - current_test = scheduled_tests.pop(0) - current_test.start() - else: - current_test = None - - schedule_lock.release() - - - def percent_done(self): - return 100.0*float(self.done_files)/float(self.total_files) - - def get_version(self): - return self.git_commit - - def get_progress(self): - if self.built: - return '{0} out of {1} ({2})'.format(self.done_files, self.total_files, self.suite) - else: - return 'Building FINESSE...' - - + class FinesseTestProcessRun(): def __init__(self, init_class=FinesseTestProcess): self.init_class = init_class @@ -185,16 +135,4 @@ class FinesseTestProcessRun(): print fjp.get_progress() print '%s threaded process complete. Now exiting.' % fjp.__class__.__name__ - - -if __name__ == '__main__': - - # need local copy of src - if not os.path.exists("./finesse_src"): - print "finesse src folder didn't exist, cloning now..." - utils.git("clone git://gitmaster.atlas.aei.uni-hannover.de/finesse/src.git finesse_src") - else: - # get the latest version for logs etc. - utils.git("--git-dir ./finesse_src/.git pull") - - app.run(debug=True) \ No newline at end of file + \ No newline at end of file diff --git a/pykat/testing/web_server.py b/pykat/testing/web_server.py new file mode 100644 index 0000000000000000000000000000000000000000..ebb27d74ad5502199ebc1dbb089d5e0a836c3168 --- /dev/null +++ b/pykat/testing/web_server.py @@ -0,0 +1,64 @@ +import os +import sys +from flask import Flask +from pykat.testing import utils + +def start(instance_path,port=5000, debug=True): + global app + + print "starting web server..." + + if instance_path is None: + raise Exception("instance_path must be defined") + elif type(instance_path) is not str: + raise Exception("instance_path must be a string") + + if not os.path.exists(instance_path): + os.mkdir(instance_path) + + os.chdir(instance_path) + + from pykat.testing.web import app + + # load up the actual interface code + import pykat.testing.web.web_interface + + # need local copy of src + if not os.path.exists("./finesse_src"): + print "finesse src folder didn't exist, cloning now..." + utils.git("clone git://gitmaster.atlas.aei.uni-hannover.de/finesse/src.git finesse_src") + else: + # get the latest version for logs etc. + utils.git("--git-dir ./finesse_src/.git pull") + + app.secret_key = os.urandom(24) + app.run(debug=debug, port=int(port)) + +if __name__ == "__main__": + n = len(sys.argv) + + if n != 2 and n != 3: + print """ +Command starts a Flask based website on which allows +users to manage the FINESSE test builds. + +start_test_server usage: + + python -m pykat.testing.web_server <server_path> <port> + + Arguments: + server_path: Directory where website should be run from + port: TCP port which the server should run under + default is port 5000. +""" + exit() + + instance_path = sys.argv[1] + + if n == 3: + port = sys.argv[2] + else: + port = 5000 + + start(instance_path, port=port) + \ No newline at end of file diff --git a/setup.py b/setup.py index 6f9b1fab8797808d1205867fd050c57c547c9087..e5adc0a9abff8863a68c5ea4f688024dca6d4c34 100644 --- a/setup.py +++ b/setup.py @@ -19,6 +19,7 @@ setup( long_description=open('README.txt').read(), install_requires=[ "PyQt4 >= 4.8.3", - "numpy >= 1.6.2" + "numpy >= 1.6.2", + "flask >= 0.10.1" ], ) \ No newline at end of file