Commit 9ee26fa6 authored by Daniel Brown's avatar Daniel Brown
Browse files

moving some files around, added new finesse test routines and Flask web interface to package

parent 126cf72b
#!/bin/python #!/bin/python
from threading import Thread, Lock
from time import sleep
from optparse import OptionParser from optparse import OptionParser
import os import os
import subprocess as sub import subprocess as sub
...@@ -11,11 +12,9 @@ import smtplib ...@@ -11,11 +12,9 @@ import smtplib
import string import string
import time import time
import pickle import pickle
import datetime from datetime import datetime
from pykat.testing import utils
options = None import sys
diff_rel_eps = 1e-13
GIT_BIN = ""
class RunException(Exception): class RunException(Exception):
def __init__(self, returncode, args, err, out): def __init__(self, returncode, args, err, out):
...@@ -39,226 +38,364 @@ def runcmd(args): ...@@ -39,226 +38,364 @@ def runcmd(args):
return [out,err] return [out,err]
def git(args): class FinesseTestProcess(Thread):
p = sub.Popen([GIT_BIN] + args, stdout=sub.PIPE, stderr=sub.PIPE)
out, err = p.communicate() queue_time = None
status = "Not started"
if p.returncode != 0: built = False
print err total_kats = 0
raise RunException(p.returncode, args, err, out) done_kats = 0
git_commit = ""
return [out, err] test_id = -1
finished_test = False
diff_rel_eps = 1e-13
BASE_DIR = os.getcwd() running_kat = ""
running_suite = ""
if not options.suites:
suites = ["physics","random"] def __init__(self, TEST_DIR, BASE_DIR, git_commit,
else: run_fast=False, suites=[], test_id="0",
suites = [] git_bin="",emails="", nobuild=False,*args, **kqwargs):
suites.extend(options.suites.split(","))
Thread.__init__(self)
self.git_commit = git_commit
if not options.git: self.queue_time = datetime.now()
GIT_BIN = "/usr/git/bin" self.test_id = test_id
else: self.TEST_DIR = TEST_DIR
GIT_BIN = options.git self.BASE_DIR = BASE_DIR
self.emails = ""
if not options.fast:
run_fast = False if type(nobuild) is str:
else: if nobuild.lower() == "true":
run_fast = True self.nobuild = True
print "Running fast test" elif nobuild.lower() == "false":
self.nobuild = False
# Firstly we need to build the latest version of finesse else:
if os.path.isdir("build") and not options.nobuild: raise Exception("nobuild is not a boolean value")
print "deleting build dir..." elif type(nobuild) is bool:
shutil.rmtree("build") self.nobuild = nobuild
else:
print "Checking out finesse base..." raise Exception("nobuild is not a boolean value")
git(["clone","git://gitmaster.atlas.aei.uni-hannover.de/finesse/base.git","build"])
if type(run_fast) is str:
os.chdir("build") if run_fast.lower() == "true":
print "Checking out develop version of finesse..." self.run_fast = True
runcmd(["./finesse.sh","--checkout","develop"]) elif run_fast.lower() == "false":
print "Building finesse..." self.run_fast = False
runcmd(["./finesse.sh","--build"]) else:
os.chdir(BASE_DIR) raise Exception("run_fast is not a boolean value")
# check if kat runs elif type(run_fast) is bool:
if not os.path.exists("./build/kat"): self.run_fast = run_fast
raise Exception("Kat file was not found") else:
raise Exception("nobuild is not a boolean value")
FINESSE_EXE = os.path.join(os.getcwd(),"build","kat")
if not os.path.isdir(self.BASE_DIR):
print "kat file found in " + FINESSE_EXE raise Exception("BASE_DIR was not a valid directory")
if not os.path.isdir(self.TEST_DIR):
OUTPUTS_DIR = os.path.join(BASE_DIR,"outputs") raise Exception("TEST_DIR was not a valid directory, should point to a clone of the FINESSE test repository")
if os.path.isdir(OUTPUTS_DIR):
print "deleting outputs dir..." if not suites:
shutil.rmtree(OUTPUTS_DIR) self.suites = ["physics","random"]
else:
os.mkdir(OUTPUTS_DIR) self.suites = []
self.suites.extend(suites)
os.environ["KATINI"]=os.path.join(BASE_DIR,"build","kat.ini")
self.GIT_BIN = git_bin
# Clean up and pull latest test repository
os.chdir(os.path.join(options.test_git)) def percent_done(self):
print "Cleaning test repository...." return 100.0*float(self.done_files)/float(self.total_files)
git(["clean","-xdf"])
git(["reset","--hard"]) def get_version(self):
print "Pulling latest test..." return self.git_commit
git(["pull"])
def get_progress(self):
# Define storage structures for generating report later if self.built:
return '{0} out of {1} ({2} in {3})'.format(self.done_kats, self.total_kats,self.running_kat, self.running_suite)
kat_run_exceptions = {} else:
output_differences = {} return 'Building FINESSE executable'
run_times = {}
def startFinesseTest(self):
# create dictionary structures if sys.platform == "win32":
for suite in suites: EXE = ".exe"
kat_run_exceptions[suite] = {} else:
output_differences[suite] = {} EXE = ""
run_times[suite] = {}
self.built = False
for suite in suites: print type(self.nobuild), self.nobuild
print "Running suite: " + suite + "..."
kats = [] if not os.path.exists("build"):
os.chdir(os.path.join(options.test_git,"kat_test",suite)) os.mkdir("build")
for files in os.listdir("."): # Firstly we need to build the latest version of finesse
if files.endswith(".kat"): if os.path.isdir("build") and not self.nobuild:
kats.append(files) print "deleting build dir..."
shutil.rmtree("build")
SUITE_OUTPUT_DIR = os.path.join(OUTPUTS_DIR,suite)
os.mkdir(SUITE_OUTPUT_DIR) print "Checking out finesse base..."
utils.git(["clone","git://gitmaster.atlas.aei.uni-hannover.de/finesse/base.git","build"])
for kat in kats:
print "Running kat: " + kat os.chdir("build")
basename = os.path.splitext(kat)[0] print "Checking out and building develop version of finesse..."
if run_fast and ('map ' in open(kat).read()): if sys.platform == "win32":
print "skipping " + kat runcmd(["bash","./finesse.sh","--checkout","develop"])
else: runcmd(["bash","./finesse.sh","--build"])
try: else:
start = time.time() EXE = ""
out,err = runcmd([FINESSE_EXE, "--noheader", kat]) runcmd(["./finesse.sh","--checkout","develop"])
finish = time.time()-start runcmd(["./finesse.sh","--build"])
run_times[suite][kat] = finish
shutil.move(basename + ".out", SUITE_OUTPUT_DIR) os.chdir(self.BASE_DIR)
except RunException as e:
print "Error running " + kat + ": " + e.err FINESSE_EXE = os.path.join(self.BASE_DIR,"build","kat" + EXE)
kat_run_exceptions[suite][kat] = e
# check if kat runs
for suite in suites: if not os.path.exists(FINESSE_EXE):
if len(kat_run_exceptions[suite].keys()) > 0: raise Exception("Kat file was not found")
print "Could not run the following kats:\n" + "\n".join(kat_run_exceptions.keys()) + " in " + suite
else: self.built = True
print "No errors whilst running" + suite
print "kat file found in " + FINESSE_EXE
# Now we have generated the output files compare them to the references OUTPUTS_DIR = os.path.join(self.BASE_DIR,"outputs")
for suite in suites:
print "Diffing suite: " + suite + "..." if os.path.isdir(OUTPUTS_DIR):
print "deleting outputs dir..."
outs = [] shutil.rmtree(OUTPUTS_DIR)
os.chdir(os.path.join(OUTPUTS_DIR,suite))
os.mkdir(OUTPUTS_DIR)
for files in os.listdir("."):
if files.endswith(".out"): os.environ["KATINI"] = os.path.join(self.TEST_DIR,"kat.ini")
outs.append(files)
# Clean up and pull latest test repository
REF_DIR = os.path.join(options.test_git,"kat_test",suite,"reference") print "Cleaning test repository..."
os.chdir(self.TEST_DIR)
if not os.path.exists(REF_DIR): utils.git(["clean","-xdf"])
raise Exception("Suite reference directory doesn't exist: " + REF_DIR) utils.git(["reset","--hard"])
for out in outs: print "Pulling latest test..."
#print "Diffing " + out utils.git(["pull"])
ref_file = os.path.join(REF_DIR,out)
# Define storage structures for generating report later
if not os.path.exists(ref_file): kat_run_exceptions = {}
raise DiffException("Reference file doesn't exist for " + out, out) output_differences = {}
ref_arr = np.loadtxt(ref_file) run_times = {}
out_arr = np.loadtxt(out)
self.total_kats = 0
if ref_arr.shape != out_arr.shape:
raise DiffException("Reference and output are different shapes", out) # create dictionary structures
# and count up total number of files to process
# for computing relative errors we need to make sure we for suite in self.suites:
# have no zeros in the data kat_run_exceptions[suite] = {}
ref_arr_c = np.where(ref_arr == 0, ref_arr, 1) output_differences[suite] = {}
ref_arr_c[ref_arr_c==0] = 1 run_times[suite] = {}
rel_diff = np.abs(out_arr-ref_arr)/np.abs(ref_arr_c) os.chdir(os.path.join(self.TEST_DIR,"kat_test",suite))
print suite
diff = np.any(rel_diff >= diff_rel_eps)
for files in os.listdir("."):
if diff: if files.endswith(".kat"):
# store the rows which are different self.total_kats += 1
ix = np.where(rel_diff >= diff_rel_eps)[0][0] print self.total_kats
output_differences[suite][out] = (ref_arr[ix], out_arr[ix], np.max(rel_diff))
for suite in self.suites:
print "Running suite: " + suite + "..."
os.chdir(BASE_DIR) kats = []
if not os.path.exists("reports"): os.chdir(os.path.join(self.TEST_DIR,"kat_test",suite))
os.mkdir("reports")
for files in os.listdir("."):
os.chdir("reports") if files.endswith(".kat"):
today = datetime.datetime.utcnow() kats.append(files)
reportname = today.strftime('%d%m%y')
print "Writing report to " + reportname SUITE_OUTPUT_DIR = os.path.join(OUTPUTS_DIR,suite)
os.mkdir(SUITE_OUTPUT_DIR)
f = open(reportname,'w')
f.write("Python Nightly Test\n") self.running_suite = suite
f.write(today.strftime('%A, %d. %B %Y %I:%M%p') + "\n")
for kat in kats:
self.running_kat = kat
# add kat file header
p = sub.Popen([FINESSE_EXE], stdout=sub.PIPE, stderr=sub.PIPE) print self.get_progress()
out, err = p.communicate() basename = os.path.splitext(kat)[0]
f.write(out)
if self.run_fast and ('map ' in open(kat).read()):
# Now time to generate a report... print "skipping " + kat
np.set_printoptions(precision=16) else:
try:
isError = False start = time.time()
out,err = runcmd([FINESSE_EXE, "--noheader", kat])
for suite in suites: finish = time.time()-start
f.write("\n\n" + str(len(output_differences[suite].keys())) + " differences in suite " + suite) run_times[suite][kat] = finish
for k in output_differences[suite].keys(): shutil.move(basename + ".out", SUITE_OUTPUT_DIR)
isError = True except RunException as e:
f.write(k + ":\n") print "Error running " + kat + ": " + e.err
f.write(" ref: " + str(output_differences[suite][k][0]) + "\n") kat_run_exceptions[suite][kat] = e
f.write(" out: " + str(output_differences[suite][k][1]) + "\n") finally:
f.write(" Max relative difference: " + str(output_differences[suite][k][2]) + "\n") self.done_kats += 1
f.write("\n\n" + str(len(output_differences[suite].keys())) + " errors in suite " + suite) for suite in self.suites:
for k in kat_run_exceptions[suite].keys(): if len(kat_run_exceptions[suite].keys()) > 0:
isError = True print "Could not run the following kats:\n" + "\n".join(kat_run_exceptions.keys()) + " in " + suite
f.write(k + ":\n") else:
f.write("err: " + kat_run_exceptions[suite][k].err + "\n") print "No errors whilst running" + suite
# Now we have generated the output files compare them to the references
f.close() for suite in self.suites:
print "Diffing suite: " + suite + "..."
if options.emails:
outs = []
if isError: os.chdir(os.path.join(OUTPUTS_DIR,suite))
subject = "Finesse test ERROR"
else: for files in os.listdir("."):
subject = "Finesse test OK" if files.endswith(".out"):
outs.append(files)
emails = options.emails
REF_DIR = os.path.join(self.TEST_DIR,"kat_test",suite,"reference")
args = ["mailx", "-s", subject, emails]
p = sub.Popen(args, stdout=sub.PIPE, stderr=sub.PIPE, stdin=sub.PIPE) if not os.path.exists(REF_DIR):
r = open(reportname,"r") raise Exception("Suite reference directory doesn't exist: " + REF_DIR)
out, err = p.communicate(r.read()) for out in outs:
else: #print "Diffing " + out
print "No emails specified" ref_file = os.path.join(REF_DIR,out)
if not os.path.exists(ref_file):
raise DiffException("Reference file doesn't exist for " + out, out)
ref_arr = np.loadtxt(ref_file)
out_arr = np.loadtxt(out)
if ref_arr.shape != out_arr.shape:
raise DiffException("Reference and output are different shapes", out)
# for computing relative errors we need to make sure we
# have no zeros in the data
ref_arr_c = np.where(ref_arr == 0, ref_arr, 1)
ref_arr_c[ref_arr_c==0] = 1
rel_diff = np.abs(out_arr-ref_arr)/np.abs(ref_arr_c)
diff = np.any(rel_diff >= self.diff_rel_eps)
if diff:
# store the rows which are different
ix = np.where(rel_diff >= self.diff_rel_eps)[0][0]
output_differences[suite][out] = (ref_arr[ix], out_arr[ix], np.max(rel_diff))
os.chdir(BASE_DIR)
if not os.path.exists("reports"):
os.mkdir("reports")
os.chdir("reports")
today = datetime.datetime.utcnow()
reportname = today.strftime('%d%m%y')
print "Writing report to " + reportname
f = open(reportname,'w')
f.write("Python Nightly Test\n")
f.write(today.strftime('%A, %d. %B %Y %I:%M%p') + "\n")
# add kat file header
p = sub.Popen([FINESSE_EXE], stdout=sub.PIPE, stderr=sub.PIPE)
out, err = p.communicate()
f.write(out)
# Now time to generate a report...
np.set_printoptions(precision=16)
isError = False
for suite in suites:
f.write("\n\n" + str(len(output_differences[suite].keys())) + " differences in suite " + suite)
for k in output_differences[suite].keys():
isError = True
f.write(k + ":\n")
f.write(" ref: " + str(output_differences[suite][k][0]) + "\n")
f.write(" out: " + str(output_differences[suite][k][1]) + "\n")
f.write(" Max relative difference: " + str(output_differences[suite][k][2]) + "\n")
f.write("\n\n" + str(len(output_differences[suite].keys())) + " errors in suite " + suite)
for k in kat_run_exceptions[suite].keys():
isError = True
f.write(k + ":\n")
f.write("err: " + kat_run_exceptions[suite][k].err + "\n")
f.close()
if self.emails:
if isError:
subject = "Finesse test ERROR"
else:
subject = "Finesse test OK"
emails = self.emails
args = ["mailx", "-s", subject, emails]
p = sub.Popen(args, stdout=sub.PIPE, stderr=sub.PIPE, stdin=sub.PIPE)
r = open(reportname,"r")
out, err = p.communicate(r.read())
else:
print "No emails specified"
def run(self):
try:
self.startFinesseTest()
finally:
finished_test = True
# once done check if any other tests need to be ran
#schedule_lock.acquire()
#if len(scheduled_tests) > 0:
# current_test = scheduled_tests.pop(0)
# current_test.start()
#else:
# current_test = None
#schedule_lock.release()
if __name__ == "__main__":
parser = OptionParser()
parser.add_option("-t","--test-dir",dest="test_dir",help="")
parser.add_option("-b","--base-dir",dest="base_dir",help="")
parser.add_option("-c","--test-commit",dest="test_commit",help="")
parser.add_option("-s","--suites",dest="suites",help="comma delimited list of each suite to run")
parser.add_option("-g","--git-bin",dest="git_bin", default="git",help="")
parser.add_option("-e","--emails",dest="emails", help="")
parser.add_option("-n","--no-build",default="False",dest="nobuild",action="store_true")
parser.add_option("-f","--fast",default="True",dest="fast",action="store_true")
options, args = parser.parse_args()
if options.test_dir is None:
print "--test-dir argument is missing"
exit()
if options.test_commit is None:
print "--test-commit argument is missing"
exit()
if options.base_dir is None:
options.base_dir = os.getcwd()
if options.suites is None:
suites = []
else:
suites = options.suites.split(",")
test = FinesseTestProcess(options.test_dir,
options.base_dir,
options.test_commit,
run_fast=options.fast,
suites=suites,
git_bin=options.git_bin,
emails=options.emails,
nobuild=options.nobuild)
test.run()
\ No newline at end of file
...@@ -18,9 +18,7 @@ def git(args): ...@@ -18,9 +18,7 @@ def git(args):
cmd = " ".join(args) cmd = " ".join(args)
else: else:
cmd = GIT_BIN + " " + args cmd = GIT_BIN + " " + args
print cmd
p = sub.Popen(cmd, stdout=sub.PIPE, stderr=sub.PIPE) p = sub.Popen(cmd, stdout=sub.PIPE, stderr=sub.PIPE)
out, err = p.communicate() out, err = p.communicate()
......
from flask import Flask
import os
global app
app = Flask(__name__, instance_path=os.getcwd())
\ No newline at end of file
...@@ -8,8 +8,10 @@ from flask import request ...@@ -8,8 +8,10 @@ from flask import request
from flask import render_template from flask import render_template
from datetime import datetime from datetime import datetime