Skip to content
GitLab
Commit 9ee26fa6 authored by Daniel Brown's avatar Daniel Brown
Browse files

moving some files around, added new finesse test routines and Flask web interface to package

parent 126cf72b
Hide whitespace changes
Inline Side-by-side
pykat/testing/test.py
View file @ 9ee26fa6
#!/bin/python
#!/bin/python
from threading import Thread, Lock
from time import sleep
from optparse import OptionParser
import os
import subprocess as sub
......@@ -11,11 +12,9 @@ import smtplib
import string
import time
import pickle
import datetime
options = None
diff_rel_eps = 1e-13
GIT_BIN = ""
from datetime import datetime
from pykat.testing import utils
import sys
class RunException(Exception):
def __init__(self, returncode, args, err, out):
......@@ -39,226 +38,364 @@ def runcmd(args):
return [out,err]
def git(args):
p = sub.Popen([GIT_BIN] + args, stdout=sub.PIPE, stderr=sub.PIPE)
out, err = p.communicate()
if p.returncode != 0:
print err
raise RunException(p.returncode, args, err, out)
return [out, err]
BASE_DIR = os.getcwd()
if not options.suites:
suites = ["physics","random"]
else:
suites = []
suites.extend(options.suites.split(","))
if not options.git:
GIT_BIN = "/usr/git/bin"
else:
GIT_BIN = options.git
if not options.fast:
run_fast = False
else:
run_fast = True
print "Running fast test"
# Firstly we need to build the latest version of finesse
if os.path.isdir("build") and not options.nobuild:
print "deleting build dir..."
shutil.rmtree("build")
print "Checking out finesse base..."
git(["clone","git://gitmaster.atlas.aei.uni-hannover.de/finesse/base.git","build"])
os.chdir("build")
print "Checking out develop version of finesse..."
runcmd(["./finesse.sh","--checkout","develop"])
print "Building finesse..."
runcmd(["./finesse.sh","--build"])
os.chdir(BASE_DIR)
# check if kat runs
if not os.path.exists("./build/kat"):
raise Exception("Kat file was not found")
FINESSE_EXE = os.path.join(os.getcwd(),"build","kat")
print "kat file found in " + FINESSE_EXE
OUTPUTS_DIR = os.path.join(BASE_DIR,"outputs")
if os.path.isdir(OUTPUTS_DIR):
print "deleting outputs dir..."
shutil.rmtree(OUTPUTS_DIR)
os.mkdir(OUTPUTS_DIR)
os.environ["KATINI"]=os.path.join(BASE_DIR,"build","kat.ini")
# Clean up and pull latest test repository
os.chdir(os.path.join(options.test_git))
print "Cleaning test repository...."
git(["clean","-xdf"])
git(["reset","--hard"])
print "Pulling latest test..."
git(["pull"])
# Define storage structures for generating report later
kat_run_exceptions = {}
output_differences = {}
run_times = {}
# create dictionary structures
for suite in suites:
kat_run_exceptions[suite] = {}
output_differences[suite] = {}
run_times[suite] = {}
for suite in suites:
print "Running suite: " + suite + "..."
kats = []
os.chdir(os.path.join(options.test_git,"kat_test",suite))
for files in os.listdir("."):
if files.endswith(".kat"):
kats.append(files)
SUITE_OUTPUT_DIR = os.path.join(OUTPUTS_DIR,suite)
os.mkdir(SUITE_OUTPUT_DIR)
for kat in kats:
print "Running kat: " + kat
basename = os.path.splitext(kat)[0]
if run_fast and ('map ' in open(kat).read()):
print "skipping " + kat
else:
try:
start = time.time()
out,err = runcmd([FINESSE_EXE, "--noheader", kat])
finish = time.time()-start
run_times[suite][kat] = finish
shutil.move(basename + ".out", SUITE_OUTPUT_DIR)
except RunException as e:
print "Error running " + kat + ": " + e.err
kat_run_exceptions[suite][kat] = e
for suite in suites:
if len(kat_run_exceptions[suite].keys()) > 0:
print "Could not run the following kats:\n" + "\n".join(kat_run_exceptions.keys()) + " in " + suite
else:
print "No errors whilst running" + suite
# Now we have generated the output files compare them to the references
for suite in suites:
print "Diffing suite: " + suite + "..."
outs = []
os.chdir(os.path.join(OUTPUTS_DIR,suite))
for files in os.listdir("."):
if files.endswith(".out"):
outs.append(files)
REF_DIR = os.path.join(options.test_git,"kat_test",suite,"reference")
if not os.path.exists(REF_DIR):
raise Exception("Suite reference directory doesn't exist: " + REF_DIR)
for out in outs:
#print "Diffing " + out
ref_file = os.path.join(REF_DIR,out)
if not os.path.exists(ref_file):
raise DiffException("Reference file doesn't exist for " + out, out)
ref_arr = np.loadtxt(ref_file)
out_arr = np.loadtxt(out)
if ref_arr.shape != out_arr.shape:
raise DiffException("Reference and output are different shapes", out)
# for computing relative errors we need to make sure we
# have no zeros in the data
ref_arr_c = np.where(ref_arr == 0, ref_arr, 1)
ref_arr_c[ref_arr_c==0] = 1
rel_diff = np.abs(out_arr-ref_arr)/np.abs(ref_arr_c)
diff = np.any(rel_diff >= diff_rel_eps)
if diff:
# store the rows which are different
ix = np.where(rel_diff >= diff_rel_eps)[0][0]
output_differences[suite][out] = (ref_arr[ix], out_arr[ix], np.max(rel_diff))
os.chdir(BASE_DIR)
if not os.path.exists("reports"):
os.mkdir("reports")
os.chdir("reports")
today = datetime.datetime.utcnow()
reportname = today.strftime('%d%m%y')
print "Writing report to " + reportname
f = open(reportname,'w')
f.write("Python Nightly Test\n")
f.write(today.strftime('%A, %d. %B %Y %I:%M%p') + "\n")
# add kat file header
p = sub.Popen([FINESSE_EXE], stdout=sub.PIPE, stderr=sub.PIPE)
out, err = p.communicate()
f.write(out)
# Now time to generate a report...
np.set_printoptions(precision=16)
isError = False
for suite in suites:
f.write("\n\n" + str(len(output_differences[suite].keys())) + " differences in suite " + suite)
for k in output_differences[suite].keys():
isError = True
f.write(k + ":\n")
f.write(" ref: " + str(output_differences[suite][k][0]) + "\n")
f.write(" out: " + str(output_differences[suite][k][1]) + "\n")
f.write(" Max relative difference: " + str(output_differences[suite][k][2]) + "\n")
f.write("\n\n" + str(len(output_differences[suite].keys())) + " errors in suite " + suite)
for k in kat_run_exceptions[suite].keys():
isError = True
f.write(k + ":\n")
f.write("err: " + kat_run_exceptions[suite][k].err + "\n")
f.close()
if options.emails:
if isError:
subject = "Finesse test ERROR"
else:
subject = "Finesse test OK"
emails = options.emails
args = ["mailx", "-s", subject, emails]
p = sub.Popen(args, stdout=sub.PIPE, stderr=sub.PIPE, stdin=sub.PIPE)
r = open(reportname,"r")
out, err = p.communicate(r.read())
else:
print "No emails specified"
class FinesseTestProcess(Thread):
queue_time = None
status = "Not started"
built = False
total_kats = 0
done_kats = 0
git_commit = ""
test_id = -1
finished_test = False
diff_rel_eps = 1e-13
running_kat = ""
running_suite = ""
def __init__(self, TEST_DIR, BASE_DIR, git_commit,
run_fast=False, suites=[], test_id="0",
git_bin="",emails="", nobuild=False,*args, **kqwargs):
Thread.__init__(self)
self.git_commit = git_commit
self.queue_time = datetime.now()
self.test_id = test_id
self.TEST_DIR = TEST_DIR
self.BASE_DIR = BASE_DIR
self.emails = ""
if type(nobuild) is str:
if nobuild.lower() == "true":
self.nobuild = True
elif nobuild.lower() == "false":
self.nobuild = False
else:
raise Exception("nobuild is not a boolean value")
elif type(nobuild) is bool:
self.nobuild = nobuild
else:
raise Exception("nobuild is not a boolean value")
if type(run_fast) is str:
if run_fast.lower() == "true":
self.run_fast = True
elif run_fast.lower() == "false":
self.run_fast = False
else:
raise Exception("run_fast is not a boolean value")
elif type(run_fast) is bool:
self.run_fast = run_fast
else:
raise Exception("nobuild is not a boolean value")
if not os.path.isdir(self.BASE_DIR):
raise Exception("BASE_DIR was not a valid directory")
if not os.path.isdir(self.TEST_DIR):
raise Exception("TEST_DIR was not a valid directory, should point to a clone of the FINESSE test repository")
if not suites:
self.suites = ["physics","random"]
else:
self.suites = []
self.suites.extend(suites)
self.GIT_BIN = git_bin
def percent_done(self):
return 100.0*float(self.done_files)/float(self.total_files)
def get_version(self):
return self.git_commit
def get_progress(self):
if self.built:
return '{0} out of {1} ({2} in {3})'.format(self.done_kats, self.total_kats,self.running_kat, self.running_suite)
else:
return 'Building FINESSE executable'
def startFinesseTest(self):
if sys.platform == "win32":
EXE = ".exe"
else:
EXE = ""
self.built = False
print type(self.nobuild), self.nobuild
if not os.path.exists("build"):
os.mkdir("build")
# Firstly we need to build the latest version of finesse
if os.path.isdir("build") and not self.nobuild:
print "deleting build dir..."
shutil.rmtree("build")
print "Checking out finesse base..."
utils.git(["clone","git://gitmaster.atlas.aei.uni-hannover.de/finesse/base.git","build"])
os.chdir("build")
print "Checking out and building develop version of finesse..."
if sys.platform == "win32":
runcmd(["bash","./finesse.sh","--checkout","develop"])
runcmd(["bash","./finesse.sh","--build"])
else:
EXE = ""
runcmd(["./finesse.sh","--checkout","develop"])
runcmd(["./finesse.sh","--build"])
os.chdir(self.BASE_DIR)
FINESSE_EXE = os.path.join(self.BASE_DIR,"build","kat" + EXE)
# check if kat runs
if not os.path.exists(FINESSE_EXE):
raise Exception("Kat file was not found")
self.built = True
print "kat file found in " + FINESSE_EXE
OUTPUTS_DIR = os.path.join(self.BASE_DIR,"outputs")
if os.path.isdir(OUTPUTS_DIR):
print "deleting outputs dir..."
shutil.rmtree(OUTPUTS_DIR)
os.mkdir(OUTPUTS_DIR)
os.environ["KATINI"] = os.path.join(self.TEST_DIR,"kat.ini")
# Clean up and pull latest test repository
print "Cleaning test repository..."
os.chdir(self.TEST_DIR)
utils.git(["clean","-xdf"])
utils.git(["reset","--hard"])
print "Pulling latest test..."
utils.git(["pull"])
# Define storage structures for generating report later
kat_run_exceptions = {}
output_differences = {}
run_times = {}
self.total_kats = 0
# create dictionary structures
# and count up total number of files to process
for suite in self.suites:
kat_run_exceptions[suite] = {}
output_differences[suite] = {}
run_times[suite] = {}
os.chdir(os.path.join(self.TEST_DIR,"kat_test",suite))
print suite
for files in os.listdir("."):
if files.endswith(".kat"):
self.total_kats += 1
print self.total_kats
for suite in self.suites:
print "Running suite: " + suite + "..."
kats = []
os.chdir(os.path.join(self.TEST_DIR,"kat_test",suite))
for files in os.listdir("."):
if files.endswith(".kat"):
kats.append(files)
SUITE_OUTPUT_DIR = os.path.join(OUTPUTS_DIR,suite)
os.mkdir(SUITE_OUTPUT_DIR)
self.running_suite = suite
for kat in kats:
self.running_kat = kat
print self.get_progress()
basename = os.path.splitext(kat)[0]
if self.run_fast and ('map ' in open(kat).read()):
print "skipping " + kat
else:
try:
start = time.time()
out,err = runcmd([FINESSE_EXE, "--noheader", kat])
finish = time.time()-start
run_times[suite][kat] = finish
shutil.move(basename + ".out", SUITE_OUTPUT_DIR)
except RunException as e:
print "Error running " + kat + ": " + e.err
kat_run_exceptions[suite][kat] = e
finally:
self.done_kats += 1
for suite in self.suites:
if len(kat_run_exceptions[suite].keys()) > 0:
print "Could not run the following kats:\n" + "\n".join(kat_run_exceptions.keys()) + " in " + suite
else:
print "No errors whilst running" + suite
# Now we have generated the output files compare them to the references
for suite in self.suites:
print "Diffing suite: " + suite + "..."
outs = []
os.chdir(os.path.join(OUTPUTS_DIR,suite))
for files in os.listdir("."):
if files.endswith(".out"):
outs.append(files)
REF_DIR = os.path.join(self.TEST_DIR,"kat_test",suite,"reference")
if not os.path.exists(REF_DIR):
raise Exception("Suite reference directory doesn't exist: " + REF_DIR)
for out in outs:
#print "Diffing " + out
ref_file = os.path.join(REF_DIR,out)
if not os.path.exists(ref_file):
raise DiffException("Reference file doesn't exist for " + out, out)
ref_arr = np.loadtxt(ref_file)
out_arr = np.loadtxt(out)
if ref_arr.shape != out_arr.shape:
raise DiffException("Reference and output are different shapes", out)
# for computing relative errors we need to make sure we
# have no zeros in the data
ref_arr_c = np.where(ref_arr == 0, ref_arr, 1)
ref_arr_c[ref_arr_c==0] = 1
rel_diff = np.abs(out_arr-ref_arr)/np.abs(ref_arr_c)
diff = np.any(rel_diff >= self.diff_rel_eps)
if diff:
# store the rows which are different
ix = np.where(rel_diff >= self.diff_rel_eps)[0][0]
output_differences[suite][out] = (ref_arr[ix], out_arr[ix], np.max(rel_diff))
os.chdir(BASE_DIR)
if not os.path.exists("reports"):
os.mkdir("reports")
os.chdir("reports")
today = datetime.datetime.utcnow()
reportname = today.strftime('%d%m%y')
print "Writing report to " + reportname
f = open(reportname,'w')
f.write("Python Nightly Test\n")
f.write(today.strftime('%A, %d. %B %Y %I:%M%p') + "\n")
# add kat file header
p = sub.Popen([FINESSE_EXE], stdout=sub.PIPE, stderr=sub.PIPE)
out, err = p.communicate()
f.write(out)
# Now time to generate a report...
np.set_printoptions(precision=16)
isError = False
for suite in suites:
f.write("\n\n" + str(len(output_differences[suite].keys())) + " differences in suite " + suite)
for k in output_differences[suite].keys():
isError = True
f.write(k + ":\n")
f.write(" ref: " + str(output_differences[suite][k][0]) + "\n")
f.write(" out: " + str(output_differences[suite][k][1]) + "\n")
f.write(" Max relative difference: " + str(output_differences[suite][k][2]) + "\n")
f.write("\n\n" + str(len(output_differences[suite].keys())) + " errors in suite " + suite)
for k in kat_run_exceptions[suite].keys():
isError = True
f.write(k + ":\n")
f.write("err: " + kat_run_exceptions[suite][k].err + "\n")
f.close()
if self.emails:
if isError:
subject = "Finesse test ERROR"
else:
subject = "Finesse test OK"
emails = self.emails
args = ["mailx", "-s", subject, emails]
p = sub.Popen(args, stdout=sub.PIPE, stderr=sub.PIPE, stdin=sub.PIPE)
r = open(reportname,"r")
out, err = p.communicate(r.read())
else:
print "No emails specified"
def run(self):
try:
self.startFinesseTest()
finally:
finished_test = True
# once done check if any other tests need to be ran
#schedule_lock.acquire()
#if len(scheduled_tests) > 0:
# current_test = scheduled_tests.pop(0)
# current_test.start()
#else:
# current_test = None
#schedule_lock.release()
if __name__ == "__main__":
parser = OptionParser()
parser.add_option("-t","--test-dir",dest="test_dir",help="")
parser.add_option("-b","--base-dir",dest="base_dir",help="")
parser.add_option("-c","--test-commit",dest="test_commit",help="")
parser.add_option("-s","--suites",dest="suites",help="comma delimited list of each suite to run")
parser.add_option("-g","--git-bin",dest="git_bin", default="git",help="")
parser.add_option("-e","--emails",dest="emails", help="")
parser.add_option("-n","--no-build",default="False",dest="nobuild",action="store_true")
parser.add_option("-f","--fast",default="True",dest="fast",action="store_true")
options, args = parser.parse_args()
if options.test_dir is None:
print "--test-dir argument is missing"
exit()
if options.test_commit is None:
print "--test-commit argument is missing"
exit()
if options.base_dir is None:
options.base_dir = os.getcwd()
if options.suites is None:
suites = []
else:
suites = options.suites.split(",")
test = FinesseTestProcess(options.test_dir,
options.base_dir,
options.test_commit,
run_fast=options.fast,
suites=suites,