From b978043bedc4856dc4a81ee0ef3332736e2acc67 Mon Sep 17 00:00:00 2001
From: Daniel Brown <ddb@star.sr.bham.ac.uk>
Date: Mon, 5 Aug 2013 17:33:50 +0100
Subject: [PATCH] increasing round off range as many test files only store 12
instead of the full 15 precision range. more fixes to the nightly test
relative error computation and displaying previous test results
---
pykat/testing/test.py | 27 +++++++++-----
.../web/templates/finesse_test_view.html | 2 +-
pykat/testing/web/web_interface.py | 37 +++++++++++++------
3 files changed, 44 insertions(+), 22 deletions(-)
diff --git a/pykat/testing/test.py b/pykat/testing/test.py
index 25483a7..fbd4894 100644
--- a/pykat/testing/test.py
+++ b/pykat/testing/test.py
@@ -31,7 +31,7 @@ class FinesseTestProcess(Thread):
git_commit = ""
test_id = -1
finished_test = False
- diff_rel_eps = 1e-13
+ diff_rel_eps = 1e-12
running_kat = ""
running_suite = ""
cancelling = False
@@ -47,7 +47,7 @@ class FinesseTestProcess(Thread):
def __init__(self, TEST_DIR, BASE_DIR, test_commit,
run_fast=False, kats=[], test_id="0",
- git_bin="",emails="", nobuild=True,*args, **kqwargs):
+ git_bin="",emails="", nobuild=False,*args, **kqwargs):
Thread.__init__(self)
self.git_commit = test_commit
@@ -179,7 +179,7 @@ class FinesseTestProcess(Thread):
os.mkdir(OUTPUTS_DIR)
- os.environ["KATINI"] = os.path.join(self.TEST_DIR,"kat_test","kat.ini")
+ os.environ["KATINI"] = os.path.join(BUILD_PATH,"kat.ini")
self.cancelCheck()
# Clean up and pull latest test repository
@@ -230,7 +230,10 @@ class FinesseTestProcess(Thread):
else:
try:
start = time.time()
- out,err = utils.runcmd([FINESSE_EXE, "--noheader", kat],cwd=SUITE_PATH)
+
+ print "running", kat, FINESSE_EXE
+
+ out,err = utils.runcmd([FINESSE_EXE, "--noheader", kat], cwd=SUITE_PATH)
OUT_FILE = os.path.join(SUITE_PATH,basename + ".out")
shutil.move(OUT_FILE, SUITE_OUTPUT_DIR)
@@ -285,21 +288,21 @@ class FinesseTestProcess(Thread):
if not os.path.exists(ref_file):
raise DiffException("Reference file doesn't exist for " + out, out)
- ref_arr = np.loadtxt(ref_file)
- out_arr = np.loadtxt(out_file)
-
+ ref_arr = np.loadtxt(ref_file, dtype=np.float64)
+ out_arr = np.loadtxt(out_file, dtype=np.float64)
+
if ref_arr.shape != out_arr.shape:
raise DiffException("Reference and output are different shapes", out)
# for computing relative errors we need to make sure we
# have no zeros in the data
- ref_arr_c = np.where(ref_arr == 0, ref_arr, 1)
+ ref_arr_c = np.where(ref_arr == 0, 1, ref_arr)
ref_arr_c[ref_arr_c==0] = 1
rel_diff = np.abs(out_arr-ref_arr)/np.abs(ref_arr_c)
-
- diff = np.any(rel_diff >= self.diff_rel_eps)
+ diff = np.any(rel_diff >= self.diff_rel_eps)
+
if diff:
self.diffFound = True
@@ -384,6 +387,10 @@ class FinesseTestProcess(Thread):
self.errorOccurred = dict(value=str(exc_value), traceback=str(traceback.format_exc(5)))
+ if exc_type is utils.RunException:
+ self.errorOccurred["stdout"] = ex.out
+ self.errorOccurred["stderr"] = ex.err
+
print "*** Exception for test_id = " + str(self.test_id)
traceback.print_exception(exc_type, exc_value, exc_traceback,
limit=5, file=sys.stdout)
diff --git a/pykat/testing/web/templates/finesse_test_view.html b/pykat/testing/web/templates/finesse_test_view.html
index 240deda..23e7d38 100644
--- a/pykat/testing/web/templates/finesse_test_view.html
+++ b/pykat/testing/web/templates/finesse_test_view.html
@@ -23,7 +23,7 @@
<li>{{ suite }}</li>
<ul>
{% for item in kats[suite] %}
- <li><span style="width: 200px">{{item[0]}}</span> - Max rel diff = {{item[1]}}</li>
+ <li><span style="display:inline-block;width: 300px">{{item[0]}}</span>Max rel diff = {{item[1]}}</li>
{% if item[2][0] != "" %}
<ul>
<li>stdout: <pre>{{item[2][0]}}</pre></li>
diff --git a/pykat/testing/web/web_interface.py b/pykat/testing/web/web_interface.py
index 27ae5aa..8e78996 100644
--- a/pykat/testing/web/web_interface.py
+++ b/pykat/testing/web/web_interface.py
@@ -152,14 +152,17 @@ class FinesseProcessWatcher(Thread):
try:
doc = db.get('kattest', key, with_doc=True)["doc"]
+
+ doc["max_diff"].append(max_diff)
doc["test_id"].append(self.process_to_watch.test_id)
doc["commit"].append(self.process_to_watch.get_version())
doc["timing"].append(self.process_to_watch.run_times[suite][kat])
db.update(doc)
except RecordNotFound:
- doc = dict(t="kattest",test_id=[],commit=[],timing=[],suite=suite,kat=kat)
-
+ doc = dict(t="kattest",max_diff=[],test_id=[],commit=[],timing=[],suite=suite,kat=kat)
+
+ doc["max_diff"].append(max_diff)
doc["test_id"].append(self.process_to_watch.test_id)
doc["commit"].append(self.process_to_watch.get_version())
doc["timing"].append(self.process_to_watch.run_times[suite][kat])
@@ -468,14 +471,18 @@ def finesse_get_prev_tests(count):
else:
status = "OK"
- dt = datetime.strptime(endTime,"%Y-%m-%d %H:%M:%S.%f")-datetime.strptime(startTime,"%Y-%m-%d %H:%M:%S.%f")
+ if len(endTime) > 0 and len(startTime) > 0:
+ difftime = datetime.strptime(endTime,"%Y-%m-%d %H:%M:%S.%f")-datetime.strptime(startTime,"%Y-%m-%d %H:%M:%S.%f")
+ dt = float(difftime.seconds)
+ else:
+ dt = float(0)
obj = dict(test_id=i['test_id'],
git_commit=i['git_commit'],
status=status,
startTime=startTime,
endTime=endTime,
- duration=float(dt.seconds))
+ duration=dt)
rtn.insert(0,obj)
@@ -539,17 +546,25 @@ def finesse_view(view_test_id):
doc = doc["doc"]
kats = {}
- for run in doc["kats_run"]:
- suite = run["suite"]
- if not suite in kats:
- kats[suite] = []
-
- kats[suite].append((run["kat"], run["max_diff"], run["runexception"]))
-
+ if "kats_run" in doc:
+ for run in doc["kats_run"]:
+ suite = run["suite"]
+ if not suite in kats:
+ kats[suite] = []
+
+ kats[suite].append((run["kat"], run["max_diff"], run["runexception"]))
+ else:
+ kats = {}
if "error" in doc and doc["error"] is not None:
traceback = doc["error"]["traceback"]
message = doc["error"]["value"]
+
+ if "stdout" in doc["error"]:
+ message += "\n\nstdout: " + doc["error"]["stdout"]
+
+ if "stderr" in doc["error"]:
+ message += "\n\nstderr: " + doc["error"]["stderr"]
else:
traceback = ""
message = "No test exceptions thrown"
--
GitLab