test.py 19.5 KB
Newer Older
1
#!/bin/python
Daniel Brown's avatar
Daniel Brown committed
2
import gzip
3
4
from threading import Thread, Lock
from time import sleep
5
6
from optparse import OptionParser
import os
7
8
import multiprocessing
from multiprocessing import Pool, Queue
9
10
11
12
13
14
15
16
17
import subprocess as sub
import numpy as np
import difflib
from StringIO import StringIO
import shutil
import smtplib
import string
import time
import pickle
18
19
from datetime import datetime
from pykat.testing import utils
20
import sys, traceback
Daniel Brown's avatar
Daniel Brown committed
21
import stat
22
23
24
25
26
27
28
29
30
31
import math

def initProcess(dkats):
    #print "init!!!", dkats
    global done_kats
    done_kats = dkats

def run_kat_file(item):
    kat = item["kat"]
    suite = item["suite"]
32
    runtime = 0.0;
Daniel Brown's avatar
Daniel Brown committed
33
34
35
    try:
        #print os.getpid(),"getting kat...",item["kat"]
        global done_kats
36
        
Daniel Brown's avatar
Daniel Brown committed
37
38
39
40
        FINESSE_EXE = item["FINESSE_EXE"]
        SUITE_PATH = item["SUITE_PATH"]
        SUITE_OUTPUT_DIR  = item["SUITE_OUTPUT_DIR"]
        basename = os.path.splitext(kat)[0]
Daniel Brown's avatar
Daniel Brown committed
41
        exp = None
42
        
Daniel Brown's avatar
Daniel Brown committed
43
44
45
        if item["run_fast"] and ('map ' in open(kat).read()):
            print "skipping " + kat			
        else:
46
            
Daniel Brown's avatar
Daniel Brown committed
47
48
49
            #try:
            start = time.time()
            
50
            out,err = utils.runcmd([FINESSE_EXE, "--noheader", kat], cwd=SUITE_PATH)
51
            runtime = time.time()-start
Daniel Brown's avatar
Daniel Brown committed
52
53
54
55
56
57
58
59
60
            
            OUT_FILE = os.path.join(SUITE_PATH,basename + ".out")
            LOG_FILE = os.path.join(SUITE_PATH,basename + ".log")
            
            f_in = open(LOG_FILE, 'rb')
            f_out = gzip.open(LOG_FILE + ".gz", 'wb')
            f_out.writelines(f_in)
            f_out.close()
            f_in.close()
61
            
Daniel Brown's avatar
Daniel Brown committed
62
63
            shutil.move(OUT_FILE, SUITE_OUTPUT_DIR)
            shutil.move(LOG_FILE + ".gz", SUITE_OUTPUT_DIR)
Daniel Brown's avatar
Daniel Brown committed
64
            
Daniel Brown's avatar
Daniel Brown committed
65
66
67
68
69
70
            #except utils.RunException as e:
            #
             #   print "STDERR: " + e.out
              #  print "STDOUT: " + e.err
               # 
               # print "Error running " + kat
Daniel Brown's avatar
Daniel Brown committed
71
                
Daniel Brown's avatar
Daniel Brown committed
72
73
              #  exp = e
            #finally:
Daniel Brown's avatar
Daniel Brown committed
74
            done_kats.value += 1
Daniel Brown's avatar
Daniel Brown committed
75
            #    return [time.time()-start, suite, kat, exp]
Daniel Brown's avatar
Daniel Brown committed
76
                    
77
78
79
    except Exception as e:
        print "main error in kat call"
        exp = None
Daniel Brown's avatar
Daniel Brown committed
80
    finally:
81
82
83
84
85
        # Annoyingly it seems that exceptions can't be returned
	# from a process that has been mapped because exceptions
	# or some at least, can't be pickled, so they will appear as 
	# nan in the tests
        return [runtime, suite, kat, None] #[runtime,suite,kat,e]
Daniel Brown's avatar
Daniel Brown committed
86
        
87
88
89
90
91
92

class DiffException(Exception):
	def __init__(self, msg, outfile):
		self.msg = msg
		self.outfile = outfile

93
class FinesseTestProcess(Thread):
94
        
95
    def __init__(self, TEST_DIR, BASE_DIR, test_commit, 
96
                 run_fast=False, kats={}, test_id="0",
Daniel Brown's avatar
Daniel Brown committed
97
                 git_bin="",emails="", nobuild=False, pool_size=int(multiprocessing.cpu_count()*3.5/4.0),*args, **kqwargs):
98
                 
99
         
100
101
102
103
        self.queue_time = None
        self.status = ""
        self.built = False
        self.total_kats = 0
104
        self.done_kats = multiprocessing.Value('i', 0)
105
106
107
108
109
110
111
112
113
114
        self.git_commit = ""
        self.test_id = -1
        self.finished_test = False
        self.diff_rel_eps = 1e-12
        self.running_kat = ""
        self.running_suite = ""
        self.cancelling = False
        self.errorOccurred = None
        self.diffFound = False
        self.diffing = False
115
        
116
117
118
        if pool_size < 1:
            self.pool_size = 1
        else:
Daniel Brown's avatar
Daniel Brown committed
119
            self.pool_size = pool_size
120
        
121
        Thread.__init__(self)
122
123
124
125
126
        self.git_commit = test_commit
        
        if test_commit is None:
            raise Exception("A git commit ID must be provided for the test")
        
127
128
129
130
        self.kat_run_exceptions = {}
        self.output_differences = {}
        self.run_times = {}
        
131
132
133
134
        self.queue_time = datetime.now()
        self.test_id = test_id
        self.TEST_DIR = TEST_DIR
        self.BASE_DIR = BASE_DIR
Daniel Brown's avatar
Daniel Brown committed
135
        
136
137
138
        self.emails = ""
        
        if type(nobuild) is str:
139
        
140
141
142
143
144
145
            if nobuild.lower() == "true":
                self.nobuild = True
            elif nobuild.lower() == "false":
                self.nobuild = False
            else:
                raise Exception("nobuild is not a boolean value")
146
                
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
        elif type(nobuild) is bool:
            self.nobuild = nobuild
        else:
            raise Exception("nobuild is not a boolean value")
        
        if type(run_fast) is str:
            if run_fast.lower() == "true":
                self.run_fast = True
            elif run_fast.lower() == "false":
                self.run_fast = False
            else:
                raise Exception("run_fast is not a boolean value")
                
        elif type(run_fast) is bool:
            self.run_fast = run_fast
        else:
            raise Exception("nobuild is not a boolean value")
            
        if not os.path.isdir(self.TEST_DIR):
            raise Exception("TEST_DIR was not a valid directory, should point to a clone of the FINESSE test repository")
            
168
        self.kats_to_run = kats
169
        self.GIT_BIN = git_bin
170
171
172
173
174
    
    def cancelCheck(self):
        if self.cancelling:
            raise SystemExit()
    
175
    def percent_done(self):
176
177
178
        if self.total_kats == 0:
            return 0.0
        else:
179
            return 100.0*float(self.done_kats.value)/float(self.total_kats)
180
181
182
183
184
        
    def get_version(self):
        return self.git_commit
        
    def get_progress(self):
185
        if self.diffing:
186
            return 'Diffing {0} out of {1} ({2} in {3})'.format(self.done_kats.value, self.total_kats/2, self.running_kat, self.running_suite)
187
        if self.built:
188
            return 'Running {0} out of {1} ({2} in {3})'.format(self.done_kats.value, self.total_kats/2, self.running_kat, self.running_suite)
189
190
191
192
        else:
            return 'Building FINESSE executable'
            
    def startFinesseTest(self):
193
194
        self.done_kats.value = 0
        
195
196
197
198
        if sys.platform == "win32":
            EXE = ".exe"
        else:
            EXE = ""
199
200
201
        
        print "Using", self.pool_size, "processes..."
        
202
203
        self.built = False

204
        BUILD_PATH = os.path.join(self.BASE_DIR, "build")
205
        
206
        # Firstly we need to build the latest version of finesse
207
        if not self.nobuild:
208
209
210
211
212
213
214
            
            if os.path.exists(self.BASE_DIR):
                print "Deleting previous base_dir " + self.BASE_DIR
                shutil.rmtree(self.BASE_DIR)
            
            os.mkdir(self.BASE_DIR)
            
215
216
217
            print "deleting build dir..." + BUILD_PATH
            if os.path.exists(BUILD_PATH):
                shutil.rmtree(BUILD_PATH)
218
219

            print "Checking out finesse base..."
220
            utils.git(["clone","git://gitmaster.atlas.aei.uni-hannover.de/finesse/finesse.git", BUILD_PATH])
221

222
223
224
            print "Checking out and building develop version of finesse " + self.git_commit
            
            SRC_PATH = os.path.join(BUILD_PATH,"src")
225
            
226
            if sys.platform == "win32":                
227
                utils.git(["checkout",self.git_commit],cwd=SRC_PATH)
228
229
                self.cancelCheck()
                
230
                utils.runcmd(["bash","./finesse.sh","--build"],cwd=BUILD_PATH)
231
                self.cancelCheck()
232
            else:                
233
                utils.git(["checkout",self.git_commit],cwd=SRC_PATH)
234
235
                self.cancelCheck()
                
236
                utils.runcmd(["./finesse.sh","--build"],cwd=BUILD_PATH)
237
                self.cancelCheck()
238
                          
239
        FINESSE_EXE = os.path.join(self.BASE_DIR,"build","kat" + EXE)
Daniel Brown's avatar
Daniel Brown committed
240
        print "NOBUILD",self.nobuild
241
242
        # check if kat runs
        if not os.path.exists(FINESSE_EXE):
243
            raise Exception("Kat file was not found in " + FINESSE_EXE)
244
        
Daniel Brown's avatar
Daniel Brown committed
245
        
Daniel Brown's avatar
Daniel Brown committed
246
        if not os.access(FINESSE_EXE, os.X_OK):
Daniel Brown's avatar
Daniel Brown committed
247
248
            if sys.platform != "win32":
                print "Trying to chmod " + FINESSE_EXE
249
                os.chmod(FINESSE_EXE, stat.S_IRWXU)
250
        
251
        out = None
Daniel Brown's avatar
Daniel Brown committed
252
        
253
        # check version numbers match upkat
254
255
256
257
258
259
260
261
262
        out = utils.runcmd([FINESSE_EXE,"-v"])
            
        # I am sure there is a regex expression that could make this
        # easier but I am being lazy
        out = out[0]
        out = out.split("\n")
        out = out[0].split("(")[1].split(")")
        out = out[0].split("-")[-1]
        shortid = out.lstrip("g")
Daniel Brown's avatar
Daniel Brown committed
263
264
        
        if shortid != self.git_commit[0:len(shortid)]:
265
            raise Exception("Version of kat {0} did not match the version that it was requested to build {1}.".format(shortid, self.git_commit[0:len(shortid)]))
Daniel Brown's avatar
Daniel Brown committed
266
        
267
268
269
270
271
272
273
274
275
276
277
278
        self.built = True
        
        print "kat file found in " + FINESSE_EXE
        
        OUTPUTS_DIR = os.path.join(self.BASE_DIR,"outputs")
        
        if os.path.isdir(OUTPUTS_DIR):
            print "deleting outputs dir..."
            shutil.rmtree(OUTPUTS_DIR)
            
        os.mkdir(OUTPUTS_DIR)
        
279
        os.environ["KATINI"] = os.path.join(BUILD_PATH,"kat.ini")
280
        
281
        self.cancelCheck()
282
283
        # Clean up and pull latest test repository
        print "Cleaning test repository..."
284
        utils.git(["clean","-xdf"], cwd=self.TEST_DIR)
285
        self.cancelCheck()
286
        utils.git(["reset","--hard"], cwd=self.TEST_DIR)
287
        self.cancelCheck()
288
        print "Pulling latest test..."
289
        utils.git(["pull"],cwd=self.TEST_DIR)
290
291
        self.cancelCheck()
    
292
293
294
295
        self.total_kats = 0
        
        # create dictionary structures
        # and count up total number of files to process
296
        for suite in self.kats_to_run.keys():
297
298
            print "RUNNING SUITES", suite
            
299
300
301
            self.kat_run_exceptions[suite] = {}
            self.output_differences[suite] = {}
            self.run_times[suite] = {}
302
            
303
            self.total_kats += len(self.kats_to_run[suite])
304
        
305
306
307
        # multiply as we include the diffining in the percentage
        # done
        self.total_kats *= 2
308
        runs = []
309
        
310
        for suite in self.kats_to_run.keys():
311
            self.cancelCheck()
312
            print "Queuing up suite: " + suite + "..."
313
            kats = self.kats_to_run[suite]
314
            SUITE_PATH = os.path.join(self.TEST_DIR,"kat_test",suite)
315
316
317
318
319

            SUITE_OUTPUT_DIR = os.path.join(OUTPUTS_DIR,suite)
            os.mkdir(SUITE_OUTPUT_DIR)
            
            for kat in kats:
320
321
322
323
                runs.append({'SUITE_OUTPUT_DIR':SUITE_OUTPUT_DIR,'suite':suite, 'run_fast':self.run_fast, 'kat':kat, 'FINESSE_EXE':FINESSE_EXE, 'SUITE_PATH':SUITE_PATH})
        
        self.pool = Pool(initializer=initProcess,initargs=(self.done_kats,) ,processes = self.pool_size)    
        results = self.pool.imap_unordered(run_kat_file, runs, 1)
324
        print "Finsihed running pool of kats"
325
326
327
328
329
330
331
332
333
334
        self.pool.close()
        
        for result in results:
            
            if result[3] is not None:
                self.kat_run_exceptions[result[1]][result[2]] = result[3]
                self.diffFound = True
            
            self.run_times[result[1]][result[2]] = result[0]
        
335
336
        self.cancelCheck()
        
337
        for suite in self.kats_to_run.keys():
338
339
            if len(self.kat_run_exceptions[suite].keys()) > 0:
                print "Could not run the following kats:\n" + "\n".join(self.kat_run_exceptions[suite].keys()) + " in " + suite
340
            else:
341
                print "No exceptions whilst running: " + suite
342

343
        self.diffing = True
344
345
        
        # Now we have generated the output files compare them to the references
346
        for suite in self.kats_to_run.keys():
347
            self.cancelCheck()
348
349
350
            print "Diffing suite: " + suite + "..."

            outs = []
351
352
            SUITE_PATH = os.path.join(OUTPUTS_DIR, suite)
            
353
            for files in os.listdir(SUITE_PATH):
354
355
356
357
                if files.endswith(".out"):
                    outs.append(files)

            REF_DIR = os.path.join(self.TEST_DIR,"kat_test",suite,"reference")
358
            
359
360
            if not os.path.exists(REF_DIR):
                raise Exception("Suite reference directory doesn't exist: " + REF_DIR)
361
                
362
            for out in outs:
363
                self.cancelCheck()
364
                self.running_kat = out
365
                
366
367
368
                try:        
                    ref_file = os.path.join(REF_DIR,out)
                    out_file = os.path.join(SUITE_PATH,out)
369
                
370
371
                    if not os.path.exists(ref_file):
                        raise DiffException("Reference file doesn't exist for " + out, out)
372
                    
373
374
                    ref_arr = np.loadtxt(ref_file, dtype=np.float64)
                    out_arr = np.loadtxt(out_file, dtype=np.float64)
375
                
376
377
                    if ref_arr.shape != out_arr.shape:
                        raise DiffException("Reference and output are different shapes", out)
378

379
380
381
                    # for computing relative errors we need to make sure we
                    # have no zeros in the data
                    nzix = (ref_arr != 0)
382
                
383
384
385
                    rel_diff = np.abs(out_arr-ref_arr)
                    # only compute rel diff for non zero values
                    rel_diff[nzix] = np.divide(rel_diff[nzix], np.abs(ref_arr[nzix]))
386
                
387
                    diff = np.any(rel_diff >= self.diff_rel_eps)
388
                                
389
390
                    if diff:
                        self.diffFound = True
391
                    
392
393
                        # store the rows which are different
                        ix = np.where(rel_diff >= self.diff_rel_eps)[0][0]
394
                
395
396
397
398
399
400
                        self.output_differences[suite][out] = (True,
                                                               ref_arr[ix],
                                                               out_arr[ix],
                                                               np.max(rel_diff))
                    else:
                        max = np.max(rel_diff) 
401
                    
402
                        self.output_differences[suite][out] = (False, max)
Daniel Brown's avatar
Daniel Brown committed
403
404
405
                
                
                        
406
407
408
409
410
411
                    # compress the data
                    f_in = open(out_file, 'rb')
                    f_out = gzip.open(out_file + ".gz", 'wb')
                    f_out.writelines(f_in)
                    f_out.close()
                    f_in.close()
Daniel Brown's avatar
Daniel Brown committed
412
                
413
414
415
416
417
418
                    print "removing out file ", out_file
                    os.remove(out_file)
                    
                except DiffException as ex:
                    print(str(ex), "output =", out)
                    
419
                self.done_kats.value += 1
420
421
422
        
        print "Finished diffing..."
        
423
        REPORT_PATH = os.path.join(self.BASE_DIR,"reports")
424
        
425
426
        if not os.path.exists(REPORT_PATH):
            os.mkdir(REPORT_PATH)
427

428
        today = datetime.utcnow()
429
        reportname = os.path.join(REPORT_PATH,"report.log") #today.strftime('%d%m%y')
430
        print "Writing report to " + reportname
431
        
432
433
434
435
436
437
438
439
440
441
442
443
444
445
        f = open(reportname,'w')
        f.write("Python Nightly Test\n")
        f.write(today.strftime('%A, %d. %B %Y %I:%M%p') + "\n")

        # add kat file header
        p = sub.Popen([FINESSE_EXE], stdout=sub.PIPE, stderr=sub.PIPE)
        out, err = p.communicate()
        f.write(out)
        
        # Now time to generate a report...
        np.set_printoptions(precision=16)
        
        isError = False

446
        for suite in self.kats_to_run.keys():
447
448
            f.write("\n\n" + str(len(self.output_differences[suite].keys())) + " differences in suite " + suite)
            for k in self.output_differences[suite].keys():
449
450
                isError = True
                f.write(k + ":\n")
451
452
453
454
455
456
457
458
459
                if self.output_differences[suite][k][0]:
                    f.write("     Differences larger than " + str(self.diff_rel_eps) + "\n")
                    f.write("     ref: " + str(self.output_differences[suite][k][1]) + "\n")
                    f.write("     out: " + str(self.output_differences[suite][k][2]) + "\n")
                    f.write("     Max relative difference: " + str(self.output_differences[suite][k][3]) + "\n")
                else:
                    f.write("     Differences smaller than " + str(self.diff_rel_eps) + "\n")
                    f.write("     Max relative difference: " + str(self.output_differences[suite][k][1]) + "\n")
                    
460
461
            f.write("\n\n" + str(len(self.output_differences[suite].keys())) + " errors in suite " + suite)
            for k in self.kat_run_exceptions[suite].keys():
462
463
                isError = True
                f.write(k + ":\n")
464
                f.write("err: " + self.kat_run_exceptions[suite][k].err + "\n")
465
466
467

        f.close()
        
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
        #if self.emails:
        #    
        #    if isError:
        #        subject = "Finesse test ERROR"
        #    else:
        #        subject = "Finesse test OK"
        #    emails = self.emails

        #    args = ["mailx", "-s", subject, emails]
        #    p = sub.Popen(args, stdout=sub.PIPE, stderr=sub.PIPE, stdin=sub.PIPE)
        #    r = open(reportname,"r")
        #    out, err = p.communicate(r.read())
        #else:
        #    print "No emails specified"

            
484
485
486
487
488
            
    def run(self):
        
        try:
            self.startFinesseTest()
489
        except Exception as ex:
490
            
491
            exc_type, exc_value, exc_traceback = sys.exc_info()
492
493
            
            self.errorOccurred = dict(value=str(exc_value), traceback=str(traceback.format_exc(5)))
494
            
495
496
497
498
            if exc_type is utils.RunException:
                self.errorOccurred["stdout"] = ex.out
                self.errorOccurred["stderr"] = ex.err
                
499
500
501
            print "*** Exception for test_id = " + str(self.test_id)
            traceback.print_exception(exc_type, exc_value, exc_traceback,
                                      limit=5, file=sys.stdout)
502
        finally:
503
            self.finished_test = True
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
        
        

if __name__ == "__main__":
    
    parser = OptionParser()
    
    parser.add_option("-t","--test-dir",dest="test_dir",help="")
    parser.add_option("-b","--base-dir",dest="base_dir",help="")
    parser.add_option("-c","--test-commit",dest="test_commit",help="")
    parser.add_option("-s","--suites",dest="suites",help="comma delimited list of each suite to run")
    parser.add_option("-g","--git-bin",dest="git_bin", default="git",help="")
    parser.add_option("-e","--emails",dest="emails", help="")
    parser.add_option("-n","--no-build",default="False",dest="nobuild",action="store_true")
    parser.add_option("-f","--fast",default="True",dest="fast",action="store_true")

    options, args = parser.parse_args()

    if options.test_dir is None:
        print "--test-dir argument is missing"
        exit()
        
    if options.test_commit is None:
        print "--test-commit argument is missing"
        exit()
    
    if options.base_dir is None:
        options.base_dir = os.getcwd()
    
    if options.suites is None:
        suites = []
    else:
        suites = options.suites.split(",")
        
    test = FinesseTestProcess(options.test_dir,
                              options.base_dir,
                              options.test_commit,
                              run_fast=options.fast,
                              suites=suites,
                              git_bin=options.git_bin,
                              emails=options.emails,
                              nobuild=options.nobuild)
546
547
    test.run()
    
548