test.py 19.1 KB
Newer Older
1
#!/bin/python
Daniel Brown's avatar
Daniel Brown committed
2
import gzip
3
4
from threading import Thread, Lock
from time import sleep
5
6
from optparse import OptionParser
import os
7
8
import multiprocessing
from multiprocessing import Pool, Queue
9
10
11
12
13
14
15
16
17
import subprocess as sub
import numpy as np
import difflib
from StringIO import StringIO
import shutil
import smtplib
import string
import time
import pickle
18
19
from datetime import datetime
from pykat.testing import utils
20
import sys, traceback
Daniel Brown's avatar
Daniel Brown committed
21
import stat
22
23
24
25
26
27
28
29
30
31
import math

def initProcess(dkats):
    #print "init!!!", dkats
    global done_kats
    done_kats = dkats

def run_kat_file(item):
    kat = item["kat"]
    suite = item["suite"]
32
    runtime = 0.0;
Daniel Brown's avatar
Daniel Brown committed
33
34
35
    try:
        #print os.getpid(),"getting kat...",item["kat"]
        global done_kats
36
        
Daniel Brown's avatar
Daniel Brown committed
37
38
39
40
        FINESSE_EXE = item["FINESSE_EXE"]
        SUITE_PATH = item["SUITE_PATH"]
        SUITE_OUTPUT_DIR  = item["SUITE_OUTPUT_DIR"]
        basename = os.path.splitext(kat)[0]
Daniel Brown's avatar
Daniel Brown committed
41
        exp = None
42
        
Daniel Brown's avatar
Daniel Brown committed
43
44
45
        if item["run_fast"] and ('map ' in open(kat).read()):
            print "skipping " + kat			
        else:
46
            
Daniel Brown's avatar
Daniel Brown committed
47
48
49
            #try:
            start = time.time()
            
50
            out,err = utils.runcmd(["nice", "--18", FINESSE_EXE, "--noheader", kat], cwd=SUITE_PATH)
51
            runtime = time.time()-start
Daniel Brown's avatar
Daniel Brown committed
52
53
54
55
56
57
58
59
60
            
            OUT_FILE = os.path.join(SUITE_PATH,basename + ".out")
            LOG_FILE = os.path.join(SUITE_PATH,basename + ".log")
            
            f_in = open(LOG_FILE, 'rb')
            f_out = gzip.open(LOG_FILE + ".gz", 'wb')
            f_out.writelines(f_in)
            f_out.close()
            f_in.close()
61
            
Daniel Brown's avatar
Daniel Brown committed
62
63
            shutil.move(OUT_FILE, SUITE_OUTPUT_DIR)
            shutil.move(LOG_FILE + ".gz", SUITE_OUTPUT_DIR)
Daniel Brown's avatar
Daniel Brown committed
64
            
Daniel Brown's avatar
Daniel Brown committed
65
66
67
68
69
70
            #except utils.RunException as e:
            #
             #   print "STDERR: " + e.out
              #  print "STDOUT: " + e.err
               # 
               # print "Error running " + kat
Daniel Brown's avatar
Daniel Brown committed
71
                
Daniel Brown's avatar
Daniel Brown committed
72
73
              #  exp = e
            #finally:
Daniel Brown's avatar
Daniel Brown committed
74
            done_kats.value += 1
Daniel Brown's avatar
Daniel Brown committed
75
            #    return [time.time()-start, suite, kat, exp]
Daniel Brown's avatar
Daniel Brown committed
76
                    
77
78
79
    except Exception as e:
        print "main error in kat call"
        exp = None
Daniel Brown's avatar
Daniel Brown committed
80
    finally:
81
82
83
84
85
        # Annoyingly it seems that exceptions can't be returned
	# from a process that has been mapped because exceptions
	# or some at least, can't be pickled, so they will appear as 
	# nan in the tests
        return [runtime, suite, kat, None] #[runtime,suite,kat,e]
Daniel Brown's avatar
Daniel Brown committed
86
        
87
88
89
90
91
92

class DiffException(Exception):
	def __init__(self, msg, outfile):
		self.msg = msg
		self.outfile = outfile

93
class FinesseTestProcess(Thread):
94
        
95
    def __init__(self, TEST_DIR, BASE_DIR, test_commit, 
96
                 run_fast=False, kats={}, test_id="0",
Daniel Brown's avatar
Daniel Brown committed
97
                 git_bin="",emails="", nobuild=False, pool_size=int(multiprocessing.cpu_count()*3.5/4.0),*args, **kqwargs):
98
                 
99
         
100
101
102
103
        self.queue_time = None
        self.status = ""
        self.built = False
        self.total_kats = 0
104
        self.done_kats = multiprocessing.Value('i', 0)
105
106
107
108
109
110
111
112
113
114
        self.git_commit = ""
        self.test_id = -1
        self.finished_test = False
        self.diff_rel_eps = 1e-12
        self.running_kat = ""
        self.running_suite = ""
        self.cancelling = False
        self.errorOccurred = None
        self.diffFound = False
        self.diffing = False
115
        
116
117
118
        if pool_size < 1:
            self.pool_size = 1
        else:
Daniel Brown's avatar
Daniel Brown committed
119
            self.pool_size = pool_size
120
        
121
        Thread.__init__(self)
122
123
124
125
126
        self.git_commit = test_commit
        
        if test_commit is None:
            raise Exception("A git commit ID must be provided for the test")
        
127
128
129
130
        self.kat_run_exceptions = {}
        self.output_differences = {}
        self.run_times = {}
        
131
132
133
134
        self.queue_time = datetime.now()
        self.test_id = test_id
        self.TEST_DIR = TEST_DIR
        self.BASE_DIR = BASE_DIR
Daniel Brown's avatar
Daniel Brown committed
135
        
136
137
138
        self.emails = ""
        
        if type(nobuild) is str:
139
        
140
141
142
143
144
145
            if nobuild.lower() == "true":
                self.nobuild = True
            elif nobuild.lower() == "false":
                self.nobuild = False
            else:
                raise Exception("nobuild is not a boolean value")
146
                
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
        elif type(nobuild) is bool:
            self.nobuild = nobuild
        else:
            raise Exception("nobuild is not a boolean value")
        
        if type(run_fast) is str:
            if run_fast.lower() == "true":
                self.run_fast = True
            elif run_fast.lower() == "false":
                self.run_fast = False
            else:
                raise Exception("run_fast is not a boolean value")
                
        elif type(run_fast) is bool:
            self.run_fast = run_fast
        else:
            raise Exception("nobuild is not a boolean value")
            
        if not os.path.isdir(self.TEST_DIR):
            raise Exception("TEST_DIR was not a valid directory, should point to a clone of the FINESSE test repository")
            
168
        self.kats_to_run = kats
169
        self.GIT_BIN = git_bin
170
171
172
173
174
    
    def cancelCheck(self):
        if self.cancelling:
            raise SystemExit()
    
175
    def percent_done(self):
176
177
178
        if self.total_kats == 0:
            return 0.0
        else:
179
            return 100.0*float(self.done_kats.value)/float(self.total_kats)
180
181
182
183
184
        
    def get_version(self):
        return self.git_commit
        
    def get_progress(self):
185
        if self.diffing:
186
            return 'Diffing {0} out of {1} ({2} in {3})'.format(self.done_kats.value, self.total_kats/2, self.running_kat, self.running_suite)
187
        if self.built:
188
            return 'Running {0} out of {1} ({2} in {3})'.format(self.done_kats.value, self.total_kats/2, self.running_kat, self.running_suite)
189
190
191
192
        else:
            return 'Building FINESSE executable'
            
    def startFinesseTest(self):
193
194
        self.done_kats.value = 0
        
195
196
197
198
        if sys.platform == "win32":
            EXE = ".exe"
        else:
            EXE = ""
199
200
201
        
        print "Using", self.pool_size, "processes..."
        
202
203
        self.built = False

204
        BUILD_PATH = os.path.join(self.BASE_DIR, "build")
205
        
206
        # Firstly we need to build the latest version of finesse
207
        if not self.nobuild:
208
209
210
211
212
213
214
            
            if os.path.exists(self.BASE_DIR):
                print "Deleting previous base_dir " + self.BASE_DIR
                shutil.rmtree(self.BASE_DIR)
            
            os.mkdir(self.BASE_DIR)
            
215
216
217
            print "deleting build dir..." + BUILD_PATH
            if os.path.exists(BUILD_PATH):
                shutil.rmtree(BUILD_PATH)
218
219

            print "Checking out finesse base..."
220
            utils.git(["clone","git://gitmaster.atlas.aei.uni-hannover.de/finesse/finesse.git", BUILD_PATH])
221

222
223
224
            print "Checking out and building develop version of finesse " + self.git_commit
            
            SRC_PATH = os.path.join(BUILD_PATH,"src")
225
            
226
            if sys.platform == "win32":                
227
                utils.git(["checkout",self.git_commit],cwd=SRC_PATH)
228
229
                self.cancelCheck()
                
230
                utils.runcmd(["bash","./finesse.sh","--build"],cwd=BUILD_PATH)
231
                self.cancelCheck()
232
            else:                
233
                utils.git(["checkout",self.git_commit],cwd=SRC_PATH)
234
235
                self.cancelCheck()
                
236
                utils.runcmd(["./finesse.sh","--build"],cwd=BUILD_PATH)
237
                self.cancelCheck()
238
                          
239
        FINESSE_EXE = os.path.join(self.BASE_DIR,"build","kat" + EXE)
Daniel Brown's avatar
Daniel Brown committed
240
        print "NOBUILD",self.nobuild
241
242
        # check if kat runs
        if not os.path.exists(FINESSE_EXE):
243
            raise Exception("Kat file was not found in " + FINESSE_EXE)
244
        
Daniel Brown's avatar
Daniel Brown committed
245
        
Daniel Brown's avatar
Daniel Brown committed
246
        if not os.access(FINESSE_EXE, os.X_OK):
Daniel Brown's avatar
Daniel Brown committed
247
248
            if sys.platform != "win32":
                print "Trying to chmod " + FINESSE_EXE
249
                os.chmod(FINESSE_EXE, stat.S_IRWXU)
250
        
251
        out = None
Daniel Brown's avatar
Daniel Brown committed
252
        
253
        # check version numbers match upkat
254
255
256
257
258
259
260
261
262
        out = utils.runcmd([FINESSE_EXE,"-v"])
            
        # I am sure there is a regex expression that could make this
        # easier but I am being lazy
        out = out[0]
        out = out.split("\n")
        out = out[0].split("(")[1].split(")")
        out = out[0].split("-")[-1]
        shortid = out.lstrip("g")
Daniel Brown's avatar
Daniel Brown committed
263
264
        
        if shortid != self.git_commit[0:len(shortid)]:
265
            raise Exception("Version of kat {0} did not match the version that it was requested to build {1}.".format(shortid, self.git_commit[0:len(shortid)]))
Daniel Brown's avatar
Daniel Brown committed
266
        
267
268
269
270
271
272
273
274
275
276
277
278
        self.built = True
        
        print "kat file found in " + FINESSE_EXE
        
        OUTPUTS_DIR = os.path.join(self.BASE_DIR,"outputs")
        
        if os.path.isdir(OUTPUTS_DIR):
            print "deleting outputs dir..."
            shutil.rmtree(OUTPUTS_DIR)
            
        os.mkdir(OUTPUTS_DIR)
        
279
        os.environ["KATINI"] = os.path.join(BUILD_PATH,"kat.ini")
280
        
281
        self.cancelCheck()
282
283
        # Clean up and pull latest test repository
        print "Cleaning test repository..."
284
        utils.git(["clean","-xdf"], cwd=self.TEST_DIR)
285
        self.cancelCheck()
286
        utils.git(["reset","--hard"], cwd=self.TEST_DIR)
287
        self.cancelCheck()
288
        print "Pulling latest test..."
289
        utils.git(["pull"],cwd=self.TEST_DIR)
290
291
        self.cancelCheck()
    
292
293
294
295
        self.total_kats = 0
        
        # create dictionary structures
        # and count up total number of files to process
296
        for suite in self.kats_to_run.keys():
297
298
            print "RUNNING SUITES", suite
            
299
300
301
            self.kat_run_exceptions[suite] = {}
            self.output_differences[suite] = {}
            self.run_times[suite] = {}
302
            
303
            self.total_kats += len(self.kats_to_run[suite])
304
        
305
306
307
        # multiply as we include the diffining in the percentage
        # done
        self.total_kats *= 2
308
        runs = []
309
        
310
        for suite in self.kats_to_run.keys():
311
            self.cancelCheck()
312
            print "Queuing up suite: " + suite + "..."
313
            kats = self.kats_to_run[suite]
314
            SUITE_PATH = os.path.join(self.TEST_DIR,"kat_test",suite)
315
316
317
318
319

            SUITE_OUTPUT_DIR = os.path.join(OUTPUTS_DIR,suite)
            os.mkdir(SUITE_OUTPUT_DIR)
            
            for kat in kats:
320
321
322
323
                runs.append({'SUITE_OUTPUT_DIR':SUITE_OUTPUT_DIR,'suite':suite, 'run_fast':self.run_fast, 'kat':kat, 'FINESSE_EXE':FINESSE_EXE, 'SUITE_PATH':SUITE_PATH})
        
        self.pool = Pool(initializer=initProcess,initargs=(self.done_kats,) ,processes = self.pool_size)    
        results = self.pool.imap_unordered(run_kat_file, runs, 1)
Daniel Brown's avatar
Daniel Brown committed
324
        print "DONE POOL!!!!"
325
326
327
328
329
330
331
332
333
334
        self.pool.close()
        
        for result in results:
            
            if result[3] is not None:
                self.kat_run_exceptions[result[1]][result[2]] = result[3]
                self.diffFound = True
            
            self.run_times[result[1]][result[2]] = result[0]
        
335
336
        self.cancelCheck()
        
337
        for suite in self.kats_to_run.keys():
338
339
            if len(self.kat_run_exceptions[suite].keys()) > 0:
                print "Could not run the following kats:\n" + "\n".join(self.kat_run_exceptions[suite].keys()) + " in " + suite
340
341
342
            else:
                print "No errors whilst running" + suite

343
        self.diffing = True
344
345
        
        # Now we have generated the output files compare them to the references
346
        for suite in self.kats_to_run.keys():
347
            self.cancelCheck()
348
349
350
            print "Diffing suite: " + suite + "..."

            outs = []
351
            SUITE_PATH = os.path.join(OUTPUTS_DIR,suite)
352

353
            for files in os.listdir(SUITE_PATH):
354
355
356
357
358
359
360
                if files.endswith(".out"):
                    outs.append(files)

            REF_DIR = os.path.join(self.TEST_DIR,"kat_test",suite,"reference")

            if not os.path.exists(REF_DIR):
                raise Exception("Suite reference directory doesn't exist: " + REF_DIR)
361
                
362
            for out in outs:
363
                self.cancelCheck()
364
                self.running_kat = out
365
                
366
                ref_file = os.path.join(REF_DIR,out)
367
                out_file = os.path.join(SUITE_PATH,out)
368
369
370
                
                if not os.path.exists(ref_file):
                    raise DiffException("Reference file doesn't exist for " + out, out)
371
                    
372
373
374
                ref_arr = np.loadtxt(ref_file, dtype=np.float64)
                out_arr = np.loadtxt(out_file, dtype=np.float64)
                                
375
376
377
378
379
                if ref_arr.shape != out_arr.shape:
                    raise DiffException("Reference and output are different shapes", out)

                # for computing relative errors we need to make sure we
                # have no zeros in the data
380
381
382
383
384
                nzix = (ref_arr != 0)
                
                rel_diff = np.abs(out_arr-ref_arr)
                # only compute rel diff for non zero values
                rel_diff[nzix] = np.divide(rel_diff[nzix], np.abs(ref_arr[nzix]))
385
                
386
387
                diff = np.any(rel_diff >= self.diff_rel_eps)
                                
388
                if diff:
389
                    self.diffFound = True
390
                    
391
392
                    # store the rows which are different
                    ix = np.where(rel_diff >= self.diff_rel_eps)[0][0]
393
                
394
395
                    self.output_differences[suite][out] = (True,
                                                           ref_arr[ix],
396
397
398
399
400
                                                           out_arr[ix],
                                                           np.max(rel_diff))
                else:
                    max = np.max(rel_diff) 
                    
401
                    self.output_differences[suite][out] = (False,
402
                                                           max)
Daniel Brown's avatar
Daniel Brown committed
403
404
405
406
407
408
409
410
411
412
413
414
                
                
                        
                # compress the data
                f_in = open(out_file, 'rb')
                f_out = gzip.open(out_file + ".gz", 'wb')
                f_out.writelines(f_in)
                f_out.close()
                f_in.close()
                
                os.remove(out_file)
                
415
                self.done_kats.value += 1
416
                
417
        REPORT_PATH = os.path.join(self.BASE_DIR,"reports")
418
        
419
420
        if not os.path.exists(REPORT_PATH):
            os.mkdir(REPORT_PATH)
421

422
        today = datetime.utcnow()
423
        reportname = os.path.join(REPORT_PATH,"report.log") #today.strftime('%d%m%y')
424
        print "Writing report to " + reportname
425
        
426
427
428
429
430
431
432
433
434
435
436
437
438
439
        f = open(reportname,'w')
        f.write("Python Nightly Test\n")
        f.write(today.strftime('%A, %d. %B %Y %I:%M%p') + "\n")

        # add kat file header
        p = sub.Popen([FINESSE_EXE], stdout=sub.PIPE, stderr=sub.PIPE)
        out, err = p.communicate()
        f.write(out)
        
        # Now time to generate a report...
        np.set_printoptions(precision=16)
        
        isError = False

440
        for suite in self.kats_to_run.keys():
441
442
            f.write("\n\n" + str(len(self.output_differences[suite].keys())) + " differences in suite " + suite)
            for k in self.output_differences[suite].keys():
443
444
                isError = True
                f.write(k + ":\n")
445
446
447
448
449
450
451
452
453
                if self.output_differences[suite][k][0]:
                    f.write("     Differences larger than " + str(self.diff_rel_eps) + "\n")
                    f.write("     ref: " + str(self.output_differences[suite][k][1]) + "\n")
                    f.write("     out: " + str(self.output_differences[suite][k][2]) + "\n")
                    f.write("     Max relative difference: " + str(self.output_differences[suite][k][3]) + "\n")
                else:
                    f.write("     Differences smaller than " + str(self.diff_rel_eps) + "\n")
                    f.write("     Max relative difference: " + str(self.output_differences[suite][k][1]) + "\n")
                    
454
455
            f.write("\n\n" + str(len(self.output_differences[suite].keys())) + " errors in suite " + suite)
            for k in self.kat_run_exceptions[suite].keys():
456
457
                isError = True
                f.write(k + ":\n")
458
                f.write("err: " + self.kat_run_exceptions[suite][k].err + "\n")
459
460
461

        f.close()
        
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
        #if self.emails:
        #    
        #    if isError:
        #        subject = "Finesse test ERROR"
        #    else:
        #        subject = "Finesse test OK"
        #    emails = self.emails

        #    args = ["mailx", "-s", subject, emails]
        #    p = sub.Popen(args, stdout=sub.PIPE, stderr=sub.PIPE, stdin=sub.PIPE)
        #    r = open(reportname,"r")
        #    out, err = p.communicate(r.read())
        #else:
        #    print "No emails specified"

            
478
479
480
481
482
            
    def run(self):
        
        try:
            self.startFinesseTest()
483
        except Exception as ex:
484
            
485
            exc_type, exc_value, exc_traceback = sys.exc_info()
486
487
            
            self.errorOccurred = dict(value=str(exc_value), traceback=str(traceback.format_exc(5)))
488
            
489
490
491
492
            if exc_type is utils.RunException:
                self.errorOccurred["stdout"] = ex.out
                self.errorOccurred["stderr"] = ex.err
                
493
494
495
            print "*** Exception for test_id = " + str(self.test_id)
            traceback.print_exception(exc_type, exc_value, exc_traceback,
                                      limit=5, file=sys.stdout)
496
        finally:
497
            self.finished_test = True
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
        
        

if __name__ == "__main__":
    
    parser = OptionParser()
    
    parser.add_option("-t","--test-dir",dest="test_dir",help="")
    parser.add_option("-b","--base-dir",dest="base_dir",help="")
    parser.add_option("-c","--test-commit",dest="test_commit",help="")
    parser.add_option("-s","--suites",dest="suites",help="comma delimited list of each suite to run")
    parser.add_option("-g","--git-bin",dest="git_bin", default="git",help="")
    parser.add_option("-e","--emails",dest="emails", help="")
    parser.add_option("-n","--no-build",default="False",dest="nobuild",action="store_true")
    parser.add_option("-f","--fast",default="True",dest="fast",action="store_true")

    options, args = parser.parse_args()

    if options.test_dir is None:
        print "--test-dir argument is missing"
        exit()
        
    if options.test_commit is None:
        print "--test-commit argument is missing"
        exit()
    
    if options.base_dir is None:
        options.base_dir = os.getcwd()
    
    if options.suites is None:
        suites = []
    else:
        suites = options.suites.split(",")
        
    test = FinesseTestProcess(options.test_dir,
                              options.base_dir,
                              options.test_commit,
                              run_fast=options.fast,
                              suites=suites,
                              git_bin=options.git_bin,
                              emails=options.emails,
                              nobuild=options.nobuild)
540
541
    test.run()
    
542