compare_to_gold.py 2.04 KB
Newer Older
1
2
3
4
5
6
7
"""Compares a test result to the gold standard.

Error codes:
----------------------------------------------------------
  G     | Error in the gold file -- bad testname?
  D     | Error in my file -- test did not complete?
  E     | Error in my file size -- test did not complete.
8
9
10
  N/A   | Unknown error
  FAIL  | Failing test
  PASS  | Passing test
11
12
13
14
15
16
17
18
19
20
----------------------------------------------------------

"""

import argparse
import numpy as np
import os

TOL = 1.e-5

21
22
23
def run_comparison(testname, full_message=False):
    # get gold file
    fname_gold = os.path.join('..', 'tests_gold', '%s.soln'%testname)
24
25
26
27
    try:
        gold = np.loadtxt(fname_gold, skiprows=1)
    except IOError:
        print 'ERROR: cannot find gold file "%s", bad testname?'%fname_gold
28
        return 'G '
29

30
31
    # get my file
    fname_mine = '%s.soln'%testname
32
33
34
    try:
        mine = np.loadtxt(fname_mine, skiprows=1)
    except IOError:
35
36
37
38
39
        fname_mine = '%s.stdout'%testname
        try:
            mine = np.loadtxt(fname_mine, skiprows=1)
        except IOError:
            print 'ERROR: cannot find my file "%s", failed test run?'%fname_mine
40
            return 'D '
41
42
        
    # compare
43
44
    try:
        close = np.allclose(gold, mine)
45
46
47
48
    except ValueError as err:
        if full_message:
            print ''
            print 'Test: "%s" FAILED with error: "%r"'%(testname, err)
49
        return 'E '
50
51

    if close:
52
        return 'PASS '
53
    else:
54
        return 'FAIL '
55
56
57
58
59
60
                      

if __name__ == "__main__":
    parser = argparse.ArgumentParser(__doc__)
    parser.add_argument('testnames', nargs='+',
                      help='Name of tests to run')
61
62
    parser.add_argument('-v', '--verbose', action='store_true',
                        help='Print error messages')
63
64
65
66
67
68
    args = parser.parse_args()

    results = []
    print 'Comparing tests:',
    for test in args.testnames:
        try:
69
            results.append(run_comparison(test, args.verbose))
70
71
        except Exception as err:
            print err
72
            results.append('N/A ')
73
    print ''.join(results)