MITgcm/verification/verification_parser.py

dcd2743bdb Ed D*0001 """Parse verification outputs."""
                0002
                0003 import re
                0004 import glob
                0005 import os
                0006
ccfd6c9b7c Chri*0007 def verification_parser(filename, threshold, input_dir_pat):
dcd2743bdb Ed D*0008
                0009     # function should be given a threshold value for each sub test
                0010     (directory, _) = os.path.split(filename)
                0011     # how many additional tests are run with tweaks to this configuration
ccfd6c9b7c Chri*0012     num_exps = len(glob.glob(directory+input_dir_pat))+1
dcd2743bdb Ed D*0013
                0014     # check that the correct number of values for `threshold` have been given
                0015     if len(threshold) != num_exps:
                0016         # some if statements to deal with grammar
                0017         if len(threshold)==1:
                0018             error_message = '{0} value given for threshold, '.format(len(threshold))
                0019         else:
                0020             error_message = '{0} values given for threshold, '.format(len(threshold))
                0021
                0022         if num_exps==1:
                0023             error_message = error_message + 'but {0} subtest found.'.format(num_exps)
                0024         else:
                0025             error_message = error_message + 'but {0} subtests found.'.format(num_exps)
                0026
                0027         raise ValueError(error_message)
                0028
                0029     # open the testreport output to assess pass/fail
                0030     with open(filename) as f:
                0031         lines = f.readlines()
                0032
                0033         first_match = True
                0034
                0035         # extract lines from output
                0036         for i, line in enumerate(lines):
                0037             if line[:5] == '2 d e':
                0038                 if first_match:
                0039                     # skip the first match, since it doesn't contain output,
                0040                     # but set to false to catch next matches.
                0041                     first_match = False
                0042                 else:
                0043                     # save the line number where the output is found
                0044                     output_line = i
                0045                     break
                0046
                0047         # loop through each of the subexperiments:
                0048         for j in xrange(len(threshold)):
                0049             test_results = lines[output_line+2+j]
                0050
                0051             # split test_results into a list with values for each number.
                0052             # this uses spaces and the < > characters to separate the numbers.
b27f340083 Oliv*0053             test_results = re.split('[ ><]+',test_results)
1f2be90d9e Ed D*0054             # Check the Genmake, depend, make, and run checks
                0055
                0056             for status in test_results[:4]:
                0057                 assert status== 'Y'
                0058
                0059             # Ignore the build status varaibles that were just checked, as
                0060             # well as "pass" or "fail" and test name at the end of the line
b27f340083 Oliv*0061             test_results = test_results[4:-2]
1f2be90d9e Ed D*0062
dcd2743bdb Ed D*0063             # convert to floats
                0064             dp_similarity = []
                0065             for i, x in enumerate(test_results):
                0066                 try:
                0067                     dp_similarity.append(float(x))
                0068                 except ValueError:
                0069                     pass
                0070
                0071
b27f340083 Oliv*0072             if len(dp_similarity) == 3:
                0073                 # adjoint test.
                0074                 # Remove forward gradient as it may have few matching digits.
                0075                 del dp_similarity[2]
                0076             elif len(dp_similarity) >= 17:
dcd2743bdb Ed D*0077                 # this means that the test wasn't an offline advection test.
                0078                 # Remove the means of u and v since they are constrained
                0079                 # to ~0 by domain geometry and can cause the test to fail
                0080                 # when it shouldn't.
                0081                 del dp_similarity[15]
                0082                 del dp_similarity[11]
                0083
                0084             assert all(elements >= threshold[j] for elements in dp_similarity)
                0085
                0086 if __name__ == '__main__':
                0087
                0088     import argparse
                0089
                0090     parser = argparse.ArgumentParser(description='Check that verification simulation passed the test.')
                0091
                0092     parser.add_argument('-filename', type=str,
                0093                         help='path to output file from the verification test')
                0094
                0095     parser.add_argument('-threshold',nargs='+', type=int, default=15,
                0096                         help='number of decimal places of similarity required for test to pass. Requires a value for each sub test. Separate values with a space.')
                0097
ccfd6c9b7c Chri*0098     parser.add_argument('-input_dir_pat', type=str, default='/input.*',
c32b71a53d chri*0099                         help='Directory pattern for searching for sub-experiments for base, oad, adm, tlm. Default /input.*')
                0100
dcd2743bdb Ed D*0101     args = parser.parse_args()
                0102
                0103     verification_parser(**vars(args))