3 ## This file is part of the libsigrokdecode project.
5 ## Copyright (C) 2013 Bert Vermeulen <bert@biot.com>
7 ## This program is free software: you can redistribute it and/or modify
8 ## it under the terms of the GNU General Public License as published by
9 ## the Free Software Foundation, either version 3 of the License, or
10 ## (at your option) any later version.
12 ## This program is distributed in the hope that it will be useful,
13 ## but WITHOUT ANY WARRANTY; without even the implied warranty of
14 ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 ## GNU General Public License for more details.
17 ## You should have received a copy of the GNU General Public License
18 ## along with this program. If not, see <http://www.gnu.org/licenses/>.
23 from getopt import getopt
24 from tempfile import mkstemp
25 from subprocess import Popen, PIPE
26 from difflib import Differ
27 from hashlib import md5
28 from shutil import copy
34 class E_syntax(Exception):
36 class E_badline(Exception):
39 def INFO(msg, end='\n'):
51 print(msg, file=sys.stderr)
56 print(msg.strip() + '\n')
57 print("""Usage: testpd [-dvarslR] [test, ...]
65 -R <directory> Save test reports to <directory>
66 <test> Protocol decoder name ("i2c") and optionally test name ("i2c/icc")""")
71 if 'pdlist' not in tc or not tc['pdlist']:
72 return("No protocol decoders")
73 if 'input' not in tc or not tc['input']:
75 if 'output' not in tc or not tc['output']:
77 for op in tc['output']:
79 return("No match in output")
84 def parse_testfile(path, pd, tc, op_type, op_class):
85 DBG("Opening '%s'" % path)
87 for line in open(path).read().split('\n'):
90 if len(line) == 0 or line[0] == "#":
93 if not tclist and f[0] != "test":
107 elif key == 'protocol-decoder':
117 # Always needs <key> <value>
123 opt, val = b.split('=')
129 pd_spec['probes'].append([opt, val])
131 pd_spec['options'].append([opt, val])
134 tclist[-1]['pdlist'].append(pd_spec)
138 tclist[-1]['stack'] = f
142 tclist[-1]['input'] = f[0]
143 elif key == 'output':
150 # Always needs <key> <value>
160 tclist[-1]['output'].append(op_spec)
163 except E_badline as e:
164 ERR("Invalid syntax in %s: line '%s'" % (path, line))
166 except E_syntax as e:
167 ERR("Unable to parse %s: unknown line '%s'" % (path, line))
170 # If a specific testcase was requested, keep only that one.
177 # ...and a specific output type
178 if op_type is not None:
180 for op in target_tc['output']:
181 if op['type'] == op_type:
182 # ...and a specific output class
183 if op_class is None or ('class' in op and op['class'] == op_class):
184 target_oplist.append(op)
185 DBG("match on [%s]" % str(op))
186 target_tc['output'] = target_oplist
187 if target_tc is None:
192 error = check_tclist(t)
194 ERR("Error in %s: %s" % (path, error))
200 def get_tests(testnames):
202 for testspec in testnames:
203 # Optional testspec in the form i2c/rtc
204 tc = op_type = op_class = None
205 ts = testspec.strip("/").split("/")
213 path = os.path.join(decoders_dir, pd)
214 if not os.path.isdir(path):
215 # User specified non-existent PD
216 raise Exception("%s not found." % path)
217 path = os.path.join(decoders_dir, pd, "test/test.conf")
218 if not os.path.exists(path):
219 # PD doesn't have any tests yet
221 tests.append(parse_testfile(path, pd, tc, op_type, op_class))
226 def diff_text(f1, f2):
227 t1 = open(f1).readlines()
228 t2 = open(f2).readlines()
231 for line in d.compare(t1, t2):
232 if line[:2] in ('- ', '+ '):
233 diff.append(line.strip())
238 def compare_binary(f1, f2):
240 h1.update(open(f1, 'rb').read())
242 h2.update(open(f2, 'rb').read())
243 if h1.digest() == h2.digest():
246 result = ["Binary output does not match."]
251 def run_tests(tests, fix=False):
254 cmd = os.path.join(tests_dir, 'runtc')
260 for pd in tc['pdlist']:
261 args.extend(['-P', pd['name']])
262 for label, probe in pd['probes']:
263 args.extend(['-p', "%s=%d" % (label, probe)])
264 for option, value in pd['options']:
265 args.extend(['-o', "%s=%s" % (option, value)])
266 args.extend(['-i', os.path.join(dumps_dir, tc['input'])])
267 for op in tc['output']:
268 name = "%s/%s/%s" % (tc['pd'], tc['name'], op['type'])
269 opargs = ['-O', "%s:%s" % (op['pd'], op['type'])]
271 opargs[-1] += ":%s" % op['class']
272 name += "/%s" % op['class']
274 dots = '.' * (60 - len(name) - 2)
275 INFO("%s %s " % (name, dots), end='')
280 fd, outfile = mkstemp()
282 opargs.extend(['-f', outfile])
283 DBG("Running %s" % (' '.join(args + opargs)))
284 p = Popen(args + opargs, stdout=PIPE, stderr=PIPE)
285 stdout, stderr = p.communicate()
287 results[-1]['statistics'] = stdout.decode('utf-8').strip()
289 results[-1]['error'] = stderr.decode('utf-8').strip()
291 elif p.returncode != 0:
292 # runtc indicated an error, but didn't output a
293 # message on stderr about it
294 results[-1]['error'] = "Unknown error: runtc %d" % p.returncode
295 if 'error' not in results[-1]:
296 matchfile = os.path.join(decoders_dir, op['pd'], 'test', op['match'])
297 DBG("Comparing with %s" % matchfile)
299 diff = diff_error = None
300 if op['type'] in ('annotation', 'python'):
301 diff = diff_text(matchfile, outfile)
302 elif op['type'] == 'binary':
303 diff = compare_binary(matchfile, outfile)
305 diff = ["Unsupported output type '%s'." % op['type']]
306 except Exception as e:
309 if diff or diff_error:
310 copy(outfile, matchfile)
311 DBG("Wrote %s" % matchfile)
314 results[-1]['diff'] = diff
315 elif diff_error is not None:
317 except Exception as e:
318 results[-1]['error'] = str(e)
322 if 'diff' in results[-1]:
323 INFO("Output mismatch")
324 elif 'error' in results[-1]:
325 error = results[-1]['error']
327 error = error[:17] + '...'
331 gen_report(results[-1])
333 return results, errors
336 def gen_report(result):
338 if 'error' in result:
340 out.append(result['error'])
343 out.append("Test output mismatch:")
344 out.extend(result['diff'])
346 if 'statistics' in result:
347 out.extend(["Statistics:", result['statistics']])
351 text = "Testcase: %s\n" % result['testcase']
352 text += '\n'.join(out)
357 filename = result['testcase'].replace('/', '_')
358 open(os.path.join(report_dir, filename), 'w').write(text)
363 def show_tests(tests):
366 print("Testcase: %s/%s" % (tc['pd'], tc['name']))
367 for pd in tc['pdlist']:
368 print(" Protocol decoder: %s" % pd['name'])
369 for label, probe in pd['probes']:
370 print(" Probe %s=%d" % (label, probe))
371 for option, value in pd['options']:
372 print(" Option %s=%d" % (option, value))
374 print(" Stack: %s" % ' '.join(tc['stack']))
375 print(" Input: %s" % tc['input'])
376 for op in tc['output']:
377 print(" Output:\n Protocol decoder: %s" % op['pd'])
378 print(" Type: %s" % op['type'])
380 print(" Class: %s" % op['class'])
381 print(" Match: %s" % op['match'])
385 def list_tests(tests):
388 for op in tc['output']:
389 line = "%s/%s/%s" % (tc['pd'], tc['name'], op['type'])
391 line += "/%s" % op['class']
400 tests_dir = os.path.abspath(os.path.dirname(sys.argv[0]))
401 base_dir = os.path.abspath(os.path.join(os.curdir, tests_dir, os.path.pardir))
402 dumps_dir = os.path.abspath(os.path.join(base_dir, os.path.pardir, 'sigrok-dumps'))
403 decoders_dir = os.path.abspath(os.path.join(base_dir, 'decoders'))
405 if len(sys.argv) == 1:
408 opt_all = opt_run = opt_show = opt_list = opt_fix = False
410 opts, args = getopt(sys.argv[1:], "dvarslfR:S:")
411 for opt, arg in opts:
431 if opt_run and opt_show:
432 usage("Use either -s or -r, not both.")
434 usage("Specify either -a or tests, not both.")
435 if report_dir is not None and not os.path.isdir(report_dir):
436 usage("%s is not a directory" % report_dir)
441 testlist = get_tests(args)
443 testlist = get_tests(os.listdir(decoders_dir))
445 usage("Specify either -a or tests.")
448 if not os.path.isdir(dumps_dir):
449 ERR("Could not find sigrok-dumps repository at %s" % dumps_dir)
451 results, errors = run_tests(testlist, fix=opt_fix)
458 run_tests(testlist, fix=True)
461 except Exception as e:
462 print("Error: %s" % str(e))