from tempfile import mkstemp
from subprocess import Popen, PIPE
from difflib import Differ
+from hashlib import md5
+from shutil import copy
-DEBUG = False
+DEBUG = 0
VERBOSE = False
-l List all tests
-s Show test(s)
-r Run test(s)
+ -f Fix failed test(s)
-R <directory> Save test reports to <directory>
<test> Protocol decoder name ("i2c") and optionally test name ("i2c/icc")""")
sys.exit()
return tests
-def diff_files(f1, f2):
+def diff_text(f1, f2):
t1 = open(f1).readlines()
t2 = open(f2).readlines()
diff = []
return diff
-def run_tests(tests):
+def compare_binary(f1, f2):
+ h1 = md5()
+ h1.update(open(f1, 'rb').read())
+ h2 = md5()
+ h2.update(open(f2, 'rb').read())
+ if h1.digest() == h2.digest():
+ result = None
+ else:
+ result = ["Binary output does not match."]
+
+ return result
+
+
+def run_tests(tests, fix=False):
errors = 0
results = []
cmd = os.path.join(tests_dir, 'runtc')
for tclist in tests:
for tc in tclist:
args = [cmd]
+ if DEBUG > 1:
+ args.append('-d')
for pd in tc['pdlist']:
args.extend(['-P', pd['name']])
for label, probe in pd['probes']:
fd, outfile = mkstemp()
os.close(fd)
opargs.extend(['-f', outfile])
- DBG("Running %s %s" % (cmd, ' '.join(args + opargs)))
- stdout, stderr = Popen(args + opargs, stdout=PIPE, stderr=PIPE).communicate()
+ DBG("Running %s" % (' '.join(args + opargs)))
+ p = Popen(args + opargs, stdout=PIPE, stderr=PIPE)
+ stdout, stderr = p.communicate()
if stdout:
results[-1]['statistics'] = stdout.decode('utf-8').strip()
if stderr:
results[-1]['error'] = stderr.decode('utf-8').strip()
errors += 1
- match = "%s/%s/test/%s" % (decoders_dir, op['pd'], op['match'])
- diff = diff_files(match, outfile)
- if diff:
- results[-1]['diff'] = diff
+ elif p.returncode != 0:
+ # runtc indicated an error, but didn't output a
+ # message on stderr about it
+ results[-1]['error'] = "Unknown error: runtc %d" % p.returncode
+ if 'error' not in results[-1]:
+ match = os.path.join(decoders_dir, op['pd'], 'test', op['match'])
+ try:
+ diff = diff_error = None
+ if op['type'] in ('annotation', 'python'):
+ diff = diff_text(match, outfile)
+ elif op['type'] == 'binary':
+ diff = compare_binary(match, outfile)
+ else:
+ diff = ["Unsupported output type '%s'." % op['type']]
+ except Exception as e:
+ diff_error = e
+ if fix:
+ if diff or diff_error:
+ copy(outfile, match)
+ DBG("Wrote %s" % match)
+ else:
+ if diff:
+ results[-1]['diff'] = diff
+ elif diff_error is not None:
+ raise diff_error
except Exception as e:
results[-1]['error'] = str(e)
finally:
if len(sys.argv) == 1:
usage()
-opt_all = opt_run = opt_show = opt_list = False
+opt_all = opt_run = opt_show = opt_list = opt_fix = False
report_dir = None
-opts, args = getopt(sys.argv[1:], "dvarslR:")
+opts, args = getopt(sys.argv[1:], "dvarslfRS:")
for opt, arg in opts:
if opt == '-d':
- DEBUG = True
+ DEBUG += 1
if opt == '-v':
VERBOSE = True
elif opt == '-a':
opt_show = True
elif opt == '-l':
opt_list = True
+ elif opt == '-f':
+ opt_fix = True
elif opt == '-R':
report_dir = arg
+ elif opt == '-S':
+ dumps_dir = arg
if opt_run and opt_show:
usage("Use either -s or -r, not both.")
usage("Specify either -a or tests.")
if opt_run:
+ if not os.path.isdir(dumps_dir):
+ ERR("Could not find sigrok-dumps repository at %s" % dumps_dir)
+ sys.exit(1)
results, errors = run_tests(testlist)
ret = errors
elif opt_show:
show_tests(testlist)
elif opt_list:
list_tests(testlist)
+ elif opt_fix:
+ run_tests(testlist, fix=True)
else:
usage()
except Exception as e: