pdtest/runtc: Add support for binary output types.

[libsigrokdecode.git] / tests / pdtest

diff --git a/tests/pdtest b/tests/pdtest
index 45fb342cede6ac54464ea6f1a76bdfead660f3aa..2a90f2c2e965f4e84d0b5037dccbad2b7be559b0 100755 (executable)
--- a/tests/pdtest
+++ b/tests/pdtest
@@ -6,8 +6,10 @@ from getopt import getopt
 from tempfile import mkstemp
 from subprocess import Popen, PIPE
 from difflib import Differ
+from hashlib import md5
+from shutil import copy
 
-DEBUG = False
+DEBUG = 0
 VERBOSE = False
 
 
@@ -41,6 +43,7 @@ def usage(msg=None):
   -l  List all tests
   -s  Show test(s)
   -r  Run test(s)
+  -f  Fix failed test(s)
   -R <directory>  Save test reports to <directory>
   <test>  Protocol decoder name ("i2c") and optionally test name ("i2c/icc")""")
     sys.exit()
@@ -202,7 +205,7 @@ def get_tests(testnames):
     return tests
 
 
-def diff_files(f1, f2):
+def diff_textfiles(f1, f2):
     t1 = open(f1).readlines()
     t2 = open(f2).readlines()
     diff = []
@@ -214,13 +217,28 @@ def diff_files(f1, f2):
     return diff
 
 
-def run_tests(tests):
+def compare_binfiles(f1, f2):
+    h1 = md5()
+    h1.update(open(f1, 'rb').read())
+    h2 = md5()
+    h2.update(open(f2, 'rb').read())
+    if h1.digest() == h2.digest():
+        result = None
+    else:
+        result = ["Binary output does not match."]
+
+    return result
+
+
+def run_tests(tests, fix=False):
     errors = 0
     results = []
     cmd = os.path.join(tests_dir, 'runtc')
     for tclist in tests:
         for tc in tclist:
             args = [cmd]
+            if DEBUG > 1:
+                args.append('-d')
             for pd in tc['pdlist']:
                 args.extend(['-P', pd['name']])
                 for label, probe in pd['probes']:
@@ -244,17 +262,39 @@ def run_tests(tests):
                     fd, outfile = mkstemp()
                     os.close(fd)
                     opargs.extend(['-f', outfile])
-                    DBG("Running %s %s" % (cmd, ' '.join(args + opargs)))
-                    stdout, stderr = Popen(args + opargs, stdout=PIPE, stderr=PIPE).communicate()
+                    DBG("Running %s" % (' '.join(args + opargs)))
+                    p = Popen(args + opargs, stdout=PIPE, stderr=PIPE)
+                    stdout, stderr = p.communicate()
                     if stdout:
                         results[-1]['statistics'] = stdout.decode('utf-8').strip()
                     if stderr:
                         results[-1]['error'] = stderr.decode('utf-8').strip()
                         errors += 1
-                    match = "%s/%s/test/%s" % (decoders_dir, op['pd'], op['match'])
-                    diff = diff_files(match, outfile)
-                    if diff:
-                        results[-1]['diff'] = diff
+                    elif p.returncode != 0:
+                        # runtc indicated an error, but didn't output a
+                        # message on stderr about it
+                        results[-1]['error'] = "Unknown error: runtc %d" % p.returncode
+                    if 'error' not in results[-1]:
+                        match = os.path.join(decoders_dir, op['pd'], 'test', op['match'])
+                        try:
+                            diff = diff_error = None
+                            if op['type'] == 'annotation':
+                                diff = diff_textfiles(match, outfile)
+                            elif op['type'] == 'binary':
+                                diff = compare_binfiles(match, outfile)
+                            else:
+                                diff = ["Unsupported output type '%s'." % op['type']]
+                        except Exception as e:
+                            diff_error = e
+                        if fix:
+                            if diff or diff_error:
+                                copy(outfile, match)
+                                DBG("Wrote %s" % match)
+                        else:
+                            if diff:
+                                results[-1]['diff'] = diff
+                            elif diff_error is not None:
+                                raise diff_error
                 except Exception as e:
                     results[-1]['error'] = str(e)
                 finally:
@@ -346,12 +386,12 @@ decoders_dir = os.path.abspath(os.path.join(base_dir, 'decoders'))
 if len(sys.argv) == 1:
     usage()
 
-opt_all = opt_run = opt_show = opt_list = False
+opt_all = opt_run = opt_show = opt_list = opt_fix = False
 report_dir = None
-opts, args = getopt(sys.argv[1:], "dvarslRS:")
+opts, args = getopt(sys.argv[1:], "dvarslfRS:")
 for opt, arg in opts:
     if opt == '-d':
-        DEBUG = True
+        DEBUG += 1
     if opt == '-v':
         VERBOSE = True
     elif opt == '-a':
@@ -362,6 +402,8 @@ for opt, arg in opts:
         opt_show = True
     elif opt == '-l':
         opt_list = True
+    elif opt == '-f':
+        opt_fix = True
     elif opt == '-R':
         report_dir = arg
     elif opt == '-S':
@@ -393,6 +435,8 @@ try:
         show_tests(testlist)
     elif opt_list:
         list_tests(testlist)
+    elif opt_fix:
+        run_tests(testlist, fix=True)
     else:
         usage()
 except Exception as e: