1 #!/usr/bin/env /usr/bin/python3
5 from getopt import getopt
6 from tempfile import mkstemp
7 from subprocess import Popen, PIPE
8 from difflib import Differ
14 class E_syntax(Exception):
16 class E_badline(Exception):
19 def INFO(msg, end='\n'):
31 print(msg, file=sys.stderr)
36 print(msg.strip() + '\n')
37 print("""Usage: testpd [-dvarslR] [test, ...]
45 -R <directory> Save test reports to <directory>
46 <test> Protocol decoder name ("i2c") and optionally test name ("i2c/icc")""")
51 if 'pdlist' not in tc or not tc['pdlist']:
52 return("No protocol decoders")
53 if 'input' not in tc or not tc['input']:
55 if 'output' not in tc or not tc['output']:
57 for op in tc['output']:
59 return("No match in output")
64 def parse_testfile(path, pd, tc, op_type, op_class):
65 DBG("Opening '%s'" % path)
67 for line in open(path).read().split('\n'):
70 if len(line) == 0 or line[0] == "#":
73 if not tclist and f[0] != "test":
87 elif key == 'protocol-decoder':
97 # Always needs <key> <value>
103 opt, val = b.split('=')
109 pd_spec['probes'].append([opt, val])
111 pd_spec['options'].append([opt, val])
114 tclist[-1]['pdlist'].append(pd_spec)
118 tclist[-1]['stack'] = f
122 tclist[-1]['input'] = f[0]
123 elif key == 'output':
130 # Always needs <key> <value>
140 tclist[-1]['output'].append(op_spec)
143 except E_badline as e:
144 ERR("Invalid syntax in %s: line '%s'" % (path, line))
146 except E_syntax as e:
147 ERR("Unable to parse %s: unknown line '%s'" % (path, line))
150 # If a specific testcase was requested, keep only that one.
157 # ...and a specific output type
158 if op_type is not None:
160 for op in target_tc['output']:
161 if op['type'] == op_type:
162 # ...and a specific output class
163 if op_class is None or ('class' in op and op['class'] == op_class):
164 target_oplist.append(op)
165 DBG("match on [%s]" % str(op))
166 target_tc['output'] = target_oplist
167 if target_tc is None:
172 error = check_tclist(t)
174 ERR("Error in %s: %s" % (path, error))
180 def get_tests(testnames):
182 for testspec in testnames:
183 # Optional testspec in the form i2c/rtc
184 tc = op_type = op_class = None
185 ts = testspec.strip("/").split("/")
193 path = os.path.join(decoders_dir, pd)
194 if not os.path.isdir(path):
195 # User specified non-existent PD
196 raise Exception("%s not found." % path)
197 path = os.path.join(decoders_dir, pd, "test/test.conf")
198 if not os.path.exists(path):
199 # PD doesn't have any tests yet
201 tests.append(parse_testfile(path, pd, tc, op_type, op_class))
206 def diff_files(f1, f2):
207 t1 = open(f1).readlines()
208 t2 = open(f2).readlines()
211 for line in d.compare(t1, t2):
212 if line[:2] in ('- ', '+ '):
213 diff.append(line.strip())
218 def run_tests(tests, fix=False):
221 cmd = os.path.join(tests_dir, 'runtc')
227 for pd in tc['pdlist']:
228 args.extend(['-P', pd['name']])
229 for label, probe in pd['probes']:
230 args.extend(['-p', "%s=%d" % (label, probe)])
231 for option, value in pd['options']:
232 args.extend(['-o', "%s=%s" % (option, value)])
233 args.extend(['-i', os.path.join(dumps_dir, tc['input'])])
234 for op in tc['output']:
235 name = "%s/%s/%s" % (tc['pd'], tc['name'], op['type'])
236 opargs = ['-O', "%s:%s" % (op['pd'], op['type'])]
238 opargs[-1] += ":%s" % op['class']
239 name += "/%s" % op['class']
241 dots = '.' * (60 - len(name) - 2)
242 INFO("%s %s " % (name, dots), end='')
247 fd, outfile = mkstemp()
249 opargs.extend(['-f', outfile])
250 DBG("Running %s" % (' '.join(args + opargs)))
251 p = Popen(args + opargs, stdout=PIPE, stderr=PIPE)
252 stdout, stderr = p.communicate()
254 results[-1]['statistics'] = stdout.decode('utf-8').strip()
256 results[-1]['error'] = stderr.decode('utf-8').strip()
258 elif p.returncode != 0:
259 # runtc indicated an error, but didn't output a
260 # message on stderr about it
261 results[-1]['error'] = "Unknown error: runtc %d" % p.returncode
262 if 'error' not in results[-1]:
263 match = os.path.join(decoders_dir, op['pd'], 'test', op['match'])
265 diff = diff_error = None
266 if op['type'] == 'annotation':
267 diff = diff_textfiles(match, outfile)
269 diff = ["Unsupported output type '%s'." % op['type']]
270 except Exception as e:
273 if diff or diff_error:
275 DBG("Wrote %s" % match)
278 results[-1]['diff'] = diff
279 elif diff_error is not None:
281 except Exception as e:
282 results[-1]['error'] = str(e)
286 if 'diff' in results[-1]:
287 INFO("Output mismatch")
288 elif 'error' in results[-1]:
289 error = results[-1]['error']
291 error = error[:17] + '...'
295 gen_report(results[-1])
297 return results, errors
300 def gen_report(result):
302 if 'error' in result:
304 out.append(result['error'])
307 out.append("Test output mismatch:")
308 out.extend(result['diff'])
310 if 'statistics' in result:
311 out.extend(["Statistics:", result['statistics']])
315 text = "Testcase: %s\n" % result['testcase']
316 text += '\n'.join(out)
321 filename = result['testcase'].replace('/', '_')
322 open(os.path.join(report_dir, filename), 'w').write(text)
327 def show_tests(tests):
330 print("Testcase: %s/%s" % (tc['pd'], tc['name']))
331 for pd in tc['pdlist']:
332 print(" Protocol decoder: %s" % pd['name'])
333 for label, probe in pd['probes']:
334 print(" Probe %s=%d" % (label, probe))
335 for option, value in pd['options']:
336 print(" Option %s=%d" % (option, value))
338 print(" Stack: %s" % ' '.join(tc['stack']))
339 print(" Input: %s" % tc['input'])
340 for op in tc['output']:
341 print(" Output:\n Protocol decoder: %s" % op['pd'])
342 print(" Type: %s" % op['type'])
344 print(" Class: %s" % op['class'])
345 print(" Match: %s" % op['match'])
349 def list_tests(tests):
352 for op in tc['output']:
353 line = "%s/%s/%s" % (tc['pd'], tc['name'], op['type'])
355 line += "/%s" % op['class']
364 tests_dir = os.path.abspath(os.path.dirname(sys.argv[0]))
365 base_dir = os.path.abspath(os.path.join(os.curdir, tests_dir, os.path.pardir))
366 dumps_dir = os.path.abspath(os.path.join(base_dir, os.path.pardir, 'sigrok-dumps'))
367 decoders_dir = os.path.abspath(os.path.join(base_dir, 'decoders'))
369 if len(sys.argv) == 1:
372 opt_all = opt_run = opt_show = opt_list = opt_fix = False
374 opts, args = getopt(sys.argv[1:], "dvarslfRS:")
375 for opt, arg in opts:
395 if opt_run and opt_show:
396 usage("Use either -s or -r, not both.")
398 usage("Specify either -a or tests, not both.")
399 if report_dir is not None and not os.path.isdir(report_dir):
400 usage("%s is not a directory" % report_dir)
405 testlist = get_tests(args)
407 testlist = get_tests(os.listdir(decoders_dir))
409 usage("Specify either -a or tests.")
412 if not os.path.isdir(dumps_dir):
413 ERR("Could not find sigrok-dumps repository at %s" % dumps_dir)
415 results, errors = run_tests(testlist)
422 run_tests(testlist, fix=True)
425 except Exception as e:
426 print("Error: %s" % str(e))