3 ## This file is part of the libsigrokdecode project.
5 ## Copyright (C) 2013 Bert Vermeulen <bert@biot.com>
7 ## This program is free software: you can redistribute it and/or modify
8 ## it under the terms of the GNU General Public License as published by
9 ## the Free Software Foundation, either version 3 of the License, or
10 ## (at your option) any later version.
12 ## This program is distributed in the hope that it will be useful,
13 ## but WITHOUT ANY WARRANTY; without even the implied warranty of
14 ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 ## GNU General Public License for more details.
17 ## You should have received a copy of the GNU General Public License
18 ## along with this program. If not, see <http://www.gnu.org/licenses/>.
23 from getopt import getopt
24 from tempfile import mkstemp
25 from subprocess import Popen, PIPE
26 from difflib import Differ
27 from hashlib import md5
28 from shutil import copy
34 class E_syntax(Exception):
36 class E_badline(Exception):
39 def INFO(msg, end='\n'):
51 print(msg, file=sys.stderr)
56 print(msg.strip() + '\n')
57 print("""Usage: testpd [-dvarslR] [test, ...]
65 -R <directory> Save test reports to <directory>
66 <test> Protocol decoder name ("i2c") and optionally test name ("i2c/icc")""")
71 if 'pdlist' not in tc or not tc['pdlist']:
72 return("No protocol decoders")
73 if 'input' not in tc or not tc['input']:
75 if 'output' not in tc or not tc['output']:
77 for op in tc['output']:
79 return("No match in output")
84 def parse_testfile(path, pd, tc, op_type, op_class):
85 DBG("Opening '%s'" % path)
87 for line in open(path).read().split('\n'):
90 if len(line) == 0 or line[0] == "#":
93 if not tclist and f[0] != "test":
107 elif key == 'protocol-decoder':
117 # Always needs <key> <value>
123 opt, val = b.split('=')
129 pd_spec['probes'].append([opt, val])
131 pd_spec['options'].append([opt, val])
134 tclist[-1]['pdlist'].append(pd_spec)
138 tclist[-1]['stack'] = f
142 tclist[-1]['input'] = f[0]
143 elif key == 'output':
150 # Always needs <key> <value>
160 tclist[-1]['output'].append(op_spec)
163 except E_badline as e:
164 ERR("Invalid syntax in %s: line '%s'" % (path, line))
166 except E_syntax as e:
167 ERR("Unable to parse %s: unknown line '%s'" % (path, line))
170 # If a specific testcase was requested, keep only that one.
177 # ...and a specific output type
178 if op_type is not None:
180 for op in target_tc['output']:
181 if op['type'] == op_type:
182 # ...and a specific output class
183 if op_class is None or ('class' in op and op['class'] == op_class):
184 target_oplist.append(op)
185 DBG("match on [%s]" % str(op))
186 target_tc['output'] = target_oplist
187 if target_tc is None:
192 error = check_tclist(t)
194 ERR("Error in %s: %s" % (path, error))
200 def get_tests(testnames):
202 for testspec in testnames:
203 # Optional testspec in the form i2c/rtc
204 tc = op_type = op_class = None
205 ts = testspec.strip("/").split("/")
213 path = os.path.join(decoders_dir, pd)
214 if not os.path.isdir(path):
215 # User specified non-existent PD
216 raise Exception("%s not found." % path)
217 path = os.path.join(decoders_dir, pd, "test/test.conf")
218 if not os.path.exists(path):
219 # PD doesn't have any tests yet
221 tests.append(parse_testfile(path, pd, tc, op_type, op_class))
226 def diff_text(f1, f2):
227 t1 = open(f1).readlines()
228 t2 = open(f2).readlines()
231 for line in d.compare(t1, t2):
232 if line[:2] in ('- ', '+ '):
233 diff.append(line.strip())
238 def compare_binary(f1, f2):
240 h1.update(open(f1, 'rb').read())
242 h2.update(open(f2, 'rb').read())
243 if h1.digest() == h2.digest():
246 result = ["Binary output does not match."]
251 def run_tests(tests, fix=False):
254 cmd = os.path.join(tests_dir, 'runtc')
260 for pd in tc['pdlist']:
261 args.extend(['-P', pd['name']])
262 for label, probe in pd['probes']:
263 args.extend(['-p', "%s=%d" % (label, probe)])
264 for option, value in pd['options']:
265 args.extend(['-o', "%s=%s" % (option, value)])
266 args.extend(['-i', os.path.join(dumps_dir, tc['input'])])
267 for op in tc['output']:
268 name = "%s/%s/%s" % (tc['pd'], tc['name'], op['type'])
269 opargs = ['-O', "%s:%s" % (op['pd'], op['type'])]
271 opargs[-1] += ":%s" % op['class']
272 name += "/%s" % op['class']
274 dots = '.' * (60 - len(name) - 2)
275 INFO("%s %s " % (name, dots), end='')
280 fd, outfile = mkstemp()
282 opargs.extend(['-f', outfile])
283 DBG("Running %s" % (' '.join(args + opargs)))
284 p = Popen(args + opargs, stdout=PIPE, stderr=PIPE)
285 stdout, stderr = p.communicate()
287 results[-1]['statistics'] = stdout.decode('utf-8').strip()
289 results[-1]['error'] = stderr.decode('utf-8').strip()
291 elif p.returncode != 0:
292 # runtc indicated an error, but didn't output a
293 # message on stderr about it
294 results[-1]['error'] = "Unknown error: runtc %d" % p.returncode
295 if 'error' not in results[-1]:
296 match = os.path.join(decoders_dir, op['pd'], 'test', op['match'])
298 diff = diff_error = None
299 if op['type'] in ('annotation', 'python'):
300 diff = diff_text(match, outfile)
301 elif op['type'] == 'binary':
302 diff = compare_binary(match, outfile)
304 diff = ["Unsupported output type '%s'." % op['type']]
305 except Exception as e:
308 if diff or diff_error:
310 DBG("Wrote %s" % match)
313 results[-1]['diff'] = diff
314 elif diff_error is not None:
316 except Exception as e:
317 results[-1]['error'] = str(e)
321 if 'diff' in results[-1]:
322 INFO("Output mismatch")
323 elif 'error' in results[-1]:
324 error = results[-1]['error']
326 error = error[:17] + '...'
330 gen_report(results[-1])
332 return results, errors
335 def gen_report(result):
337 if 'error' in result:
339 out.append(result['error'])
342 out.append("Test output mismatch:")
343 out.extend(result['diff'])
345 if 'statistics' in result:
346 out.extend(["Statistics:", result['statistics']])
350 text = "Testcase: %s\n" % result['testcase']
351 text += '\n'.join(out)
356 filename = result['testcase'].replace('/', '_')
357 open(os.path.join(report_dir, filename), 'w').write(text)
362 def show_tests(tests):
365 print("Testcase: %s/%s" % (tc['pd'], tc['name']))
366 for pd in tc['pdlist']:
367 print(" Protocol decoder: %s" % pd['name'])
368 for label, probe in pd['probes']:
369 print(" Probe %s=%d" % (label, probe))
370 for option, value in pd['options']:
371 print(" Option %s=%d" % (option, value))
373 print(" Stack: %s" % ' '.join(tc['stack']))
374 print(" Input: %s" % tc['input'])
375 for op in tc['output']:
376 print(" Output:\n Protocol decoder: %s" % op['pd'])
377 print(" Type: %s" % op['type'])
379 print(" Class: %s" % op['class'])
380 print(" Match: %s" % op['match'])
384 def list_tests(tests):
387 for op in tc['output']:
388 line = "%s/%s/%s" % (tc['pd'], tc['name'], op['type'])
390 line += "/%s" % op['class']
399 tests_dir = os.path.abspath(os.path.dirname(sys.argv[0]))
400 base_dir = os.path.abspath(os.path.join(os.curdir, tests_dir, os.path.pardir))
401 dumps_dir = os.path.abspath(os.path.join(base_dir, os.path.pardir, 'sigrok-dumps'))
402 decoders_dir = os.path.abspath(os.path.join(base_dir, 'decoders'))
404 if len(sys.argv) == 1:
407 opt_all = opt_run = opt_show = opt_list = opt_fix = False
409 opts, args = getopt(sys.argv[1:], "dvarslfR:S:")
410 for opt, arg in opts:
430 if opt_run and opt_show:
431 usage("Use either -s or -r, not both.")
433 usage("Specify either -a or tests, not both.")
434 if report_dir is not None and not os.path.isdir(report_dir):
435 usage("%s is not a directory" % report_dir)
440 testlist = get_tests(args)
442 testlist = get_tests(os.listdir(decoders_dir))
444 usage("Specify either -a or tests.")
447 if not os.path.isdir(dumps_dir):
448 ERR("Could not find sigrok-dumps repository at %s" % dumps_dir)
450 results, errors = run_tests(testlist)
457 run_tests(testlist, fix=True)
460 except Exception as e:
461 print("Error: %s" % str(e))