]>
Commit | Line | Data |
---|---|---|
1 | #!/usr/bin/env python3 | |
2 | ## | |
3 | ## This file is part of the sigrok-test project. | |
4 | ## | |
5 | ## Copyright (C) 2013 Bert Vermeulen <bert@biot.com> | |
6 | ## | |
7 | ## This program is free software: you can redistribute it and/or modify | |
8 | ## it under the terms of the GNU General Public License as published by | |
9 | ## the Free Software Foundation, either version 3 of the License, or | |
10 | ## (at your option) any later version. | |
11 | ## | |
12 | ## This program is distributed in the hope that it will be useful, | |
13 | ## but WITHOUT ANY WARRANTY; without even the implied warranty of | |
14 | ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
15 | ## GNU General Public License for more details. | |
16 | ## | |
17 | ## You should have received a copy of the GNU General Public License | |
18 | ## along with this program. If not, see <http://www.gnu.org/licenses/>. | |
19 | ## | |
20 | ||
21 | import os | |
22 | import sys | |
23 | import re | |
24 | from getopt import getopt | |
25 | from tempfile import mkstemp | |
26 | from subprocess import Popen, PIPE | |
27 | from difflib import unified_diff | |
28 | from hashlib import md5 | |
29 | from shutil import copy | |
30 | ||
31 | DEBUG = 0 | |
32 | VERBOSE = False | |
33 | ||
34 | ||
35 | class E_syntax(Exception): | |
36 | pass | |
37 | class E_badline(Exception): | |
38 | pass | |
39 | ||
40 | def INFO(msg, end='\n'): | |
41 | if VERBOSE: | |
42 | print(msg, end=end) | |
43 | sys.stdout.flush() | |
44 | ||
45 | ||
46 | def DBG(msg): | |
47 | if DEBUG: | |
48 | print(msg) | |
49 | ||
50 | ||
51 | def ERR(msg): | |
52 | print(msg, file=sys.stderr) | |
53 | ||
54 | ||
55 | def usage(msg=None): | |
56 | if msg: | |
57 | print(msg.strip() + '\n') | |
58 | print("""Usage: testpd [-dvalsrfcR] [<test1> <test2> ...] | |
59 | -d Turn on debugging | |
60 | -v Verbose | |
61 | -a All tests | |
62 | -l List test(s) | |
63 | -s Show test(s) | |
64 | -r Run test(s) | |
65 | -f Fix failed test(s) / create initial output for new test(s) | |
66 | -c Report decoder code coverage | |
67 | -R <directory> Save test reports to <directory> | |
68 | <test> Protocol decoder name ("i2c") and optionally test name ("i2c/rtc")""") | |
69 | sys.exit() | |
70 | ||
71 | ||
72 | def check_tclist(tc): | |
73 | if 'pdlist' not in tc or not tc['pdlist']: | |
74 | return("No protocol decoders") | |
75 | if 'input' not in tc or not tc['input']: | |
76 | return("No input") | |
77 | if 'output' not in tc or not tc['output']: | |
78 | return("No output") | |
79 | for op in tc['output']: | |
80 | if 'match' not in op: | |
81 | return("No match in output") | |
82 | ||
83 | return None | |
84 | ||
85 | ||
86 | def parse_testfile(path, pd, tc, op_type, op_class): | |
87 | DBG("Opening '%s'" % path) | |
88 | tclist = [] | |
89 | for line in open(path).read().split('\n'): | |
90 | try: | |
91 | line = line.strip() | |
92 | if len(line) == 0 or line[0] == "#": | |
93 | continue | |
94 | f = line.split() | |
95 | if not tclist and f[0] != "test": | |
96 | # That can't be good. | |
97 | raise E_badline | |
98 | key = f.pop(0) | |
99 | if key == 'test': | |
100 | if len(f) != 1: | |
101 | raise E_syntax | |
102 | # new testcase | |
103 | tclist.append({ | |
104 | 'pd': pd, | |
105 | 'name': f[0], | |
106 | 'pdlist': [], | |
107 | 'output': [], | |
108 | }) | |
109 | elif key == 'protocol-decoder': | |
110 | if len(f) < 1: | |
111 | raise E_syntax | |
112 | pd_spec = { | |
113 | 'name': f.pop(0), | |
114 | 'channels': [], | |
115 | 'options': [], | |
116 | 'initial_pins': [], | |
117 | } | |
118 | while len(f): | |
119 | if len(f) == 1: | |
120 | # Always needs <key> <value> | |
121 | raise E_syntax | |
122 | a, b = f[:2] | |
123 | f = f[2:] | |
124 | if '=' not in b: | |
125 | raise E_syntax | |
126 | opt, val = b.split('=') | |
127 | if a == 'channel': | |
128 | try: | |
129 | val = int(val) | |
130 | except: | |
131 | raise E_syntax | |
132 | pd_spec['channels'].append([opt, val]) | |
133 | elif a == 'option': | |
134 | pd_spec['options'].append([opt, val]) | |
135 | elif a == 'initial_pin': | |
136 | try: | |
137 | val = int(val) | |
138 | except: | |
139 | raise E_syntax | |
140 | pd_spec['initial_pins'].append([opt, val]) | |
141 | else: | |
142 | raise E_syntax | |
143 | tclist[-1]['pdlist'].append(pd_spec) | |
144 | elif key == 'stack': | |
145 | if len(f) < 2: | |
146 | raise E_syntax | |
147 | tclist[-1]['stack'] = f | |
148 | elif key == 'input': | |
149 | if len(f) < 1: | |
150 | raise E_syntax | |
151 | input_spec = { | |
152 | 'name': f.pop(0), | |
153 | 'format': None, | |
154 | 'options': [], | |
155 | } | |
156 | while len(f): | |
157 | if len(f) < 2: | |
158 | # Always needs <key> <value> | |
159 | raise E_syntax | |
160 | a, b = f[:2] | |
161 | f = f[2:] | |
162 | if a == 'format': | |
163 | input_spec['format'] = b | |
164 | elif a == 'option': | |
165 | input_spec['options'].append(b) | |
166 | else: | |
167 | raise E_syntax | |
168 | tclist[-1]['input'] = input_spec | |
169 | elif key == 'output': | |
170 | op_spec = { | |
171 | 'pd': f.pop(0), | |
172 | 'type': f.pop(0), | |
173 | } | |
174 | while len(f): | |
175 | if len(f) == 1: | |
176 | # Always needs <key> <value> | |
177 | raise E_syntax | |
178 | a, b = f[:2] | |
179 | f = f[2:] | |
180 | if a == 'class': | |
181 | op_spec['class'] = b | |
182 | elif a == 'match': | |
183 | op_spec['match'] = b | |
184 | else: | |
185 | raise E_syntax | |
186 | tclist[-1]['output'].append(op_spec) | |
187 | else: | |
188 | raise E_badline | |
189 | except E_badline as e: | |
190 | ERR("Invalid syntax in %s: line '%s'" % (path, line)) | |
191 | return [] | |
192 | except E_syntax as e: | |
193 | ERR("Unable to parse %s: unknown line '%s'" % (path, line)) | |
194 | return [] | |
195 | ||
196 | # If a specific testcase was requested, keep only that one. | |
197 | if tc is not None: | |
198 | target_tc = None | |
199 | for t in tclist: | |
200 | if t['name'] == tc: | |
201 | target_tc = t | |
202 | break | |
203 | # ...and a specific output type | |
204 | if op_type is not None: | |
205 | target_oplist = [] | |
206 | for op in target_tc['output']: | |
207 | if op['type'] == op_type: | |
208 | # ...and a specific output class | |
209 | if op_class is None or ('class' in op and op['class'] == op_class): | |
210 | target_oplist.append(op) | |
211 | DBG("match on [%s]" % str(op)) | |
212 | target_tc['output'] = target_oplist | |
213 | if target_tc is None: | |
214 | tclist = [] | |
215 | else: | |
216 | tclist = [target_tc] | |
217 | for t in tclist: | |
218 | error = check_tclist(t) | |
219 | if error: | |
220 | ERR("Error in %s: %s" % (path, error)) | |
221 | return [] | |
222 | ||
223 | return tclist | |
224 | ||
225 | ||
226 | def get_tests(testnames): | |
227 | tests = {} | |
228 | for testspec in testnames: | |
229 | # Optional testspec in the form pd/testcase/type/class | |
230 | tc = op_type = op_class = None | |
231 | ts = testspec.strip("/").split("/") | |
232 | pd = ts.pop(0) | |
233 | tests[pd] = [] | |
234 | if ts: | |
235 | tc = ts.pop(0) | |
236 | if ts: | |
237 | op_type = ts.pop(0) | |
238 | if ts: | |
239 | op_class = ts.pop(0) | |
240 | path = os.path.join(tests_dir, pd) | |
241 | if not os.path.isdir(path): | |
242 | # User specified non-existent PD | |
243 | raise Exception("%s not found." % path) | |
244 | path = os.path.join(tests_dir, pd, "test.conf") | |
245 | if not os.path.exists(path): | |
246 | # PD doesn't have any tests yet | |
247 | continue | |
248 | tests[pd].append(parse_testfile(path, pd, tc, op_type, op_class)) | |
249 | ||
250 | return tests | |
251 | ||
252 | ||
253 | def diff_text(f1, f2): | |
254 | t1 = open(f1).readlines() | |
255 | t2 = open(f2).readlines() | |
256 | diff = list(unified_diff(t1, t2)) | |
257 | diff = diff[2:] # Strip two from/to filename lines with "+++"/"---". | |
258 | diff = [d.strip() for d in diff if d[0] in ('+', '-')] | |
259 | return diff | |
260 | ||
261 | ||
262 | def compare_binary(f1, f2): | |
263 | h1 = md5() | |
264 | h1.update(open(f1, 'rb').read()) | |
265 | h2 = md5() | |
266 | h2.update(open(f2, 'rb').read()) | |
267 | if h1.digest() == h2.digest(): | |
268 | result = None | |
269 | else: | |
270 | result = ["Binary output does not match."] | |
271 | ||
272 | return result | |
273 | ||
274 | ||
275 | # runtc's stdout can have lines like: | |
276 | # coverage: lines=161 missed=2 coverage=99% | |
277 | def parse_stats(text): | |
278 | stats = {} | |
279 | for line in text.strip().split('\n'): | |
280 | fields = line.split() | |
281 | key = fields.pop(0).strip(':') | |
282 | if key not in stats: | |
283 | stats[key] = [] | |
284 | stats[key].append({}) | |
285 | for f in fields: | |
286 | k, v = f.split('=') | |
287 | stats[key][-1][k] = v | |
288 | ||
289 | return stats | |
290 | ||
291 | ||
292 | # take result set of all tests in a PD, and summarize which lines | |
293 | # were not covered by any of the tests. | |
294 | def coverage_sum(cvglist): | |
295 | lines = 0 | |
296 | missed = 0 | |
297 | missed_lines = {} | |
298 | for record in cvglist: | |
299 | lines = int(record['lines']) | |
300 | missed += int(record['missed']) | |
301 | if 'missed_lines' not in record: | |
302 | continue | |
303 | for linespec in record['missed_lines'].split(','): | |
304 | if linespec not in missed_lines: | |
305 | missed_lines[linespec] = 1 | |
306 | else: | |
307 | missed_lines[linespec] += 1 | |
308 | ||
309 | # keep only those lines that didn't show up in every non-summary record | |
310 | final_missed = [] | |
311 | for linespec in missed_lines: | |
312 | if missed_lines[linespec] != len(cvglist): | |
313 | continue | |
314 | final_missed.append(linespec) | |
315 | ||
316 | return lines, final_missed | |
317 | ||
318 | ||
319 | def run_tests(tests, fix=False): | |
320 | errors = 0 | |
321 | results = [] | |
322 | cmd = [os.path.join(runtc_dir, 'runtc')] | |
323 | if opt_coverage: | |
324 | fd, coverage = mkstemp() | |
325 | os.close(fd) | |
326 | cmd.extend(['-c', coverage]) | |
327 | else: | |
328 | coverage = None | |
329 | for pd in sorted(tests.keys()): | |
330 | pd_cvg = [] | |
331 | for tclist in tests[pd]: | |
332 | for tc in tclist: | |
333 | args = cmd[:] | |
334 | if DEBUG > 1: | |
335 | args.append('-d') | |
336 | # Set up PD stack for this test. | |
337 | for spd in tc['pdlist']: | |
338 | args.extend(['-P', spd['name']]) | |
339 | for label, channel in spd['channels']: | |
340 | args.extend(['-p', "%s=%d" % (label, channel)]) | |
341 | for option, value in spd['options']: | |
342 | args.extend(['-o', "%s=%s" % (option, value)]) | |
343 | for label, initial_pin in spd['initial_pins']: | |
344 | args.extend(['-N', "%s=%d" % (label, initial_pin)]) | |
345 | # Setup input spec for this test (optional format spec). | |
346 | in_spec = tc['input'] | |
347 | infile = os.path.join(dumps_dir, in_spec['name']) | |
348 | args.extend(['-i', infile]) | |
349 | if in_spec['format']: | |
350 | args.extend(['-I', in_spec['format']]) | |
351 | for opt in in_spec['options']: | |
352 | args.extend(['-I', opt]) | |
353 | # Setup output spec for this test. | |
354 | for op in tc['output']: | |
355 | name = "%s/%s/%s" % (pd, tc['name'], op['type']) | |
356 | opargs = ['-O', "%s:%s" % (op['pd'], op['type'])] | |
357 | if 'class' in op: | |
358 | opargs[-1] += ":%s" % op['class'] | |
359 | name += "/%s" % op['class'] | |
360 | if VERBOSE: | |
361 | dots = '.' * (77 - len(name) - 2) | |
362 | INFO("%s %s " % (name, dots), end='') | |
363 | results.append({ | |
364 | 'testcase': name, | |
365 | }) | |
366 | try: | |
367 | fd, outfile = mkstemp() | |
368 | os.close(fd) | |
369 | opargs.extend(['-f', outfile]) | |
370 | DBG("Running %s" % (' '.join(args + opargs))) | |
371 | p = Popen(args + opargs, stdout=PIPE, stderr=PIPE) | |
372 | stdout, stderr = p.communicate() | |
373 | if stdout: | |
374 | # statistics and coverage data on stdout | |
375 | results[-1].update(parse_stats(stdout.decode('utf-8'))) | |
376 | if stderr: | |
377 | results[-1]['error'] = stderr.decode('utf-8').strip() | |
378 | errors += 1 | |
379 | elif p.returncode != 0: | |
380 | # runtc indicated an error, but didn't output a | |
381 | # message on stderr about it | |
382 | results[-1]['error'] = "Unknown error: runtc %d" % p.returncode | |
383 | if 'error' not in results[-1]: | |
384 | matchfile = os.path.join(tests_dir, op['pd'], op['match']) | |
385 | DBG("Comparing with %s" % matchfile) | |
386 | try: | |
387 | diff = diff_error = None | |
388 | if op['type'] in ('annotation', 'python'): | |
389 | diff = diff_text(matchfile, outfile) | |
390 | elif op['type'] == 'binary': | |
391 | diff = compare_binary(matchfile, outfile) | |
392 | else: | |
393 | diff = ["Unsupported output type '%s'." % op['type']] | |
394 | except Exception as e: | |
395 | diff_error = e | |
396 | if fix: | |
397 | if diff or diff_error: | |
398 | copy(outfile, matchfile) | |
399 | DBG("Wrote %s" % matchfile) | |
400 | else: | |
401 | if diff: | |
402 | results[-1]['diff'] = diff | |
403 | elif diff_error is not None: | |
404 | raise diff_error | |
405 | except Exception as e: | |
406 | results[-1]['error'] = str(e) | |
407 | finally: | |
408 | if coverage: | |
409 | results[-1]['coverage_report'] = coverage | |
410 | os.unlink(outfile) | |
411 | if op['type'] == 'exception' and 'error' in results[-1]: | |
412 | # filter out the exception we were looking for | |
413 | reg = "^Error: srd: %s:" % op['match'] | |
414 | if re.match(reg, results[-1]['error']): | |
415 | # found it, not an error | |
416 | results[-1].pop('error') | |
417 | errors -= 1 | |
418 | if VERBOSE: | |
419 | if 'diff' in results[-1]: | |
420 | INFO("Output mismatch") | |
421 | elif 'error' in results[-1]: | |
422 | error = results[-1]['error'] | |
423 | if len(error) > 20: | |
424 | error = error[:17] + '...' | |
425 | INFO(error) | |
426 | elif 'coverage' in results[-1]: | |
427 | # report coverage of this PD | |
428 | for record in results[-1]['coverage']: | |
429 | # but not others used in the stack | |
430 | # as part of the test. | |
431 | if record['scope'] == pd: | |
432 | INFO(record['coverage']) | |
433 | break | |
434 | else: | |
435 | INFO("OK") | |
436 | gen_report(results[-1]) | |
437 | if coverage: | |
438 | os.unlink(coverage) | |
439 | # only keep track of coverage records for this PD, | |
440 | # not others in the stack just used for testing. | |
441 | for cvg in results[-1]['coverage']: | |
442 | if cvg['scope'] == pd: | |
443 | pd_cvg.append(cvg) | |
444 | if opt_coverage and len(pd_cvg) > 1: | |
445 | # report total coverage of this PD, across all the tests | |
446 | # that were done on it. | |
447 | total_lines, missed_lines = coverage_sum(pd_cvg) | |
448 | pd_coverage = 100 - (float(len(missed_lines)) / total_lines * 100) | |
449 | if VERBOSE: | |
450 | dots = '.' * (54 - len(pd) - 2) | |
451 | INFO("%s total %s %d%%" % (pd, dots, pd_coverage)) | |
452 | if report_dir: | |
453 | # generate a missing lines list across all the files in | |
454 | # the PD | |
455 | files = {} | |
456 | for entry in missed_lines: | |
457 | filename, line = entry.split(':') | |
458 | if filename not in files: | |
459 | files[filename] = [] | |
460 | files[filename].append(line) | |
461 | text = '' | |
462 | for filename in sorted(files.keys()): | |
463 | line_list = ','.join(sorted(files[filename], key=int)) | |
464 | text += "%s: %s\n" % (filename, line_list) | |
465 | open(os.path.join(report_dir, pd + "_total"), 'w').write(text) | |
466 | ||
467 | ||
468 | return results, errors | |
469 | ||
470 | def get_run_tests_error_diff_counts(results): | |
471 | """Get error and diff counters from run_tests() results.""" | |
472 | errs = 0 | |
473 | diffs = 0 | |
474 | for result in results: | |
475 | if 'error' in result: | |
476 | errs += 1 | |
477 | if 'diff' in result: | |
478 | diffs += 1 | |
479 | return errs, diffs | |
480 | ||
481 | ||
482 | def gen_report(result): | |
483 | out = [] | |
484 | if 'error' in result: | |
485 | out.append("Error:") | |
486 | out.append(result['error']) | |
487 | out.append('') | |
488 | if 'diff' in result: | |
489 | out.append("Test output mismatch:") | |
490 | out.extend(result['diff']) | |
491 | out.append('') | |
492 | if 'coverage_report' in result: | |
493 | out.append(open(result['coverage_report'], 'r').read()) | |
494 | out.append('') | |
495 | ||
496 | if out: | |
497 | text = "Testcase: %s\n" % result['testcase'] | |
498 | text += '\n'.join(out) | |
499 | else: | |
500 | return | |
501 | ||
502 | if report_dir: | |
503 | filename = result['testcase'].replace('/', '_') | |
504 | open(os.path.join(report_dir, filename), 'w').write(text) | |
505 | else: | |
506 | print(text) | |
507 | ||
508 | ||
509 | def show_tests(tests): | |
510 | for pd in sorted(tests.keys()): | |
511 | for tclist in tests[pd]: | |
512 | for tc in tclist: | |
513 | print("Testcase: %s/%s" % (tc['pd'], tc['name'])) | |
514 | for pd in tc['pdlist']: | |
515 | print(" Protocol decoder: %s" % pd['name']) | |
516 | for label, channel in pd['channels']: | |
517 | print(" Channel %s=%d" % (label, channel)) | |
518 | for option, value in pd['options']: | |
519 | print(" Option %s=%s" % (option, value)) | |
520 | for label, initial_pin in pd['initial_pins']: | |
521 | print(" Initial pin %s=%d" % (label, initial_pin)) | |
522 | if 'stack' in tc: | |
523 | print(" Stack: %s" % ' '.join(tc['stack'])) | |
524 | print(" Input: %s" % tc['input']) | |
525 | for op in tc['output']: | |
526 | print(" Output:\n Protocol decoder: %s" % op['pd']) | |
527 | print(" Type: %s" % op['type']) | |
528 | if 'class' in op: | |
529 | print(" Class: %s" % op['class']) | |
530 | print(" Match: %s" % op['match']) | |
531 | print() | |
532 | ||
533 | ||
534 | def list_tests(tests): | |
535 | for pd in sorted(tests.keys()): | |
536 | for tclist in tests[pd]: | |
537 | for tc in tclist: | |
538 | for op in tc['output']: | |
539 | line = "%s/%s/%s" % (tc['pd'], tc['name'], op['type']) | |
540 | if 'class' in op: | |
541 | line += "/%s" % op['class'] | |
542 | print(line) | |
543 | ||
544 | ||
545 | # | |
546 | # main | |
547 | # | |
548 | ||
549 | # project root | |
550 | runtc_dir = os.path.abspath(os.path.dirname(sys.argv[0])) | |
551 | base_dir = os.path.abspath(os.path.join(os.curdir, runtc_dir, os.path.pardir)) | |
552 | dumps_dir = os.path.abspath(os.path.join(base_dir, os.path.pardir, 'sigrok-dumps')) | |
553 | tests_dir = os.path.abspath(os.path.join(runtc_dir, 'test')) | |
554 | ||
555 | if len(sys.argv) == 1: | |
556 | usage() | |
557 | ||
558 | opt_all = opt_run = opt_show = opt_list = opt_fix = opt_coverage = False | |
559 | report_dir = None | |
560 | try: | |
561 | opts, args = getopt(sys.argv[1:], "dvarslfcR:S:") | |
562 | except Exception as e: | |
563 | usage('error while parsing command line arguments: {}'.format(e)) | |
564 | for opt, arg in opts: | |
565 | if opt == '-d': | |
566 | DEBUG += 1 | |
567 | if opt == '-v': | |
568 | VERBOSE = True | |
569 | elif opt == '-a': | |
570 | opt_all = True | |
571 | elif opt == '-r': | |
572 | opt_run = True | |
573 | elif opt == '-s': | |
574 | opt_show = True | |
575 | elif opt == '-l': | |
576 | opt_list = True | |
577 | elif opt == '-f': | |
578 | opt_fix = True | |
579 | elif opt == '-c': | |
580 | opt_coverage = True | |
581 | elif opt == '-R': | |
582 | report_dir = arg | |
583 | elif opt == '-S': | |
584 | dumps_dir = arg | |
585 | ||
586 | if opt_run and opt_show: | |
587 | usage("Use either -s or -r, not both.") | |
588 | if args and opt_all: | |
589 | usage("Specify either -a or tests, not both.") | |
590 | if report_dir is not None and not os.path.isdir(report_dir): | |
591 | usage("%s is not a directory" % report_dir) | |
592 | ||
593 | ret = 0 | |
594 | try: | |
595 | if args: | |
596 | testlist = get_tests(args) | |
597 | elif opt_all or opt_list: | |
598 | testlist = get_tests(os.listdir(tests_dir)) | |
599 | else: | |
600 | usage("Specify either -a or tests.") | |
601 | ||
602 | if opt_run: | |
603 | if not os.path.isdir(dumps_dir): | |
604 | ERR("Could not find sigrok-dumps repository at %s" % dumps_dir) | |
605 | sys.exit(1) | |
606 | results, errors = run_tests(testlist, fix=opt_fix) | |
607 | ret = 0 | |
608 | errs, diffs = get_run_tests_error_diff_counts(results) | |
609 | if errs: | |
610 | ret = 1 | |
611 | elif diffs: | |
612 | ret = 2 | |
613 | elif opt_show: | |
614 | show_tests(testlist) | |
615 | elif opt_list: | |
616 | list_tests(testlist) | |
617 | elif opt_fix: | |
618 | run_tests(testlist, fix=True) | |
619 | else: | |
620 | usage() | |
621 | except Exception as e: | |
622 | print("Error: %s" % str(e)) | |
623 | if DEBUG: | |
624 | raise | |
625 | ||
626 | sys.exit(ret) |