2 * This file is part of the sigrok-test project.
4 * Copyright (C) 2013 Bert Vermeulen <bert@biot.com>
6 * This program is free software: you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation, either version 3 of the License, or
9 * (at your option) any later version.
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
16 * You should have received a copy of the GNU General Public License
17 * along with this program. If not, see <http://www.gnu.org/licenses/>.
23 #include <libsigrokdecode/libsigrokdecode.h>
24 #include <libsigrok/libsigrok.h>
30 #include <sys/types.h>
35 #include <sys/resource.h>
42 static int debug = FALSE;
43 static int statistics = FALSE;
44 static char *coverage_report;
45 static struct sr_context *ctx;
57 struct initial_pin_info {
86 static struct cvg *get_mod_cov(PyObject *py_cov, const char *module_name);
87 static void cvg_add(struct cvg *dst, const struct cvg *src);
88 static struct cvg *cvg_new(void);
89 static gboolean find_missed_line(struct cvg *cvg, const char *linespec);
91 static void logmsg(const char *prefix, FILE *out, const char *format, va_list args)
94 fprintf(out, "%s", prefix);
95 vfprintf(out, format, args);
99 static void DBG(const char *format, ...)
105 va_start(args, format);
106 logmsg("DBG: runtc: ", stdout, format, args);
110 static void ERR(const char *format, ...)
114 va_start(args, format);
115 logmsg("Error: ", stderr, format, args);
119 static int sr_log(void *cb_data, int loglevel, const char *format, va_list args)
123 if (loglevel == SR_LOG_ERR || loglevel == SR_LOG_WARN)
124 logmsg("Error: sr: ", stderr, format, args);
126 logmsg("DBG: sr: ", stdout, format, args);
131 static int srd_log(void *cb_data, int loglevel, const char *format, va_list args)
135 if (loglevel == SRD_LOG_ERR || loglevel == SRD_LOG_WARN)
136 logmsg("Error: srd: ", stderr, format, args);
138 logmsg("DBG: srd: ", stdout, format, args);
143 static void usage(const char *msg)
146 fprintf(stderr, "%s\n", msg);
148 printf("Usage: runtc [-dPpoiOfcS]\n");
149 printf(" -d (enables debug output)\n");
150 printf(" -P <protocol decoder>\n");
151 printf(" -p <channelname=channelnum> (optional)\n");
152 printf(" -o <channeloption=value> (optional)\n");
153 printf(" -N <channelname=initial-pin-value> (optional)\n");
154 printf(" -i <input file>\n");
155 printf(" -O <output-pd:output-type[:output-class]>\n");
156 printf(" -f <output file> (optional)\n");
157 printf(" -c <coverage report> (optional)\n");
158 printf(" -S (enables statistics)\n");
164 * This is a neutered version of libsigrokdecode's py_str_as_str(). It
165 * does no error checking, but then the only strings it processes are
166 * generated by Python's repr(), so are known good.
168 static char *py_str_as_str(const PyObject *py_str)
173 py_encstr = PyUnicode_AsEncodedString((PyObject *)py_str, "utf-8", NULL);
174 str = PyBytes_AS_STRING(py_encstr);
175 outstr = g_strdup(str);
176 Py_DecRef(py_encstr);
182 * The following routines are callbacks for libsigrokdecode. They receive
183 * output from protocol decoders, optionally dropping data to only forward
184 * a selected decoder's or class' information. Output is written to either
185 * a specified file or stdout, an external process will compare captured
186 * output against expectations.
188 * Note that runtc(1) output emits the decoder "class" name instead of the
189 * instance name. So that generated output remains compatible with existing
190 * .output files which hold expected output of test cases. Without this
191 * approach, developers had to "anticipate" instance names from test.conf
192 * setups (and knowledge about internal implementation details of the srd
193 * library), and adjust .output files to reflect those names. Or specify
194 * instance names in each and every test.conf description (-o inst_id=ID).
196 * It's assumed that runtc(1) is used to check stacked decoders, but not
197 * multiple stacks in parallel and no stacks with multiple instances of
198 * decoders of the same type. When such configurations become desirable,
199 * runtc(1) needs to emit the instance name, and test configurations and
200 * output expectations need adjustment.
203 static void srd_cb_py(struct srd_proto_data *pdata, void *cb_data)
206 PyObject *pydata, *pyrepr;
210 DBG("Python output from %s", pdata->pdo->di->inst_id);
212 pydata = pdata->data;
213 DBG("ptr %p", pydata);
215 if (strcmp(pdata->pdo->di->inst_id, op->pd_id))
216 /* This is not the PD selected for output. */
219 if (!(pyrepr = PyObject_Repr(pydata))) {
220 ERR("Invalid Python object.");
223 s = py_str_as_str(pyrepr);
226 /* Output format for testing is '<ss>-<es> <decoder-id>: <repr>\n'. */
227 out = g_string_sized_new(128);
228 g_string_printf(out, "%" PRIu64 "-%" PRIu64 " %s: %s\n",
229 pdata->start_sample, pdata->end_sample,
230 pdata->pdo->di->decoder->id, s);
232 if (write(op->outfd, out->str, out->len) == -1)
233 ERR("SRD_OUTPUT_PYTHON callback write failure!");
234 DBG("wrote '%s'", out->str);
235 g_string_free(out, TRUE);
239 static void srd_cb_bin(struct srd_proto_data *pdata, void *cb_data)
241 struct srd_proto_data_binary *pdb;
246 DBG("Binary output from %s", pdata->pdo->di->inst_id);
250 if (strcmp(pdata->pdo->di->inst_id, op->pd_id))
251 /* This is not the PD selected for output. */
254 if (op->class_idx != -1 && op->class_idx != pdb->bin_class)
256 * This output takes a specific binary class,
257 * but not the one that just came in.
261 out = g_string_sized_new(128);
262 g_string_printf(out, "%" PRIu64 "-%" PRIu64 " %s:",
263 pdata->start_sample, pdata->end_sample,
264 pdata->pdo->di->decoder->id);
265 for (i = 0; i < pdb->size; i++) {
266 g_string_append_printf(out, " %.2x", pdb->data[i]);
268 g_string_append(out, "\n");
269 if (write(op->outfd, out->str, out->len) == -1)
270 ERR("SRD_OUTPUT_BINARY callback write failure!");
274 static void srd_cb_ann(struct srd_proto_data *pdata, void *cb_data)
276 struct srd_decoder_inst *di;
277 struct srd_decoder *dec;
278 struct srd_proto_data_annotation *pda;
285 * Only inspect received annotations when they originate from
286 * the selected protocol decoder, and an optionally specified
287 * annotation class matches the received data.
293 DBG("Annotation output from %s", di->inst_id);
294 if (strcmp(di->inst_id, op->pd_id))
295 /* This is not the PD selected for output. */
298 if (op->class_idx != -1 && op->class_idx != pda->ann_class)
300 * This output takes a specific annotation class,
301 * but not the one that just came in.
306 * Print the annotation information in textual representation
307 * to the specified output file. Prefix the annotation strings
308 * with the start and end sample number, the decoder name, and
309 * the annotation name.
311 dec_ann = g_slist_nth_data(dec->annotations, pda->ann_class);
312 line = g_string_sized_new(256);
313 g_string_printf(line, "%" PRIu64 "-%" PRIu64 " %s: %s:",
314 pdata->start_sample, pdata->end_sample,
315 dec->id, dec_ann[0]);
316 for (i = 0; pda->ann_text[i]; i++)
317 g_string_append_printf(line, " \"%s\"", pda->ann_text[i]);
318 g_string_append(line, "\n");
319 if (write(op->outfd, line->str, line->len) == -1)
320 ERR("SRD_OUTPUT_ANN callback write failure!");
321 g_string_free(line, TRUE);
325 static void sr_cb(const struct sr_dev_inst *sdi,
326 const struct sr_datafeed_packet *packet, void *cb_data)
328 static int samplecnt = 0;
329 const struct sr_datafeed_logic *logic;
330 struct srd_session *sess;
334 struct sr_dev_driver *driver;
338 driver = sr_dev_inst_driver_get(sdi);
340 switch (packet->type) {
342 DBG("Received SR_DF_HEADER");
343 if (sr_config_get(driver, sdi, NULL, SR_CONF_SAMPLERATE,
345 ERR("Getting samplerate failed");
348 samplerate = g_variant_get_uint64(gvar);
349 g_variant_unref(gvar);
350 if (srd_session_metadata_set(sess, SRD_CONF_SAMPLERATE,
351 g_variant_new_uint64(samplerate)) != SRD_OK) {
352 ERR("Setting samplerate failed");
355 if (srd_session_start(sess) != SRD_OK) {
356 ERR("Session start failed");
361 logic = packet->payload;
362 num_samples = logic->length / logic->unitsize;
363 DBG("Received SR_DF_LOGIC (%"PRIu64" bytes, unitsize = %d).",
364 logic->length, logic->unitsize);
365 srd_session_send(sess, samplecnt, samplecnt + num_samples,
366 logic->data, logic->length, logic->unitsize);
367 samplecnt += num_samples;
370 DBG("Received SR_DF_END");
371 #if defined HAVE_SRD_SESSION_SEND_EOF && HAVE_SRD_SESSION_SEND_EOF
372 (void)srd_session_send_eof(sess);
379 static int run_testcase(const char *infile, GSList *pdlist, struct output *op)
381 struct srd_session *sess;
382 struct srd_decoder *dec;
383 struct srd_decoder_inst *di, *prev_di;
384 srd_pd_output_callback cb;
386 struct channel *channel;
387 struct option *option;
389 GHashTable *channels, *opts;
390 GSList *pdl, *l, *l2, *devices;
393 char **decoder_class;
394 struct sr_session *sr_sess;
397 GArray *initial_pins;
398 struct initial_pin_info *initial_pin;
401 if ((op->outfd = open(op->outfile, O_CREAT|O_WRONLY, 0600)) == -1) {
402 ERR("Unable to open %s for writing: %s", op->outfile,
408 if (sr_session_load(ctx, infile, &sr_sess) != SR_OK){
409 ERR("sr_session_load() failed");
413 sr_session_dev_list(sr_sess, &devices);
415 if (srd_session_new(&sess) != SRD_OK) {
416 ERR("srd_session_new() failed");
419 sr_session_datafeed_callback_add(sr_sess, sr_cb, sess);
424 case SRD_OUTPUT_BINARY:
427 case SRD_OUTPUT_PYTHON:
431 ERR("Invalid op->type");
434 srd_pd_output_callback_add(sess, op->type, cb, op);
438 for (pdl = pdlist; pdl; pdl = pdl->next) {
440 if (srd_decoder_load(pd->name) != SRD_OK) {
441 ERR("srd_decoder_load() failed");
445 /* Instantiate decoder and pass in options. */
446 opts = g_hash_table_new_full(g_str_hash, g_str_equal, NULL,
447 (GDestroyNotify)g_variant_unref);
448 for (l = pd->options; l; l = l->next) {
452 s = g_variant_get_string(option->value, NULL);
453 for (i = 0; i < (int)strlen(s); i++) {
459 /* Integer option value */
460 g_hash_table_insert(opts, option->key,
461 g_variant_new_int64(strtoull(s, NULL, 10)));
463 /* String option value */
464 g_hash_table_insert(opts, option->key, option->value);
467 if (!(di = srd_inst_new(sess, pd->name, opts))) {
468 ERR("srd_inst_new() failed");
471 g_hash_table_destroy(opts);
474 * Get (a reference to) the decoder instance's ID if we
475 * are about to receive PD output from it. We need to
476 * filter output that carries the decoder instance's name.
478 if (strcmp(pd->name, op->pd) == 0) {
479 op->pd_id = di->inst_id;
480 DBG("Decoder of type \"%s\" has instance ID \"%s\".",
486 channels = g_hash_table_new_full(g_str_hash, g_str_equal, NULL,
487 (GDestroyNotify)g_variant_unref);
489 for (l = pd->channels; l; l = l->next) {
491 if (channel->channel > max_channel)
492 max_channel = channel->channel;
493 gvar = g_variant_new_int32(channel->channel);
494 g_variant_ref_sink(gvar);
495 g_hash_table_insert(channels, channel->name, gvar);
498 if (srd_inst_channel_set_all(di, channels) != SRD_OK) {
499 ERR("srd_inst_channel_set_all() failed");
502 g_hash_table_destroy(channels);
505 /* Set initial pins. */
506 if (pd->initial_pins) {
507 initial_pins = g_array_sized_new(FALSE, TRUE, sizeof(uint8_t),
508 di->dec_num_channels);
509 g_array_set_size(initial_pins, di->dec_num_channels);
510 memset(initial_pins->data, SRD_INITIAL_PIN_SAME_AS_SAMPLE0,
511 di->dec_num_channels);
513 for (l = pd->channels, idx = 0; l; l = l->next, idx++) {
515 for (l2 = pd->initial_pins; l2; l2 = l2->next) {
516 initial_pin = l2->data;
517 if (!strcmp(initial_pin->name, channel->name))
518 initial_pins->data[idx] = initial_pin->value;
522 if (srd_inst_initial_pins_set_all(di, initial_pins) != SRD_OK) {
523 ERR("srd_inst_initial_pins_set_all() failed");
526 g_array_free(initial_pins, TRUE);
530 * If this is not the first decoder in the list, stack it
531 * on top of the previous one.
534 if (srd_inst_stack(sess, prev_di, di) != SRD_OK) {
535 ERR("Failed to stack decoder instances.");
542 * Bail out if we haven't created an instance of the selected
543 * decoder type of which we shall grab output data from.
546 ERR("No / invalid decoder");
550 /* Resolve selected decoder's class index, so we can match. */
551 dec = srd_decoder_get_by_id(pd->name);
553 if (op->type == SRD_OUTPUT_ANN)
554 l = dec->annotations;
555 else if (op->type == SRD_OUTPUT_BINARY)
558 /* Only annotations and binary can have a class. */
559 ERR("Invalid decoder class");
564 decoder_class = l->data;
565 if (!strcmp(decoder_class[0], op->class_)) {
572 if (op->class_idx == -1) {
573 ERR("Output class '%s' not found in decoder %s.",
574 op->class_, pd->name);
577 DBG("Class %s index is %d", op->class_, op->class_idx);
580 sr_session_start(sr_sess);
581 sr_session_run(sr_sess);
582 sr_session_stop(sr_sess);
584 srd_session_destroy(sess);
592 static PyObject *start_coverage(GSList *pdlist)
594 PyObject *py_mod, *py_pdlist, *py_pd, *py_func, *py_args, *py_kwargs, *py_cov;
598 DBG("Starting coverage.");
600 if (!(py_mod = PyImport_ImportModule("coverage")))
603 if (!(py_pdlist = PyList_New(0)))
605 for (l = pdlist; l; l = l->next) {
607 py_pd = PyUnicode_FromFormat("*/%s/*.py", pd->name);
608 if (PyList_Append(py_pdlist, py_pd) < 0)
612 if (!(py_func = PyObject_GetAttrString(py_mod, "coverage")))
614 if (!(py_args = PyTuple_New(0)))
616 if (!(py_kwargs = Py_BuildValue("{sO}", "include", py_pdlist)))
618 if (!(py_cov = PyObject_Call(py_func, py_args, py_kwargs)))
620 if (!(PyObject_CallMethod(py_cov, "start", NULL)))
622 Py_DecRef(py_pdlist);
624 Py_DecRef(py_kwargs);
630 static struct cvg *get_mod_cov(PyObject *py_cov, const char *module_name)
632 PyObject *py_mod, *py_pathlist, *py_path, *py_func, *py_pd;
633 PyObject *py_result, *py_missed, *py_item;
637 int num_lines, num_missed, linenum, i, j;
638 char *path, *linespec;
640 if (!(py_mod = PyImport_ImportModule(module_name)))
644 py_pathlist = PyObject_GetAttrString(py_mod, "__path__");
645 for (i = 0; i < PyList_Size(py_pathlist); i++) {
646 py_path = PyList_GetItem(py_pathlist, i);
647 PyUnicode_FSConverter(PyList_GetItem(py_pathlist, i), &py_path);
648 path = PyBytes_AS_STRING(py_path);
649 if (!(d = opendir(path))) {
650 ERR("Invalid module path '%s'", path);
653 while ((de = readdir(d))) {
654 if (strncmp(de->d_name + strlen(de->d_name) - 3, ".py", 3))
657 if (!(py_func = PyObject_GetAttrString(py_cov, "analysis2")))
659 if (!(py_pd = PyUnicode_FromFormat("%s/%s", path, de->d_name)))
661 if (!(py_result = PyObject_CallFunction(py_func, "O", py_pd)))
668 if (PyTuple_Size(py_result) != 5) {
669 ERR("Invalid result from coverage of '%s/%s'", path, de->d_name);
672 num_lines = PyList_Size(PyTuple_GetItem(py_result, 1));
673 py_missed = PyTuple_GetItem(py_result, 3);
674 num_missed = PyList_Size(py_missed);
675 cvg_mod->num_lines += num_lines;
676 cvg_mod->num_missed += num_missed;
677 for (j = 0; j < num_missed; j++) {
678 py_item = PyList_GetItem(py_missed, j);
679 linenum = PyLong_AsLong(py_item);
680 linespec = g_strdup_printf("%s/%s:%d", module_name,
681 de->d_name, linenum);
682 cvg_mod->missed_lines = g_slist_append(cvg_mod->missed_lines, linespec);
684 DBG("Coverage for %s/%s: %d lines, %d missed.",
685 module_name, de->d_name, num_lines, num_missed);
686 Py_DecRef(py_result);
689 if (cvg_mod->num_lines)
690 cvg_mod->coverage = 100 - ((float)cvg_mod->num_missed / (float)cvg_mod->num_lines * 100);
698 static struct cvg *cvg_new(void)
702 cvg = calloc(1, sizeof(struct cvg));
707 static gboolean find_missed_line(struct cvg *cvg, const char *linespec)
711 for (l = cvg->missed_lines; l; l = l->next)
712 if (!strcmp(l->data, linespec))
718 static void cvg_add(struct cvg *dst, const struct cvg *src)
723 dst->num_lines += src->num_lines;
724 dst->num_missed += src->num_missed;
725 for (l = src->missed_lines; l; l = l->next) {
727 if (!find_missed_line(dst, linespec))
728 dst->missed_lines = g_slist_append(dst->missed_lines, linespec);
733 static int report_coverage(PyObject *py_cov, GSList *pdlist)
735 PyObject *py_func, *py_mod, *py_args, *py_kwargs, *py_outfile, *py_pct;
738 struct cvg *cvg_mod, *cvg_all;
739 float total_coverage;
740 int lines, missed, cnt;
742 DBG("Making coverage report.");
744 /* Get coverage for each module in the stack. */
747 for (cnt = 0, l = pdlist; l; l = l->next, cnt++) {
749 if (!(cvg_mod = get_mod_cov(py_cov, pd->name)))
751 printf("coverage: scope=%s coverage=%.0f%% lines=%d missed=%d "
752 "missed_lines=", pd->name, cvg_mod->coverage,
753 cvg_mod->num_lines, cvg_mod->num_missed);
754 for (ml = cvg_mod->missed_lines; ml; ml = ml->next) {
755 if (ml != cvg_mod->missed_lines)
757 printf("%s", (char *)ml->data);
760 lines += cvg_mod->num_lines;
761 missed += cvg_mod->num_missed;
762 cvg_add(cvg_all, cvg_mod);
763 DBG("Coverage for module %s: %d lines, %d missed", pd->name,
764 cvg_mod->num_lines, cvg_mod->num_missed);
768 total_coverage = 100 - ((float)missed / (float)lines * 100);
770 /* Machine-readable stats on stdout. */
771 printf("coverage: scope=all coverage=%.0f%% lines=%d missed=%d\n",
772 total_coverage, cvg_all->num_lines, cvg_all->num_missed);
774 /* Write text report to file. */
775 /* io.open(coverage_report, "w") */
776 if (!(py_mod = PyImport_ImportModule("io")))
778 if (!(py_func = PyObject_GetAttrString(py_mod, "open")))
780 if (!(py_args = PyTuple_New(0)))
782 if (!(py_kwargs = Py_BuildValue("{ssss}", "file", coverage_report,
785 if (!(py_outfile = PyObject_Call(py_func, py_args, py_kwargs)))
787 Py_DecRef(py_kwargs);
790 /* py_cov.report(file=py_outfile) */
791 if (!(py_func = PyObject_GetAttrString(py_cov, "report")))
793 if (!(py_kwargs = Py_BuildValue("{sO}", "file", py_outfile)))
795 if (!(py_pct = PyObject_Call(py_func, py_args, py_kwargs)))
798 Py_DecRef(py_kwargs);
801 /* py_outfile.close() */
802 if (!(py_func = PyObject_GetAttrString(py_outfile, "close")))
804 if (!PyObject_Call(py_func, py_args, NULL))
806 Py_DecRef(py_outfile);
814 int main(int argc, char **argv)
819 struct channel *channel;
820 struct option *option;
823 char *opt_infile, **kv, **opstr;
824 struct initial_pin_info *initial_pin;
826 op = malloc(sizeof(struct output));
838 while ((c = getopt(argc, argv, "dP:p:o:N:i:O:f:c:S")) != -1) {
844 pd = g_malloc(sizeof(struct pd));
845 pd->name = g_strdup(optarg);
846 pd->channels = pd->options = pd->initial_pins = NULL;
847 pdlist = g_slist_append(pdlist, pd);
852 if (g_slist_length(pdlist) == 0) {
853 /* No previous -P. */
854 ERR("Syntax error at '%s'", optarg);
857 kv = g_strsplit(optarg, "=", 0);
858 if (!kv[0] || (!kv[1] || kv[2])) {
860 ERR("Syntax error at '%s'", optarg);
865 channel = malloc(sizeof(struct channel));
866 channel->name = g_strdup(kv[0]);
867 channel->channel = strtoul(kv[1], NULL, 10);
868 /* Apply to last PD. */
869 pd->channels = g_slist_append(pd->channels, channel);
870 } else if (c == 'o') {
871 option = malloc(sizeof(struct option));
872 option->key = g_strdup(kv[0]);
873 option->value = g_variant_new_string(kv[1]);
874 g_variant_ref_sink(option->value);
875 /* Apply to last PD. */
876 pd->options = g_slist_append(pd->options, option);
878 initial_pin = malloc(sizeof(struct initial_pin_info));
879 initial_pin->name = g_strdup(kv[0]);
880 initial_pin->value = strtoul(kv[1], NULL, 10);
881 /* Apply to last PD. */
882 pd->initial_pins = g_slist_append(pd->initial_pins, initial_pin);
889 opstr = g_strsplit(optarg, ":", 0);
890 if (!opstr[0] || !opstr[1]) {
891 /* Need at least abc:def. */
892 ERR("Syntax error at '%s'", optarg);
896 op->pd = g_strdup(opstr[0]);
897 if (!strcmp(opstr[1], "annotation"))
898 op->type = SRD_OUTPUT_ANN;
899 else if (!strcmp(opstr[1], "binary"))
900 op->type = SRD_OUTPUT_BINARY;
901 else if (!strcmp(opstr[1], "python"))
902 op->type = SRD_OUTPUT_PYTHON;
903 else if (!strcmp(opstr[1], "exception"))
904 /* Doesn't matter, we just need it to bomb out. */
905 op->type = SRD_OUTPUT_PYTHON;
907 ERR("Unknown output type '%s'", opstr[1]);
912 op->class_ = g_strdup(opstr[2]);
916 op->outfile = g_strdup(optarg);
920 coverage_report = optarg;
931 if (g_slist_length(pdlist) == 0)
935 if (!op->pd || op->type == -1)
938 sr_log_callback_set(sr_log, NULL);
939 if (sr_init(&ctx) != SR_OK)
942 srd_log_callback_set(srd_log, NULL);
943 if (srd_init(DECODERS_DIR) != SRD_OK)
946 if (coverage_report) {
947 if (!(coverage = start_coverage(pdlist))) {
948 DBG("Failed to start coverage.");
949 if (PyErr_Occurred()) {
957 if (!run_testcase(opt_infile, pdlist, op))
961 DBG("Stopping coverage.");
963 if (!(PyObject_CallMethod(coverage, "stop", NULL)))
964 ERR("Failed to stop coverage.");
965 else if (!(report_coverage(coverage, pdlist)))
966 ERR("Failed to make coverage report.");
968 DBG("Coverage report in %s", coverage_report);
970 if (PyErr_Occurred()) {