2 * This file is part of the libsigrok project.
4 * Copyright (C) 2013 Marc Schink <sigrok-dev@marcschink.de>
5 * Copyright (C) 2019 Gerhard Sittig <gerhard.sittig@gmx.net>
7 * This program is free software: you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation, either version 3 of the License, or
10 * (at your option) any later version.
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
17 * You should have received a copy of the GNU General Public License
18 * along with this program. If not, see <http://www.gnu.org/licenses/>.
27 #include <libsigrok/libsigrok.h>
28 #include "libsigrok-internal.h"
29 #include "scpi.h" /* String un-quote for channel name from header line. */
31 #define LOG_PREFIX "input/csv"
33 #define CHUNK_SIZE (4 * 1024 * 1024)
36 * The CSV input module has the following options:
38 * column_formats: Specifies the data formats and channel counts for the
39 * input file's text columns. Accepts a comma separated list of tuples
40 * with: an optional column repeat count ('*' as a wildcard meaning
41 * "all remaining columns", only applicable to the last field), a format
42 * specifying character ('x' hexadecimal, 'o' octal, 'b' binary, 'l'
43 * single-bit logic), and an optional bit count (translating to: logic
44 * channels communicated in that column). This "column_formats" option
45 * is most versatile, other forms of specifying the column layout only
46 * exist for backwards compatibility.
48 * single_column: Specifies the column number which contains the logic data
49 * for single-column mode. All logic data is taken from several bits
50 * which all are kept within that one column. Only exists for backwards
51 * compatibility, see "column_formats" for more flexibility.
53 * first_column: Specifies the number of the first column with logic data
54 * in simple multi-column mode. Only exists for backwards compatibility,
55 * see "column_formats" for more flexibility.
57 * logic_channels: Specifies the number of logic channels. Is required in
58 * simple single-column mode. Is optional in simple multi-column mode
59 * (and defaults to all remaining columns). Only exists for backwards
60 * compatibility, see "column_formats" for more flexibility.
62 * single_format: Specifies the format of the input text in simple single-
63 * column mode. Available formats are: 'bin' (default), 'hex' and 'oct'.
64 * Simple multi-column mode always uses single-bit data per column.
65 * Only exists for backwards compatibility, see "column_formats" for
68 * start_line: Specifies at which line to start processing the input file.
69 * Allows to skip leading lines which neither are header nor data lines.
70 * By default all of the input file gets processed.
72 * header: Boolean option, controls whether the first processed line is used
73 * to determine channel names. Off by default. Generic channel names are
74 * used in the absence of header line content.
76 * samplerate: Specifies the samplerate of the input data. Defaults to 0.
77 * User specs take precedence over data which optionally gets derived
80 * column_separator: Specifies the sequence which separates the text file
81 * columns. Cannot be empty. Defaults to comma.
83 * comment_leader: Specifies the sequence which starts comments that run
84 * up to the end of the current text line. Can be empty to disable
85 * comment support. Defaults to semicolon.
87 * Typical examples of using these options:
88 * - ... -I csv:column_formats=*l ...
89 * All columns are single-bit logic data. Identical to the previous
90 * multi-column mode (the default when no options were given at all).
91 * - ... -I csv:column_formats=3-,*l ...
92 * Ignore the first three columns, get single-bit logic data from all
93 * remaining lines (multi-column mode with first-column above 1).
94 * - ... -I csv:column_formats=3-,4l,x8 ...
95 * Ignore the first three columns, get single-bit logic data from the
96 * next four columns, then eight-bit data in hex format from the next
97 * column. More columns may follow in the input text but won't get
98 * processed. (Mix of previous multi-column as well as single-column
100 * - ... -I csv:column_formats=4x8,b16,5l ...
101 * Get eight-bit data in hex format from the first four columns, then
102 * sixteen-bit data in binary format, then five times single-bit data.
103 * - ... -I csv:single_column=2:single_format=bin:logic_channels=8 ...
104 * Get eight logic bits in binary format from column 2. (Simple
105 * single-column mode, corresponds to the "-,b8" format.)
106 * - ... -I csv:first_column=6:logic_channels=4 ...
107 * Get four single-bit logic channels from columns 6 to 9 respectively.
108 * (Simple multi-column mode, corresponds to the "5-,4b" format.)
109 * - ... -I csv:start_line=20:header=yes:...
110 * Skip the first 19 text lines. Use line 20 to derive channel names.
111 * Data starts at line 21.
117 * - Extend support for analog input data? (optional)
118 * - Optionally get precision ('digits') from the column's format spec?
119 * From the position which is "bit count" for logic channels?
120 * - Determine why analog samples of 'double' data type get scrambled
121 * in sigrok-cli screen output. Is analog.encoding->unitsize not
122 * handled properly? A sigrok-cli or libsigrok (src/output) issue?
123 * - Optionally get sample rate from timestamp column. Just best-effort
124 * approach, not necessarily reliable. Users can always specify rates.
125 * - Add a test suite for input modules in general, and CSV in specific?
126 * Becomes more important with the multitude of options and their
127 * interaction. Could cover edge cases (BOM presence, line termination
128 * absence, etc) and auto-stuff as well (channel names, channel counts,
132 typedef float csv_analog_t; /* 'double' currently is flawed. */
134 /* Single column formats. */
135 enum single_col_format {
136 FORMAT_NONE, /* Ignore this column. */
137 FORMAT_BIN, /* Bin digits for a set of bits (or just one bit). */
138 FORMAT_HEX, /* Hex digits for a set of bits. */
139 FORMAT_OCT, /* Oct digits for a set of bits. */
140 FORMAT_ANALOG, /* Floating point number for an analog channel. */
143 static const char *col_format_text[] = {
144 [FORMAT_NONE] = "unknown",
145 [FORMAT_BIN] = "binary",
146 [FORMAT_HEX] = "hexadecimal",
147 [FORMAT_OCT] = "octal",
148 [FORMAT_ANALOG] = "analog",
151 static const char col_format_char[] = {
156 [FORMAT_ANALOG] = 'a',
159 struct column_details {
161 enum single_col_format text_format;
162 size_t channel_offset;
163 size_t channel_count;
169 /* Current selected samplerate. */
171 gboolean samplerate_sent;
173 /* Number of logic channels. List of names for analog datafeed. */
174 size_t logic_channels;
175 size_t analog_channels;
176 GSList **analog_datafeed_channels;
178 /* Column delimiter (actually separator), comment leader, EOL sequence. */
183 /* Format specs for input columns, and processing state. */
184 size_t column_seen_count;
185 const char *column_formats;
186 size_t column_want_count;
187 struct column_details *column_details;
189 /* Line number to start processing. */
193 * Determines if the first line should be treated as header and used for
194 * channel names in multi column mode.
197 gboolean header_seen;
199 size_t sample_unit_size; /**!< Byte count for a single sample. */
200 uint8_t *sample_buffer; /**!< Buffer for a single sample. */
201 csv_analog_t *analog_sample_buffer; /**!< Buffer for one set of analog values. */
203 uint8_t *datafeed_buffer; /**!< Queue for datafeed submission. */
204 size_t datafeed_buf_size;
205 size_t datafeed_buf_fill;
206 /* "Striped" layout, M samples for N channels each. */
207 csv_analog_t *analog_datafeed_buffer; /**!< Queue for analog datafeed. */
208 size_t analog_datafeed_buf_size;
209 size_t analog_datafeed_buf_fill;
211 /* Current line number. */
214 /* List of previously created sigrok channels. */
215 GSList *prev_sr_channels;
219 * Primitive operations to handle sample sets:
220 * - Keep a buffer for datafeed submission, capable of holding many
221 * samples (reduces call overhead, improves throughput).
222 * - Have a "current sample set" pointer reference one position in that
223 * large samples buffer.
224 * - Clear the current sample set before text line inspection, then set
225 * the bits which are found active in the current line of text input.
226 * Phrase the API such that call sites can be kept simple. Advance to
227 * the next sample set between lines, flush the larger buffer as needed
228 * (when it is full, or upon EOF).
231 static int flush_samplerate(const struct sr_input *in)
234 struct sr_datafeed_packet packet;
235 struct sr_datafeed_meta meta;
236 struct sr_config *src;
239 if (inc->samplerate && !inc->samplerate_sent) {
240 packet.type = SR_DF_META;
241 packet.payload = &meta;
242 src = sr_config_new(SR_CONF_SAMPLERATE, g_variant_new_uint64(inc->samplerate));
243 meta.config = g_slist_append(NULL, src);
244 sr_session_send(in->sdi, &packet);
245 g_slist_free(meta.config);
247 inc->samplerate_sent = TRUE;
253 static void clear_logic_samples(struct context *inc)
255 if (!inc->logic_channels)
257 inc->sample_buffer = &inc->datafeed_buffer[inc->datafeed_buf_fill];
258 memset(inc->sample_buffer, 0, inc->sample_unit_size);
261 static void set_logic_level(struct context *inc, size_t ch_idx, int on)
263 size_t byte_idx, bit_idx;
266 if (ch_idx >= inc->logic_channels)
271 byte_idx = ch_idx / 8;
272 bit_idx = ch_idx % 8;
273 bit_mask = 1 << bit_idx;
274 inc->sample_buffer[byte_idx] |= bit_mask;
277 static int flush_logic_samples(const struct sr_input *in)
280 struct sr_datafeed_packet packet;
281 struct sr_datafeed_logic logic;
285 if (!inc->datafeed_buf_fill)
288 rc = flush_samplerate(in);
292 memset(&packet, 0, sizeof(packet));
293 memset(&logic, 0, sizeof(logic));
294 packet.type = SR_DF_LOGIC;
295 packet.payload = &logic;
296 logic.unitsize = inc->sample_unit_size;
297 logic.length = inc->datafeed_buf_fill;
298 logic.data = inc->datafeed_buffer;
300 rc = sr_session_send(in->sdi, &packet);
304 inc->datafeed_buf_fill = 0;
308 static int queue_logic_samples(const struct sr_input *in)
314 if (!inc->logic_channels)
317 inc->datafeed_buf_fill += inc->sample_unit_size;
318 if (inc->datafeed_buf_fill == inc->datafeed_buf_size) {
319 rc = flush_logic_samples(in);
326 static void set_analog_value(struct context *inc, size_t ch_idx, csv_analog_t value);
328 static void clear_analog_samples(struct context *inc)
332 if (!inc->analog_channels)
334 inc->analog_sample_buffer = &inc->analog_datafeed_buffer[inc->analog_datafeed_buf_fill];
335 for (idx = 0; idx < inc->analog_channels; idx++)
336 set_analog_value(inc, idx, 0.0);
339 static void set_analog_value(struct context *inc, size_t ch_idx, csv_analog_t value)
341 if (ch_idx >= inc->analog_channels)
345 inc->analog_sample_buffer[ch_idx * inc->analog_datafeed_buf_size] = value;
348 static int flush_analog_samples(const struct sr_input *in)
350 /* TODO Use proper 'digits' value for this input module. */
351 static const int digits = 3;
354 struct sr_datafeed_packet packet;
355 struct sr_datafeed_analog analog;
356 struct sr_analog_encoding encoding;
357 struct sr_analog_meaning meaning;
358 struct sr_analog_spec spec;
359 csv_analog_t *samples;
364 if (!inc->analog_datafeed_buf_fill)
367 rc = flush_samplerate(in);
371 samples = inc->analog_datafeed_buffer;
372 for (ch_idx = 0; ch_idx < inc->analog_channels; ch_idx++) {
373 sr_analog_init(&analog, &encoding, &meaning, &spec, digits);
374 memset(&packet, 0, sizeof(packet));
375 packet.type = SR_DF_ANALOG;
376 packet.payload = &analog;
377 analog.num_samples = inc->analog_datafeed_buf_fill;
378 analog.data = samples;
379 analog.meaning->channels = inc->analog_datafeed_channels[ch_idx];
380 analog.meaning->mq = 0;
381 analog.meaning->mqflags = 0;
382 analog.meaning->unit = 0;
383 analog.encoding->unitsize = sizeof(samples[0]);
384 analog.encoding->is_signed = TRUE;
385 analog.encoding->is_float = TRUE;
386 #ifdef WORDS_BIGENDIAN
387 analog.encoding->is_bigendian = TRUE;
389 analog.encoding->is_bigendian = FALSE;
391 analog.encoding->digits = spec.spec_digits;
392 rc = sr_session_send(in->sdi, &packet);
395 samples += inc->analog_datafeed_buf_size;
398 inc->analog_datafeed_buf_fill = 0;
402 static int queue_analog_samples(const struct sr_input *in)
408 if (!inc->analog_channels)
411 inc->analog_datafeed_buf_fill++;
412 if (inc->analog_datafeed_buf_fill == inc->analog_datafeed_buf_size) {
413 rc = flush_analog_samples(in);
420 /* Helpers for "column processing". */
422 static int split_column_format(const char *spec,
423 size_t *column_count, enum single_col_format *format, size_t *bit_count)
426 char *endp, format_char;
427 enum single_col_format format_code;
432 /* Get the (optional, decimal, default 1) column count. Accept '*'. */
435 /* Workaround, strtoul("*") won't always yield expected endp. */
437 endp = (char *)&spec[1];
439 count = strtoul(spec, &endp, 10);
446 *column_count = count;
449 /* Get the (mandatory, single letter) type spec (-/xob/l). */
450 format_char = *spec++;
451 switch (format_char) {
455 format_code = FORMAT_NONE;
458 format_code = FORMAT_HEX;
461 format_code = FORMAT_OCT;
465 format_code = FORMAT_BIN;
468 format_code = FORMAT_ANALOG;
470 default: /* includes NUL */
474 *format = format_code;
476 /* Get the (optional, decimal, default 1) bit count. */
478 count = strtoul(spec, &endp, 10);
485 if (format_char == 'l')
487 if (format_code == FORMAT_ANALOG)
493 /* Input spec must have been exhausted. */
500 static int make_column_details_from_format(const struct sr_input *in,
501 const char *column_format, char **column_texts)
504 char **formats, *format;
505 size_t format_count, column_count, logic_count, analog_count;
506 size_t auto_column_count;
507 size_t format_idx, c, b, column_idx, channel_idx, analog_idx;
508 enum single_col_format f;
509 struct column_details *detail;
510 GString *channel_name;
514 int channel_type, channel_sdi_nr;
518 inc->column_seen_count = g_strv_length(column_texts);
520 /* Split the input spec, count involved columns and bits. */
521 formats = g_strsplit(column_format, ",", 0);
523 sr_err("Cannot parse columns format %s (comma split).", column_format);
526 format_count = g_strv_length(formats);
528 sr_err("Cannot parse columns format %s (field count).", column_format);
532 column_count = logic_count = analog_count = 0;
533 auto_column_count = 0;
534 for (format_idx = 0; format_idx < format_count; format_idx++) {
535 format = formats[format_idx];
536 ret = split_column_format(format, &c, &f, &b);
537 sr_dbg("fmt %s -> %zu cols, %s fmt, %zu bits, rc %d", format, c, col_format_text[f], b, ret);
539 sr_err("Cannot parse columns format %s (field split, %s).", column_format, format);
544 /* User requested "auto-count", must be last format. */
545 if (formats[format_idx + 1]) {
546 sr_err("Auto column count must be last format field.");
550 auto_column_count = inc->column_seen_count - column_count;
551 c = auto_column_count;
554 if (f == FORMAT_ANALOG)
557 logic_count += c * b;
559 sr_dbg("Column format %s -> %zu columns, %zu logic, %zu analog channels.",
560 column_format, column_count, logic_count, analog_count);
562 /* Allocate and fill in "column processing" details. Create channels. */
563 inc->column_want_count = column_count;
564 if (inc->column_seen_count < inc->column_want_count) {
565 sr_err("Insufficient input text width for desired data amount, got %zu but want %zu columns.",
566 inc->column_seen_count, inc->column_want_count);
570 inc->column_details = g_malloc0_n(column_count, sizeof(inc->column_details[0]));
571 column_idx = channel_idx = analog_idx = 0;
572 channel_name = g_string_sized_new(64);
573 for (format_idx = 0; format_idx < format_count; format_idx++) {
574 /* Process a format field, which can span multiple columns. */
575 format = formats[format_idx];
576 (void)split_column_format(format, &c, &f, &b);
578 c = auto_column_count;
580 /* Fill in a column's processing details. */
581 detail = &inc->column_details[column_idx++];
582 detail->col_nr = column_idx;
583 detail->text_format = f;
584 if (detail->text_format == FORMAT_ANALOG) {
585 detail->channel_offset = analog_idx;
586 detail->channel_count = 1;
587 analog_idx += detail->channel_count;
588 } else if (detail->text_format) {
589 detail->channel_offset = channel_idx;
590 detail->channel_count = b;
591 channel_idx += detail->channel_count;
593 sr_dbg("detail -> col %zu, fmt %s, ch off/cnt %zu/%zu",
594 detail->col_nr, col_format_text[detail->text_format],
595 detail->channel_offset, detail->channel_count);
596 if (!detail->text_format)
599 * Create channels with appropriate names. Optionally
600 * use text from a header line (when requested by the
601 * user). In the absence of header text, channels are
602 * assigned rather generic names.
604 * Manipulation of the column's caption (when a header
605 * line is seen) is acceptable, because this header
606 * line won't get processed another time.
608 column = column_texts[detail->col_nr - 1];
609 if (inc->use_header && column && *column)
610 caption = sr_scpi_unquote_string(column);
613 if (!caption || !*caption)
615 for (create_idx = 0; create_idx < detail->channel_count; create_idx++) {
616 if (caption && detail->channel_count == 1) {
617 g_string_assign(channel_name, caption);
618 } else if (caption) {
619 g_string_printf(channel_name, "%s[%zu]",
620 caption, create_idx);
622 g_string_printf(channel_name, "%zu",
623 detail->channel_offset + create_idx);
625 if (detail->text_format == FORMAT_ANALOG) {
626 channel_sdi_nr = logic_count + detail->channel_offset + create_idx;
627 channel_type = SR_CHANNEL_ANALOG;
629 channel_sdi_nr = detail->channel_offset + create_idx;
630 channel_type = SR_CHANNEL_LOGIC;
632 sr_channel_new(in->sdi, channel_sdi_nr,
633 channel_type, TRUE, channel_name->str);
637 inc->logic_channels = channel_idx;
638 inc->analog_channels = analog_idx;
639 g_string_free(channel_name, TRUE);
645 static const struct column_details *lookup_column_details(struct context *inc, size_t nr)
647 if (!inc || !inc->column_details)
649 if (!nr || nr > inc->column_want_count)
651 return &inc->column_details[nr - 1];
655 * Primitive operations for text input: Strip comments off text lines.
656 * Split text lines into columns. Process input text for individual
660 static void strip_comment(char *buf, const GString *prefix)
667 if ((ptr = strstr(buf, prefix->str))) {
674 * @brief Splits a text line into a set of columns.
676 * @param[in] buf The input text line to split.
677 * @param[in] inc The input module's context.
679 * @returns An array of strings, representing the columns' text.
681 * This routine splits a text line on previously determined separators.
683 static char **split_line(char *buf, struct context *inc)
685 return g_strsplit(buf, inc->delimiter->str, 0);
689 * @brief Parse a multi-bit field into several logic channels.
691 * @param[in] column The input text, a run of bin/hex/oct digits.
692 * @param[in] inc The input module's context.
693 * @param[in] details The column processing details.
695 * @retval SR_OK Success.
696 * @retval SR_ERR Invalid input data (empty, or format error).
698 * This routine modifies the logic levels in the current sample set,
699 * based on the text input and a user provided format spec.
701 static int parse_logic(const char *column, struct context *inc,
702 const struct column_details *details)
704 size_t length, ch_rem, ch_idx, ch_inc;
708 const char *type_text;
712 * Prepare to read the digits from the text end towards the start.
713 * A digit corresponds to a variable number of channels (depending
714 * on the value's radix). Prepare the mapping of text digits to
715 * (a number of) logic channels.
717 length = strlen(column);
719 sr_err("Column %zu in line %zu is empty.", details->col_nr,
723 rdptr = &column[length];
724 ch_idx = details->channel_offset;
725 ch_rem = details->channel_count;
728 * Get another digit and derive up to four logic channels' state from
729 * it. Make sure to not process more bits than the column has channels
730 * associated with it.
732 while (rdptr > column && ch_rem) {
733 /* Check for valid digits according to the input radix. */
735 switch (details->text_format) {
737 valid = g_ascii_isxdigit(c) && c < '2';
741 valid = g_ascii_isxdigit(c) && c < '8';
745 valid = g_ascii_isxdigit(c);
753 type_text = col_format_text[details->text_format];
754 sr_err("Invalid text '%s' in %s type column %zu in line %zu.",
755 column, type_text, details->col_nr, inc->line_number);
758 /* Use the digit's bits for logic channels' data. */
759 bits = g_ascii_xdigit_value(c);
760 switch (details->text_format) {
764 set_logic_level(inc, ch_idx + 3, bits & (1 << 3));
770 set_logic_level(inc, ch_idx + 2, bits & (1 << 2));
774 set_logic_level(inc, ch_idx + 1, bits & (1 << 1));
779 set_logic_level(inc, ch_idx + 0, bits & (1 << 0));
783 /* ShouldNotHappen(TM), but silences compiler warning. */
789 * TODO Determine whether the availability of extra input data
790 * for unhandled logic channels is worth warning here. In this
791 * implementation users are in control, and can have the more
792 * significant bits ignored (which can be considered a feature
793 * and not really a limitation).
800 * @brief Parse a floating point text into an analog value.
802 * @param[in] column The input text, a floating point number.
803 * @param[in] inc The input module's context.
804 * @param[in] details The column processing details.
806 * @retval SR_OK Success.
807 * @retval SR_ERR Invalid input data (empty, or format error).
809 * This routine modifies the analog values in the current sample set,
810 * based on the text input and a user provided format spec.
812 static int parse_analog(const char *column, struct context *inc,
813 const struct column_details *details)
816 double dvalue; float fvalue;
820 if (details->text_format != FORMAT_ANALOG)
823 length = strlen(column);
825 sr_err("Column %zu in line %zu is empty.", details->col_nr,
829 if (sizeof(value) == sizeof(double)) {
830 ret = sr_atod_ascii(column, &dvalue);
832 } else if (sizeof(value) == sizeof(float)) {
833 ret = sr_atof_ascii(column, &fvalue);
839 sr_err("Cannot parse analog text %s in column %zu in line %zu.",
840 column, details->col_nr, inc->line_number);
843 set_analog_value(inc, details->channel_offset, value);
849 * @brief Parse routine which ignores the input text.
851 * This routine exists to unify dispatch code paths, mapping input file
852 * columns' data types to their respective parse routines.
854 static int parse_ignore(const char *column, struct context *inc,
855 const struct column_details *details)
863 typedef int (*col_parse_cb)(const char *column, struct context *inc,
864 const struct column_details *details);
866 static const col_parse_cb col_parse_funcs[] = {
867 [FORMAT_NONE] = parse_ignore,
868 [FORMAT_BIN] = parse_logic,
869 [FORMAT_OCT] = parse_logic,
870 [FORMAT_HEX] = parse_logic,
871 [FORMAT_ANALOG] = parse_analog,
874 static int init(struct sr_input *in, GHashTable *options)
877 size_t single_column, first_column, logic_channels;
879 enum single_col_format format;
882 in->sdi = g_malloc0(sizeof(*in->sdi));
883 in->priv = inc = g_malloc0(sizeof(*inc));
885 single_column = g_variant_get_uint32(g_hash_table_lookup(options, "single_column"));
886 logic_channels = g_variant_get_uint32(g_hash_table_lookup(options, "logic_channels"));
887 inc->delimiter = g_string_new(g_variant_get_string(
888 g_hash_table_lookup(options, "column_separator"), NULL));
889 if (!inc->delimiter->len) {
890 sr_err("Column separator cannot be empty.");
893 s = g_variant_get_string(g_hash_table_lookup(options, "single_format"), NULL);
894 if (g_ascii_strncasecmp(s, "bin", 3) == 0) {
896 } else if (g_ascii_strncasecmp(s, "hex", 3) == 0) {
898 } else if (g_ascii_strncasecmp(s, "oct", 3) == 0) {
901 sr_err("Invalid single-column format: '%s'", s);
904 inc->comment = g_string_new(g_variant_get_string(
905 g_hash_table_lookup(options, "comment_leader"), NULL));
906 if (g_string_equal(inc->comment, inc->delimiter)) {
908 * Using the same sequence as comment leader and column
909 * separator won't work. The user probably specified ';'
910 * as the column separator but did not adjust the comment
911 * leader. Try DWIM, drop comment strippin support here.
913 sr_warn("Comment leader and column separator conflict, disabling comment support.");
914 g_string_truncate(inc->comment, 0);
916 inc->samplerate = g_variant_get_uint64(g_hash_table_lookup(options, "samplerate"));
917 first_column = g_variant_get_uint32(g_hash_table_lookup(options, "first_column"));
918 inc->use_header = g_variant_get_boolean(g_hash_table_lookup(options, "header"));
919 inc->start_line = g_variant_get_uint32(g_hash_table_lookup(options, "start_line"));
920 if (inc->start_line < 1) {
921 sr_err("Invalid start line %zu.", inc->start_line);
926 * Scan flexible, to get prefered format specs which describe
927 * the input file's data formats. As well as some simple specs
928 * for backwards compatibility and user convenience.
930 * This logic ends up with a copy of the format string, either
931 * user provided or internally derived. Actual creation of the
932 * column processing details gets deferred until the first line
933 * of input data was seen. To support automatic determination of
934 * e.g. channel counts from column counts.
936 s = g_variant_get_string(g_hash_table_lookup(options, "column_formats"), NULL);
938 inc->column_formats = g_strdup(s);
939 sr_dbg("User specified column_formats: %s.", s);
940 } else if (single_column && logic_channels) {
941 format_char = col_format_char[format];
942 if (single_column == 1) {
943 inc->column_formats = g_strdup_printf("%c%zu",
944 format_char, logic_channels);
946 inc->column_formats = g_strdup_printf("%zu-,%c%zu",
948 format_char, logic_channels);
950 sr_dbg("Backwards compat single_column, col %zu, fmt %s, bits %zu -> %s.",
951 single_column, col_format_text[format], logic_channels,
952 inc->column_formats);
953 } else if (!single_column) {
954 if (first_column > 1) {
955 inc->column_formats = g_strdup_printf("%zu-,%zul",
956 first_column - 1, logic_channels);
958 inc->column_formats = g_strdup_printf("%zul",
961 sr_dbg("Backwards compat multi-column, col %zu, chans %zu -> %s.",
962 first_column, logic_channels,
963 inc->column_formats);
965 sr_warn("Unknown or unsupported columns layout spec, assuming simple multi-column mode.");
966 inc->column_formats = g_strdup("*l");
973 * Check the channel list for consistency across file re-import. See
974 * the VCD input module for more details and motivation.
977 static void keep_header_for_reread(const struct sr_input *in)
982 g_slist_free_full(inc->prev_sr_channels, sr_channel_free_cb);
983 inc->prev_sr_channels = in->sdi->channels;
984 in->sdi->channels = NULL;
987 static int check_header_in_reread(const struct sr_input *in)
996 if (!inc->prev_sr_channels)
999 if (sr_channel_lists_differ(inc->prev_sr_channels, in->sdi->channels)) {
1000 sr_err("Channel list change not supported for file re-read.");
1003 g_slist_free_full(in->sdi->channels, sr_channel_free_cb);
1004 in->sdi->channels = inc->prev_sr_channels;
1005 inc->prev_sr_channels = NULL;
1010 static const char *delim_set = "\r\n";
1012 static const char *get_line_termination(GString *buf)
1017 if (g_strstr_len(buf->str, buf->len, "\r\n"))
1019 else if (memchr(buf->str, '\n', buf->len))
1021 else if (memchr(buf->str, '\r', buf->len))
1027 static int initial_parse(const struct sr_input *in, GString *buf)
1029 struct context *inc;
1031 size_t line_number, line_idx, ch_idx;
1033 char **lines, *line, **columns;
1039 /* Search for the first line to process (header or data). */
1041 if (inc->termination)
1042 lines = g_strsplit(buf->str, inc->termination, 0);
1044 lines = g_strsplit_set(buf->str, delim_set, 0);
1045 for (line_idx = 0; (line = lines[line_idx]); line_idx++) {
1047 if (inc->start_line > line_number) {
1048 sr_spew("Line %zu skipped (before start).", line_number);
1051 if (line[0] == '\0') {
1052 sr_spew("Blank line %zu skipped.", line_number);
1055 strip_comment(line, inc->comment);
1056 if (line[0] == '\0') {
1057 sr_spew("Comment-only line %zu skipped.", line_number);
1061 /* Reached first proper line. */
1065 /* Not enough data for a proper line yet. */
1070 /* Get the number of columns in the line. */
1071 columns = split_line(line, inc);
1073 sr_err("Error while parsing line %zu.", line_number);
1077 num_columns = g_strv_length(columns);
1079 sr_err("Error while parsing line %zu.", line_number);
1083 sr_dbg("DIAG Got %zu columns in text line: %s.", num_columns, line);
1086 * Interpret the user provided column format specs. This might
1087 * involve inspection of the now received input text, to support
1088 * e.g. automatic detection of channel counts in the absence of
1089 * user provided specs. Optionally a header line is used to get
1092 * Check the then created channels for consistency across .reset
1093 * and .receive sequences (file re-load).
1095 ret = make_column_details_from_format(in, inc->column_formats, columns);
1097 sr_err("Cannot parse columns format using line %zu.", line_number);
1100 if (!check_header_in_reread(in)) {
1106 * Allocate buffer memory for datafeed submission of sample data.
1107 * Calculate the minimum buffer size to store the set of samples
1108 * of all channels (unit size). Determine a larger buffer size
1109 * for datafeed submission that is a multiple of the unit size.
1110 * Allocate the larger buffer, the "sample buffer" will point
1111 * to a location within that large buffer later.
1113 if (inc->logic_channels) {
1114 inc->sample_unit_size = (inc->logic_channels + 7) / 8;
1115 inc->datafeed_buf_size = CHUNK_SIZE;
1116 inc->datafeed_buf_size *= inc->sample_unit_size;
1117 inc->datafeed_buffer = g_malloc(inc->datafeed_buf_size);
1118 if (!inc->datafeed_buffer) {
1119 sr_err("Cannot allocate datafeed send buffer (logic).");
1120 ret = SR_ERR_MALLOC;
1123 inc->datafeed_buf_fill = 0;
1126 if (inc->analog_channels) {
1127 size_t sample_size, sample_count;
1128 sample_size = sizeof(inc->analog_datafeed_buffer[0]);
1129 inc->analog_datafeed_buf_size = CHUNK_SIZE;
1130 inc->analog_datafeed_buf_size /= sample_size;
1131 inc->analog_datafeed_buf_size /= inc->analog_channels;
1132 sample_count = inc->analog_channels * inc->analog_datafeed_buf_size;
1133 inc->analog_datafeed_buffer = g_malloc0(sample_count * sample_size);
1134 if (!inc->analog_datafeed_buffer) {
1135 sr_err("Cannot allocate datafeed send buffer (analog).");
1136 ret = SR_ERR_MALLOC;
1139 inc->analog_datafeed_buf_fill = 0;
1140 inc->analog_datafeed_channels = g_malloc0_n(inc->analog_channels, sizeof(inc->analog_datafeed_channels[0]));
1141 for (ch_idx = 0; ch_idx < inc->analog_channels; ch_idx++) {
1143 channel = g_slist_nth_data(in->sdi->channels, inc->logic_channels + ch_idx);
1144 inc->analog_datafeed_channels[ch_idx] = g_slist_append(NULL, channel);
1150 g_strfreev(columns);
1157 * Gets called from initial_receive(), which runs until the end-of-line
1158 * encoding of the input stream could get determined. Assumes that this
1159 * routine receives enough buffered initial input data to either see the
1160 * BOM when there is one, or that no BOM will follow when a text line
1161 * termination sequence was seen. Silently drops the UTF-8 BOM sequence
1162 * from the input buffer if one was seen. Does not care to protect
1163 * against multiple execution or dropping the BOM multiple times --
1164 * there should be at most one in the input stream.
1166 static void initial_bom_check(const struct sr_input *in)
1168 static const char *utf8_bom = "\xef\xbb\xbf";
1170 if (in->buf->len < strlen(utf8_bom))
1172 if (strncmp(in->buf->str, utf8_bom, strlen(utf8_bom)) != 0)
1174 g_string_erase(in->buf, 0, strlen(utf8_bom));
1177 static int initial_receive(const struct sr_input *in)
1179 struct context *inc;
1183 const char *termination;
1185 initial_bom_check(in);
1189 termination = get_line_termination(in->buf);
1191 /* Don't have a full line yet. */
1194 p = g_strrstr_len(in->buf->str, in->buf->len, termination);
1196 /* Don't have a full line yet. */
1198 len = p - in->buf->str - 1;
1199 new_buf = g_string_new_len(in->buf->str, len);
1200 g_string_append_c(new_buf, '\0');
1202 inc->termination = g_strdup(termination);
1204 if (in->buf->str[0] != '\0')
1205 ret = initial_parse(in, new_buf);
1209 g_string_free(new_buf, TRUE);
1214 static int process_buffer(struct sr_input *in, gboolean is_eof)
1216 struct context *inc;
1218 size_t line_idx, col_idx, col_nr;
1219 const struct column_details *details;
1220 col_parse_cb parse_func;
1222 char *p, **lines, *line, **columns, *column;
1225 if (!inc->started) {
1226 std_session_send_df_header(in->sdi);
1227 inc->started = TRUE;
1231 * Consider empty input non-fatal. Keep accumulating input until
1232 * at least one full text line has become available. Grab the
1233 * maximum amount of accumulated data that consists of full text
1234 * lines, and process what has been received so far, leaving not
1235 * yet complete lines for the next invocation.
1237 * Enforce that all previously buffered data gets processed in
1238 * the "EOF" condition. Do not insist in the presence of the
1239 * termination sequence for the last line (may often be missing
1240 * on Windows). A present termination sequence will just result
1241 * in the "execution of an empty line", and does not harm.
1246 p = in->buf->str + in->buf->len;
1248 p = g_strrstr_len(in->buf->str, in->buf->len, inc->termination);
1252 p += strlen(inc->termination);
1254 g_strstrip(in->buf->str);
1257 lines = g_strsplit(in->buf->str, inc->termination, 0);
1258 for (line_idx = 0; (line = lines[line_idx]); line_idx++) {
1260 if (inc->line_number < inc->start_line) {
1261 sr_spew("Line %zu skipped (before start).", inc->line_number);
1264 if (line[0] == '\0') {
1265 sr_spew("Blank line %zu skipped.", inc->line_number);
1269 /* Remove trailing comment. */
1270 strip_comment(line, inc->comment);
1271 if (line[0] == '\0') {
1272 sr_spew("Comment-only line %zu skipped.", inc->line_number);
1276 /* Skip the header line, its content was used as the channel names. */
1277 if (inc->use_header && !inc->header_seen) {
1278 sr_spew("Header line %zu skipped.", inc->line_number);
1279 inc->header_seen = TRUE;
1283 /* Split the line into columns, check for minimum length. */
1284 columns = split_line(line, inc);
1286 sr_err("Error while parsing line %zu.", inc->line_number);
1290 num_columns = g_strv_length(columns);
1291 if (num_columns < inc->column_want_count) {
1292 sr_err("Insufficient column count %zu in line %zu.",
1293 num_columns, inc->line_number);
1294 g_strfreev(columns);
1299 /* Have the columns of the current text line processed. */
1300 clear_logic_samples(inc);
1301 clear_analog_samples(inc);
1302 for (col_idx = 0; col_idx < inc->column_want_count; col_idx++) {
1303 column = columns[col_idx];
1304 col_nr = col_idx + 1;
1305 details = lookup_column_details(inc, col_nr);
1306 if (!details || !details->text_format)
1308 parse_func = col_parse_funcs[details->text_format];
1311 ret = parse_func(column, inc, details);
1313 g_strfreev(columns);
1319 /* Send sample data to the session bus (buffered). */
1320 ret = queue_logic_samples(in);
1321 ret += queue_analog_samples(in);
1323 sr_err("Sending samples failed.");
1324 g_strfreev(columns);
1329 g_strfreev(columns);
1332 g_string_erase(in->buf, 0, p - in->buf->str);
1337 static int receive(struct sr_input *in, GString *buf)
1339 struct context *inc;
1342 g_string_append_len(in->buf, buf->str, buf->len);
1345 if (!inc->column_seen_count) {
1346 ret = initial_receive(in);
1347 if (ret == SR_ERR_NA)
1348 /* Not enough data yet. */
1350 else if (ret != SR_OK)
1353 /* sdi is ready, notify frontend. */
1354 in->sdi_ready = TRUE;
1358 ret = process_buffer(in, FALSE);
1363 static int end(struct sr_input *in)
1365 struct context *inc;
1369 ret = process_buffer(in, TRUE);
1375 ret = flush_logic_samples(in);
1376 ret += flush_analog_samples(in);
1382 std_session_send_df_end(in->sdi);
1387 static void cleanup(struct sr_input *in)
1389 struct context *inc;
1391 keep_header_for_reread(in);
1395 g_free(inc->termination);
1396 inc->termination = NULL;
1397 g_free(inc->datafeed_buffer);
1398 inc->datafeed_buffer = NULL;
1399 g_free(inc->analog_datafeed_buffer);
1400 inc->analog_datafeed_buffer = NULL;
1403 static int reset(struct sr_input *in)
1405 struct context *inc = in->priv;
1408 inc->started = FALSE;
1409 g_string_truncate(in->buf, 0);
1428 static struct sr_option options[] = {
1430 "column_formats", "Column format specs",
1431 "Specifies text columns data types: comma separated list of [<cols>]<fmt>[<bits>], with -/x/o/b/l format specifiers.",
1434 [OPT_SINGLE_COL] = {
1435 "single_column", "Single column",
1436 "Enable single-column mode, exclusively use text from the specified column (number starting at 1).",
1440 "first_column", "First column",
1441 "Number of the first column with logic data in simple multi-column mode (number starting at 1, default 1).",
1445 "logic_channels", "Number of logic channels",
1446 "Logic channel count, required in simple single-column mode, defaults to \"all remaining columns\" in simple multi-column mode. Obsoleted by 'column_formats'.",
1450 "single_format", "Data format for simple single-column mode.",
1451 "The number format of single-column mode input data: bin, hex, oct.",
1455 "start_line", "Start line",
1456 "The line number at which to start processing input text (default: 1).",
1460 "header", "Get channel names from first line.",
1461 "Use the first processed line's column captions (when available) as channel names.",
1465 "samplerate", "Samplerate (Hz)",
1466 "The input data's sample rate in Hz.",
1470 "column_separator", "Column separator",
1471 "The sequence which separates text columns. Non-empty text, comma by default.",
1475 "comment_leader", "Comment leader character",
1476 "The text which starts comments at the end of text lines.",
1479 [OPT_MAX] = ALL_ZERO,
1482 static const struct sr_option *get_options(void)
1486 if (!options[0].def) {
1487 options[OPT_COL_FMTS].def = g_variant_ref_sink(g_variant_new_string(""));
1488 options[OPT_SINGLE_COL].def = g_variant_ref_sink(g_variant_new_uint32(0));
1489 options[OPT_FIRST_COL].def = g_variant_ref_sink(g_variant_new_uint32(1));
1490 options[OPT_NUM_LOGIC].def = g_variant_ref_sink(g_variant_new_uint32(0));
1491 options[OPT_FORMAT].def = g_variant_ref_sink(g_variant_new_string("bin"));
1493 l = g_slist_append(l, g_variant_ref_sink(g_variant_new_string("bin")));
1494 l = g_slist_append(l, g_variant_ref_sink(g_variant_new_string("hex")));
1495 l = g_slist_append(l, g_variant_ref_sink(g_variant_new_string("oct")));
1496 options[OPT_FORMAT].values = l;
1497 options[OPT_START].def = g_variant_ref_sink(g_variant_new_uint32(1));
1498 options[OPT_HEADER].def = g_variant_ref_sink(g_variant_new_boolean(FALSE));
1499 options[OPT_RATE].def = g_variant_ref_sink(g_variant_new_uint64(0));
1500 options[OPT_DELIM].def = g_variant_ref_sink(g_variant_new_string(","));
1501 options[OPT_COMMENT].def = g_variant_ref_sink(g_variant_new_string(";"));
1507 SR_PRIV struct sr_input_module input_csv = {
1510 .desc = "Comma-separated values",
1511 .exts = (const char*[]){"csv", NULL},
1512 .options = get_options,