input/csv: Skip leading UTF-8 BOM in the input stream

[libsigrok.git] / src / input / csv.c

diff --git a/src/input/csv.c b/src/input/csv.c
index 5399027bf823305ab0a33c79b02ec9783e1ca5c3..baf37188ad6427ffd903ab31b62197e651dd45b4 100644 (file)
--- a/src/input/csv.c
+++ b/src/input/csv.c
@@ -67,6 +67,45 @@
  *                than 0. The default line number to start processing is 1.
  */
 
+/*
+ * TODO
+ *
+ * - Determine how the text line handling can get improved, regarding
+ *   all of robustness and flexibility and correctness.
+ *   - The current implementation splits on "any run of CR and LF". Which
+ *     translates to: Line numbers are wrong in the presence of empty
+ *     lines in the input stream.
+ *   - The current implementation insists in the presence of end-of-line
+ *     markers on _every_ line in the input stream. "Incomplete" text
+ *     files that are so typical on the Windows platform get rejected as
+ *     invalid.
+ *   - Dropping support for CR style end-of-line markers could improve
+ *     the situation a lot. Code could search for and split on LF, and
+ *     trim optional trailing CR. This would result in proper support
+ *     for CRLF (Windows) as well as LF (Unix), and allow for correct
+ *     line number counts.
+ *   - When support for CR-only line termination cannot get dropped,
+ *     then the current implementation is inappropriate. Currently the
+ *     input stream is scanned for the first occurance of either of the
+ *     supported termination styles (which is good). For the remaining
+ *     session a consistent encoding of the text lines is assumed (which
+ *     is acceptable). Potential absence of the terminator for the last
+ *     line is orthogonal, and can get handled by a "force" flag when
+ *     the end() routine calls the process_buffer() routine.
+ *   - When line numbers need to be correct and reliable, _and_ the full
+ *     set of previously supported line termination sequences are required,
+ *     and potentially more are to get added for improved compatibility
+ *     with more platforms or generators, then the current approach of
+ *     splitting on runs of termination characters needs to get replaced,
+ *     by the more expensive approach to scan for and count the initially
+ *     determined termination sequence.
+ *
+ * - Add support for analog input data? (optional)
+ *   - Needs a syntax first for user specs which channels (columns) are
+ *     logic and which are analog. May need heuristics(?) to guess from
+ *     input data in the absence of user provided specs.
+ */
+
 /* Single column formats. */
 enum {
        FORMAT_BIN,
@@ -574,6 +613,27 @@ out:
        return ret;
 }
 
+/*
+ * Gets called from initial_receive(), which runs until the end-of-line
+ * encoding of the input stream could get determined. Assumes that this
+ * routine receives enough buffered initial input data to either see the
+ * BOM when there is one, or that no BOM will follow when a text line
+ * termination sequence was seen. Silently drops the UTF-8 BOM sequence
+ * from the input buffer if one was seen. Does not care to protect
+ * against multiple execution or dropping the BOM multiple times --
+ * there should be at most one in the input stream.
+ */
+static void initial_bom_check(const struct sr_input *in)
+{
+       static const char *utf8_bom = "\xef\xbb\xbf";
+
+       if (in->buf->len < strlen(utf8_bom))
+               return;
+       if (strncmp(in->buf->str, utf8_bom, strlen(utf8_bom)) != 0)
+               return;
+       g_string_erase(in->buf, 0, strlen(utf8_bom));
+}
+
 static int initial_receive(const struct sr_input *in)
 {
        struct context *inc;
@@ -582,6 +642,8 @@ static int initial_receive(const struct sr_input *in)
        char *p;
        const char *termination;
 
+       initial_bom_check(in);
+
        inc = in->priv;
 
        termination = get_line_termination(in->buf);
@@ -657,6 +719,7 @@ static int process_buffer(struct sr_input *in)
        if (!p)
                return SR_ERR;
        *p = '\0';
+       p += strlen(inc->termination);
        g_strstrip(in->buf->str);
 
        ret = SR_OK;
@@ -725,7 +788,7 @@ static int process_buffer(struct sr_input *in)
                g_strfreev(columns);
        }
        g_strfreev(lines);
-       g_string_erase(in->buf, 0, p - in->buf->str + 1);
+       g_string_erase(in->buf, 0, p - in->buf->str);
 
        return ret;
 }