input/csv: Update developer comment (fix for last EOL marker)

[libsigrok.git] / src / input / csv.c

diff --git a/src/input/csv.c b/src/input/csv.c
index d58e003146f662d177823506d018e91364514ddd..1d230fdcad0e2c3777f329943fc1ddc219a6de0f 100644 (file)
--- a/src/input/csv.c
+++ b/src/input/csv.c
@@ -67,6 +67,39 @@
  *                than 0. The default line number to start processing is 1.
  */
 
+/*
+ * TODO
+ *
+ * - Determine how the text line handling can get improved, regarding
+ *   all of robustness and flexibility and correctness.
+ *   - The current implementation splits on "any run of CR and LF". Which
+ *     translates to: Line numbers are wrong in the presence of empty
+ *     lines in the input stream. See below for an (expensive) fix.
+ *   - Dropping support for CR style end-of-line markers could improve
+ *     the situation a lot. Code could search for and split on LF, and
+ *     trim optional trailing CR. This would result in proper support
+ *     for CRLF (Windows) as well as LF (Unix), and allow for correct
+ *     line number counts.
+ *   - When support for CR-only line termination cannot get dropped,
+ *     then the current implementation is inappropriate. Currently the
+ *     input stream is scanned for the first occurance of either of the
+ *     supported termination styles (which is good). For the remaining
+ *     session a consistent encoding of the text lines is assumed (which
+ *     is acceptable).
+ *   - When line numbers need to be correct and reliable, _and_ the full
+ *     set of previously supported line termination sequences are required,
+ *     and potentially more are to get added for improved compatibility
+ *     with more platforms or generators, then the current approach of
+ *     splitting on runs of termination characters needs to get replaced,
+ *     by the more expensive approach to scan for and count the initially
+ *     determined termination sequence.
+ *
+ * - Add support for analog input data? (optional)
+ *   - Needs a syntax first for user specs which channels (columns) are
+ *     logic and which are analog. May need heuristics(?) to guess from
+ *     input data in the absence of user provided specs.
+ */
+
 /* Single column formats. */
 enum {
        FORMAT_BIN,
@@ -574,6 +607,27 @@ out:
        return ret;
 }
 
+/*
+ * Gets called from initial_receive(), which runs until the end-of-line
+ * encoding of the input stream could get determined. Assumes that this
+ * routine receives enough buffered initial input data to either see the
+ * BOM when there is one, or that no BOM will follow when a text line
+ * termination sequence was seen. Silently drops the UTF-8 BOM sequence
+ * from the input buffer if one was seen. Does not care to protect
+ * against multiple execution or dropping the BOM multiple times --
+ * there should be at most one in the input stream.
+ */
+static void initial_bom_check(const struct sr_input *in)
+{
+       static const char *utf8_bom = "\xef\xbb\xbf";
+
+       if (in->buf->len < strlen(utf8_bom))
+               return;
+       if (strncmp(in->buf->str, utf8_bom, strlen(utf8_bom)) != 0)
+               return;
+       g_string_erase(in->buf, 0, strlen(utf8_bom));
+}
+
 static int initial_receive(const struct sr_input *in)
 {
        struct context *inc;
@@ -582,6 +636,8 @@ static int initial_receive(const struct sr_input *in)
        char *p;
        const char *termination;
 
+       initial_bom_check(in);
+
        inc = in->priv;
 
        termination = get_line_termination(in->buf);
@@ -609,7 +665,7 @@ static int initial_receive(const struct sr_input *in)
        return ret;
 }
 
-static int process_buffer(struct sr_input *in)
+static int process_buffer(struct sr_input *in, gboolean is_eof)
 {
        struct sr_datafeed_packet packet;
        struct sr_datafeed_meta meta;
@@ -650,14 +706,24 @@ static int process_buffer(struct sr_input *in)
         * maximum amount of accumulated data that consists of full text
         * lines, and process what has been received so far, leaving not
         * yet complete lines for the next invocation.
+        *
+        * Enforce that all previously buffered data gets processed in
+        * the "EOF" condition. Do not insist in the presence of the
+        * termination sequence for the last line (may often be missing
+        * on Windows). A present termination sequence will just result
+        * in the "execution of an empty line", and does not harm.
         */
        if (!in->buf->len)
                return SR_OK;
-       p = g_strrstr_len(in->buf->str, in->buf->len, inc->termination);
-       if (!p)
-               return SR_ERR;
-       *p = '\0';
-       p += strlen(inc->termination);
+       if (is_eof) {
+               p = in->buf->str + in->buf->len;
+       } else {
+               p = g_strrstr_len(in->buf->str, in->buf->len, inc->termination);
+               if (!p)
+                       return SR_ERR;
+               *p = '\0';
+               p += strlen(inc->termination);
+       }
        g_strstrip(in->buf->str);
 
        ret = SR_OK;
@@ -752,7 +818,7 @@ static int receive(struct sr_input *in, GString *buf)
                return SR_OK;
        }
 
-       ret = process_buffer(in);
+       ret = process_buffer(in, FALSE);
 
        return ret;
 }
@@ -763,7 +829,7 @@ static int end(struct sr_input *in)
        int ret;
 
        if (in->sdi_ready)
-               ret = process_buffer(in);
+               ret = process_buffer(in, TRUE);
        else
                ret = SR_OK;