'JSON to CSV' operation now escapes characters correctly. Added tests for CSV/JSON operations.

src/core/operations/JSONToCSV.mjs

@@ -20,7 +20,7 @@ class JSONToCSV extends Operation {
 
         this.name = "JSON to CSV";
         this.module = "Default";
-        this.description = "Converts JSON data to a CSV.";
+        this.description = "Converts JSON data to a CSV based on the definition in RFC 4180.";
         this.infoURL = "https://wikipedia.org/wiki/Comma-separated_values";
         this.inputType = "JSON";
         this.outputType = "string";
@@ -46,25 +46,65 @@ class JSONToCSV extends Operation {
     run(input, args) {
         const [cellDelim, rowDelim] = args;
 
+        this.cellDelim = cellDelim;
+        this.rowDelim = rowDelim;
+        const self = this;
+
+        // TODO: Escape cells correctly.
+
         try {
             // If the JSON is an array of arrays, this is easy
             if (input[0] instanceof Array) {
-                return input.map(row => row.join(cellDelim)).join(rowDelim) + rowDelim;
+                return input
+                    .map(row => row
+                        .map(self.escapeCellContents.bind(self))
+                        .join(cellDelim)
+                    )
+                    .join(rowDelim) +
+                    rowDelim;
             }
 
             // If it's an array of dictionaries...
             const header = Object.keys(input[0]);
-            return header.join(cellDelim) +
+            return header
+                .map(self.escapeCellContents.bind(self))
+                .join(cellDelim) +
                 rowDelim +
-                input.map(
-                    row => header.map(
-                        h => row[h]
-                    ).join(cellDelim)
-                ).join(rowDelim) +
+                input
+                    .map(row => header
+                        .map(h => row[h])
+                        .map(self.escapeCellContents.bind(self))
+                        .join(cellDelim)
+                    )
+                    .join(rowDelim) +
                 rowDelim;
         } catch (err) {
-            throw new OperationError("Unable to parse JSON to CSV: " + err);
+            throw new OperationError("Unable to parse JSON to CSV: " + err.toString());
+        }
+    }
+
+    /**
+     * Correctly escapes a cell's contents based on the cell and row delimiters.
+     *
+     * @param {string} data
+     * @returns {string}
+     */
+    escapeCellContents(data) {
+        // Double quotes should be doubled up
+        data = data.replace(/"/g, '""');
+
+        // If the cell contains a cell or row delimiter or a double quote, it mut be enclosed in double quotes
+        if (
+            data.indexOf(this.cellDelim) >= 0 ||
+            data.indexOf(this.rowDelim) >= 0 ||
+            data.indexOf("\n") >= 0 ||
+            data.indexOf("\r") >= 0 ||
+            data.indexOf('"') >= 0
+        ) {
+            data = `"${data}"`;
         }
+
+        return data;
     }
 
 }

test/index.mjs

@@ -39,6 +39,7 @@ import "./tests/operations/Comment";
 import "./tests/operations/Compress";
 import "./tests/operations/ConditionalJump";
 import "./tests/operations/Crypt";
+import "./tests/operations/CSV";
 import "./tests/operations/DateTime";
 import "./tests/operations/ExtractEmailAddresses";
 import "./tests/operations/Fork";
@@ -126,12 +127,12 @@ function handleTestResult(testResult) {
 
 
 /**
- * Fail if the process takes longer than 10 seconds.
+ * Fail if the process takes longer than 60 seconds.
  */
 setTimeout(function() {
-    console.log("Tests took longer than 10 seconds to run, returning.");
+    console.log("Tests took longer than 60 seconds to run, returning.");
     process.exit(1);
-}, 10 * 1000);
+}, 60 * 1000);
 
 
 TestRegister.runTests()

test/tests/operations/CSV.mjs

@@ -0,0 +1,179 @@
+/**
+ * CSV tests.
+ *
+ * @author n1474335 [n1474335@gmail.com]
+ *
+ * @copyright Crown Copyright 2018
+ * @license Apache-2.0
+ */
+import TestRegister from "../../TestRegister";
+
+const EXAMPLE_CSV = `A,B,C,D,E,F\r
+1,2,3,4,5,6\r
+",",;,',"""",,\r
+"""hello""","a""1","multi\r
+line",,,end\r
+`;
+
+TestRegister.addTests([
+    {
+        name: "CSV to JSON: Array of dictionaries",
+        input: EXAMPLE_CSV,
+        expectedOutput: JSON.stringify([
+            {
+                "A": "1",
+                "B": "2",
+                "C": "3",
+                "D": "4",
+                "E": "5",
+                "F": "6"
+            },
+            {
+                "A": ",",
+                "B": ";",
+                "C": "'",
+                "D": "\"",
+                "E": "",
+                "F": ""
+            },
+            {
+                "A": "\"hello\"",
+                "B": "a\"1",
+                "C": "multi\r\nline",
+                "D": "",
+                "E": "",
+                "F": "end"
+            }
+        ], null, 4),
+        recipeConfig: [
+            {
+                op: "CSV to JSON",
+                args: [",", "\r\n", "Array of dictionaries"],
+            }
+        ],
+    },
+    {
+        name: "CSV to JSON: Array of arrays",
+        input: EXAMPLE_CSV,
+        expectedOutput: JSON.stringify([
+            [
+                "A",
+                "B",
+                "C",
+                "D",
+                "E",
+                "F"
+            ],
+            [
+                "1",
+                "2",
+                "3",
+                "4",
+                "5",
+                "6"
+            ],
+            [
+                ",",
+                ";",
+                "'",
+                "\"",
+                "",
+                ""
+            ],
+            [
+                "\"hello\"",
+                "a\"1",
+                "multi\r\nline",
+                "",
+                "",
+                "end"
+            ]
+        ], null, 4),
+        recipeConfig: [
+            {
+                op: "CSV to JSON",
+                args: [",", "\r\n", "Array of arrays"],
+            }
+        ],
+    },
+    {
+        name: "JSON to CSV: Array of dictionaries",
+        input: JSON.stringify([
+            {
+                "A": "1",
+                "B": "2",
+                "C": "3",
+                "D": "4",
+                "E": "5",
+                "F": "6"
+            },
+            {
+                "A": ",",
+                "B": ";",
+                "C": "'",
+                "D": "\"",
+                "E": "",
+                "F": ""
+            },
+            {
+                "A": "\"hello\"",
+                "B": "a\"1",
+                "C": "multi\r\nline",
+                "D": "",
+                "E": "",
+                "F": "end"
+            }
+        ]),
+        expectedOutput: EXAMPLE_CSV,
+        recipeConfig: [
+            {
+                op: "JSON to CSV",
+                args: [",", "\r\n"],
+            }
+        ],
+    },
+    {
+        name: "JSON to CSV: Array of arrays",
+        input: JSON.stringify([
+            [
+                "A",
+                "B",
+                "C",
+                "D",
+                "E",
+                "F"
+            ],
+            [
+                "1",
+                "2",
+                "3",
+                "4",
+                "5",
+                "6"
+            ],
+            [
+                ",",
+                ";",
+                "'",
+                "\"",
+                "",
+                ""
+            ],
+            [
+                "\"hello\"",
+                "a\"1",
+                "multi\r\nline",
+                "",
+                "",
+                "end"
+            ]
+        ]),
+        expectedOutput: EXAMPLE_CSV,
+        recipeConfig: [
+            {
+                op: "JSON to CSV",
+                args: [",", "\r\n"],
+            }
+        ],
+    },
+]);