Merge remote-tracking branch 'upstream/master'

.eslintignore

@@ -1 +1,2 @@
 src/core/vendor/**
+src/web/static/clippy_assets/**

.travis.yml

@@ -30,8 +30,9 @@ deploy:
     skip_cleanup: true
     api_key:
       secure: "HV1WSKv4l/0Y2bKKs1iBJocBcmLj08PCRUeEM/jTwA4jqJ8EiLHWiXtER/D5sEg2iibRVKd2OQjfrmS6bo4AiwdeVgAKmv0FtS2Jw+391N8Nd5AkEANHa5Om/IpHLTL2YRAjpJTsDpY72bMUTJIwjQA3TFJkgrpOw6KYfohOcgbxLpZ4XuNJRU3VL4Hsxdv5V9aOVmfFOmMOVPQlakXy7NgtW5POp1f2WJwgcZxylkR1CjwaqMyXmSoVl46pyH3tr5+dptsQoKSGdi6sIHGA60oDotFPcm+0ifa47wZw+vapuuDi4tdNxhrHGaDMG8xiE0WFDHwQUDlk2/+W7j9SEX0H3Em7us371JXRp56EDwEcDa34VpVkC6i8HGcHK55hnxVbMZXGf3qhOFD8wY7qMbjMRvIpucrMHBi86OfkDfv0vDj2LyvIl5APj/AX50BrE0tfH1MZbH26Jkx4NdlkcxQ14GumarmUqfmVvbX/fsoA6oUuAAE9ZgRRi3KHO4wci6KUcRfdm+XOeUkaBFsL86G3EEYIvrtBTuaypdz+Cx7nd1iPZyWMx5Y1gXnVzha4nBdV4+7l9JIsFggD8QVpw2uHXQiS1KXFjOeqA3DBD8tjMB7q26Fl2fD3jkOo4BTbQ2NrRIZUu/iL+fOmMPsyMt2qulB0yaSBCfkbEq8xrUA="
+    file_glob: true
     file:
-      - build/prod/cyberchef.htm
+      - build/prod/*.zip
       - build/node/CyberChef.js
     on:
       repo: gchq/CyberChef

CHANGELOG.md

@@ -2,6 +2,18 @@
 All major and minor version changes will be documented in this file. Details of patch-level version changes can be found in [commit messages](https://github.com/gchq/CyberChef/commits/master).
 
 
+### [8.31.0] - 2019-04-12
+- The downloadable version of CyberChef is now a .zip file containing separate modules rather than a single .htm file. It is still completely standalone and will not make any external network requests. This change reduces the complexity of the build process significantly. [@n1474335]
+
+### [8.30.0] - 2019-04-12
+- 'Decode Protobuf' operation added [@n1474335] | [#533]
+
+### [8.29.0] - 2019-03-31
+- 'BLAKE2s' and 'BLAKE2b' hashing operations added [@h345983745] | [#525]
+
+### [8.28.0] - 2019-03-31
+- 'Heatmap Chart', 'Hex Density Chart', 'Scatter Chart' and 'Series Chart' operation added [@artemisbot] [@tlwr] | [#496] [#143]
+
 ### [8.27.0] - 2019-03-14
 - 'Enigma', 'Typex', 'Bombe' and 'Multiple Bombe' operations added [@s2224834] | [#516]
 - See [this wiki article](https://github.com/gchq/CyberChef/wiki/Enigma,-the-Bombe,-and-Typex) for a full explanation of these operations.
@@ -118,6 +130,10 @@ All major and minor version changes will be documented in this file. Details of
 
 
 
+[8.31.0]: https://github.com/gchq/CyberChef/releases/tag/v8.31.0
+[8.30.0]: https://github.com/gchq/CyberChef/releases/tag/v8.30.0
+[8.29.0]: https://github.com/gchq/CyberChef/releases/tag/v8.29.0
+[8.28.0]: https://github.com/gchq/CyberChef/releases/tag/v8.28.0
 [8.27.0]: https://github.com/gchq/CyberChef/releases/tag/v8.27.0
 [8.26.0]: https://github.com/gchq/CyberChef/releases/tag/v8.26.0
 [8.25.0]: https://github.com/gchq/CyberChef/releases/tag/v8.25.0
@@ -159,6 +175,7 @@ All major and minor version changes will be documented in this file. Details of
 [@h345983745]: https://github.com/h345983745
 [@s2224834]: https://github.com/s2224834
 [@artemisbot]: https://github.com/artemisbot
+[@tlwr]: https://github.com/tlwr
 [@picapi]: https://github.com/picapi
 [@Dachande663]: https://github.com/Dachande663
 [@JustAnotherMark]: https://github.com/JustAnotherMark
@@ -175,6 +192,7 @@ All major and minor version changes will be documented in this file. Details of
 
 [#95]: https://github.com/gchq/CyberChef/pull/299
 [#173]: https://github.com/gchq/CyberChef/pull/173
+[#143]: https://github.com/gchq/CyberChef/pull/143
 [#224]: https://github.com/gchq/CyberChef/pull/224
 [#239]: https://github.com/gchq/CyberChef/pull/239
 [#248]: https://github.com/gchq/CyberChef/pull/248
@@ -209,5 +227,8 @@ All major and minor version changes will be documented in this file. Details of
 [#468]: https://github.com/gchq/CyberChef/pull/468
 [#476]: https://github.com/gchq/CyberChef/pull/476
 [#489]: https://github.com/gchq/CyberChef/pull/489
+[#496]: https://github.com/gchq/CyberChef/pull/496
 [#506]: https://github.com/gchq/CyberChef/pull/506
 [#516]: https://github.com/gchq/CyberChef/pull/516
+[#525]: https://github.com/gchq/CyberChef/pull/525
+[#533]: https://github.com/gchq/CyberChef/pull/533

Gruntfile.js

@@ -4,7 +4,6 @@ const webpack = require("webpack");
 const HtmlWebpackPlugin = require("html-webpack-plugin");
 const BundleAnalyzerPlugin = require("webpack-bundle-analyzer").BundleAnalyzerPlugin;
 const NodeExternals = require("webpack-node-externals");
-const Inliner = require("web-resource-inliner");
 const glob = require("glob");
 const path = require("path");
 
@@ -43,18 +42,16 @@ module.exports = function (grunt) {
 
     grunt.registerTask("prod",
         "Creates a production-ready build. Use the --msg flag to add a compile message.",
-        ["eslint", "clean:prod", "clean:config", "exec:generateConfig", "webpack:web", "inline", "chmod"]);
+        [
+            "eslint", "clean:prod", "clean:config", "exec:generateConfig", "webpack:web",
+            "copy:standalone", "zip:standalone", "clean:standalone", "chmod"
+        ]);
 
     grunt.registerTask("default",
         "Lints the code base",
         ["eslint", "exec:repoSize"]);
 
-    grunt.registerTask("inline",
-        "Compiles a production build of CyberChef into a single, portable web page.",
-        ["exec:generateConfig", "webpack:webInline", "runInliner", "clean:inlineScripts"]);
 
-
-    grunt.registerTask("runInliner", runInliner);
     grunt.registerTask("doc", "docs");
     grunt.registerTask("tests", "test");
     grunt.registerTask("lint", "eslint");
@@ -72,6 +69,7 @@ module.exports = function (grunt) {
     grunt.loadNpmTasks("grunt-accessibility");
     grunt.loadNpmTasks("grunt-concurrent");
     grunt.loadNpmTasks("grunt-contrib-connect");
+    grunt.loadNpmTasks("grunt-zip");
 
 
     // Project configuration
@@ -94,32 +92,6 @@ module.exports = function (grunt) {
         },
         moduleEntryPoints = listEntryModules();
 
-    /**
-     * Compiles a production build of CyberChef into a single, portable web page.
-     */
-    function runInliner() {
-        const done = this.async();
-        Inliner.html({
-            relativeTo: "build/prod/",
-            fileContent: grunt.file.read("build/prod/cyberchef.htm"),
-            images: true,
-            svgs: true,
-            scripts: true,
-            links: true,
-            strict: true
-        }, function(error, result) {
-            if (error) {
-                if (error instanceof Error) {
-                    done(error);
-                } else {
-                    done(new Error(error));
-                }
-            } else {
-                grunt.file.write("build/prod/cyberchef.htm", result);
-                done(true);
-            }
-        });
-    }
 
     /**
      * Generates an entry list for all the modules.
@@ -130,7 +102,7 @@ module.exports = function (grunt) {
         glob.sync("./src/core/config/modules/*.mjs").forEach(file => {
             const basename = path.basename(file);
             if (basename !== "Default.mjs" && basename !== "OpModules.mjs")
-                entryModules[basename.split(".mjs")[0]] = path.resolve(file);
+                entryModules["modules/" + basename.split(".mjs")[0]] = path.resolve(file);
         });
 
         return entryModules;
@@ -143,7 +115,7 @@ module.exports = function (grunt) {
             node: ["build/node/*"],
             config: ["src/core/config/OperationConfig.json", "src/core/config/modules/*", "src/code/operations/index.mjs"],
             docs: ["docs/*", "!docs/*.conf.json", "!docs/*.ico", "!docs/*.png"],
-            inlineScripts: ["build/prod/scripts.js"],
+            standalone: ["build/prod/CyberChef*.html"]
         },
         eslint: {
             options: {
@@ -151,7 +123,7 @@ module.exports = function (grunt) {
             },
             configs: ["*.{js,mjs}"],
             core: ["src/core/**/*.{js,mjs}", "!src/core/vendor/**/*", "!src/core/operations/legacy/**/*"],
-            web: ["src/web/**/*.{js,mjs}"],
+            web: ["src/web/**/*.{js,mjs}", "!src/web/static/**/*"],
             node: ["src/node/**/*.{js,mjs}"],
             tests: ["tests/**/*.{js,mjs}"],
         },
@@ -195,6 +167,9 @@ module.exports = function (grunt) {
                     }, moduleEntryPoints),
                     output: {
                         path: __dirname + "/build/prod",
+                        filename: chunkData => {
+                            return chunkData.chunk.name === "main" ? "assets/[name].js": "[name].js";
+                        },
                         globalObject: "this"
                     },
                     resolve: {
@@ -225,33 +200,6 @@ module.exports = function (grunt) {
                     ]
                 };
             },
-            webInline: {
-                mode: "production",
-                target: "web",
-                entry: "./src/web/index.js",
-                output: {
-                    filename: "scripts.js",
-                    path: __dirname + "/build/prod"
-                },
-                plugins: [
-                    new webpack.DefinePlugin(Object.assign({}, BUILD_CONSTANTS, {
-                        INLINE: "true"
-                    })),
-                    new HtmlWebpackPlugin({
-                        filename: "cyberchef.htm",
-                        template: "./src/web/html/index.html",
-                        compileTime: compileTime,
-                        version: pkg.version + "s",
-                        inline: true,
-                        minify: {
-                            removeComments: true,
-                            collapseWhitespace: true,
-                            minifyJS: true,
-                            minifyCSS: true
-                        }
-                    }),
-                ]
-            },
             node: {
                 mode: "production",
                 target: "node",
@@ -284,7 +232,8 @@ module.exports = function (grunt) {
                     warningsFilter: [
                         /source-map/,
                         /dependency is an expression/,
-                        /export 'default'/
+                        /export 'default'/,
+                        /Can't resolve 'sodium'/
                     ],
                 }
             },
@@ -316,6 +265,18 @@ module.exports = function (grunt) {
                 }
             }
         },
+        zip: {
+            standalone: {
+                cwd: "build/prod/",
+                src: [
+                    "build/prod/**/*",
+                    "!build/prod/index.html",
+                    "!build/prod/BundleAnalyzerReport.html",
+                    "!build/prod/sitemap.js"
+                ],
+                dest: `build/prod/CyberChef_v${pkg.version}.zip`
+            }
+        },
         connect: {
             prod: {
                 options: {
@@ -328,10 +289,16 @@ module.exports = function (grunt) {
             ghPages: {
                 options: {
                     process: function (content, srcpath) {
-                        // Add Google Analytics code to index.html
                         if (srcpath.indexOf("index.html") >= 0) {
+                            // Add Google Analytics code to index.html
                             content = content.replace("</body></html>",
                                 grunt.file.read("src/web/static/ga.html") + "</body></html>");
+
+                            // Add Structured Data for SEO
+                            content = content.replace("</head>",
+                                "<script type='application/ld+json'>" +
+                                JSON.stringify(JSON.parse(grunt.file.read("src/web/static/structuredData.json"))) +
+                                "</script></head>");
                             return grunt.template.process(content, srcpath);
                         } else {
                             return content;
@@ -350,6 +317,28 @@ module.exports = function (grunt) {
                         dest: "build/prod/"
                     },
                 ]
+            },
+            standalone: {
+                options: {
+                    process: function (content, srcpath) {
+                        if (srcpath.indexOf("index.html") >= 0) {
+                            // Replace download link with version number
+                            content = content.replace(/<a [^>]+>Download CyberChef.+?<\/a>/,
+                                `<span>Version ${pkg.version}</span>`);
+
+                            return grunt.template.process(content, srcpath);
+                        } else {
+                            return content;
+                        }
+                    },
+                    noProcess: ["**", "!**/*.html"]
+                },
+                files: [
+                    {
+                        src: "build/prod/index.html",
+                        dest: `build/prod/CyberChef_v${pkg.version}.html`
+                    }
+                ]
             }
         },
         chmod: {
@@ -405,7 +394,7 @@ module.exports = function (grunt) {
                 command: "node --experimental-modules --no-warnings --no-deprecation tests/operations/index.mjs"
             },
             browserTests: {
-                command: "./node_modules/.bin/nightwatch --env prod,inline"
+                command: "./node_modules/.bin/nightwatch --env prod"
             }
         },
     });

babel.config.js

@@ -11,14 +11,22 @@ module.exports = function(api) {
                     "node": "6.5"
                 },
                 "modules": false,
-                "useBuiltIns": "entry"
+                "useBuiltIns": "entry",
+                "corejs": 3
             }]
         ],
         "plugins": [
             "babel-plugin-syntax-dynamic-import",
-            ["babel-plugin-transform-builtin-extend", {
-                "globals": ["Error"]
-            }]
+            [
+                "babel-plugin-transform-builtin-extend", {
+                    "globals": ["Error"]
+                }
+            ],
+            [
+                "@babel/plugin-transform-runtime", {
+                    "regenerator": true
+                }
+            ]
         ]
     };
 };

nightwatch.json

@@ -23,10 +23,6 @@
 
     "prod": {
       "launch_url": "http://localhost:8000/index.html"
-    },
-
-    "inline": {
-      "launch_url": "http://localhost:8000/cyberchef.htm"
     }
 
   }

package.json

@@ -1,6 +1,6 @@
 {
   "name": "cyberchef",
-  "version": "8.27.0",
+  "version": "8.31.1",
   "description": "The Cyber Swiss Army Knife for encryption, encoding, compression and data analysis.",
   "author": "n1474335 <n1474335@gmail.com>",
   "homepage": "https://gchq.github.io/CyberChef",
@@ -30,20 +30,20 @@
   "main": "build/node/CyberChef.js",
   "bugs": "https://github.com/gchq/CyberChef/issues",
   "devDependencies": {
-    "@babel/core": "^7.2.2",
-    "@babel/preset-env": "^7.2.3",
-    "autoprefixer": "^9.4.3",
+    "@babel/core": "^7.4.3",
+    "@babel/plugin-transform-runtime": "^7.4.3",
+    "@babel/preset-env": "^7.4.3",
+    "autoprefixer": "^9.5.1",
     "babel-eslint": "^10.0.1",
-    "babel-loader": "^8.0.4",
+    "babel-loader": "^8.0.5",
     "babel-plugin-syntax-dynamic-import": "^6.18.0",
-    "bootstrap": "^4.2.1",
-    "chromedriver": "^2.45.0",
+    "chromedriver": "^2.46.0",
     "colors": "^1.3.3",
-    "css-loader": "^2.1.0",
-    "eslint": "^5.12.1",
+    "css-loader": "^2.1.1",
+    "eslint": "^5.16.0",
     "exports-loader": "^0.7.0",
     "file-loader": "^3.0.1",
-    "grunt": "^1.0.3",
+    "grunt": "^1.0.4",
     "grunt-accessibility": "~6.0.0",
     "grunt-chmod": "~1.1.1",
     "grunt-concurrent": "^2.3.1",
@@ -53,16 +53,17 @@
     "grunt-contrib-watch": "^1.1.0",
     "grunt-eslint": "^21.0.0",
     "grunt-exec": "~3.0.0",
-    "grunt-jsdoc": "^2.3.0",
+    "grunt-jsdoc": "^2.3.1",
     "grunt-webpack": "^3.1.3",
+    "grunt-zip": "^0.18.2",
     "html-webpack-plugin": "^3.2.0",
     "imports-loader": "^0.8.0",
     "ink-docstrap": "^1.3.2",
     "jsdoc-babel": "^0.5.0",
-    "mini-css-extract-plugin": "^0.5.0",
-    "nightwatch": "^1.0.18",
+    "mini-css-extract-plugin": "^0.6.0",
+    "nightwatch": "^1.0.19",
     "node-sass": "^4.11.0",
-    "postcss-css-variables": "^0.11.0",
+    "postcss-css-variables": "^0.12.0",
     "postcss-import": "^12.0.1",
     "postcss-loader": "^3.0.0",
     "prompt": "^1.0.0",
@@ -71,62 +72,69 @@
     "style-loader": "^0.23.1",
     "svg-url-loader": "^2.3.2",
     "url-loader": "^1.1.2",
-    "web-resource-inliner": "^4.2.1",
-    "webpack": "^4.28.3",
-    "webpack-bundle-analyzer": "^3.0.3",
-    "webpack-dev-server": "^3.1.14",
+    "webpack": "^4.29.6",
+    "webpack-bundle-analyzer": "^3.3.2",
+    "webpack-dev-server": "^3.3.1",
     "webpack-node-externals": "^1.7.2",
     "worker-loader": "^2.0.0"
   },
   "dependencies": {
+    "@babel/polyfill": "^7.4.3",
+    "@babel/runtime": "^7.4.3",
     "arrive": "^2.4.1",
     "babel-plugin-transform-builtin-extend": "1.1.2",
-    "babel-polyfill": "^6.26.0",
     "bcryptjs": "^2.4.3",
-    "bignumber.js": "^8.0.2",
+    "bignumber.js": "^8.1.1",
+    "blakejs": "^1.1.0",
+    "bootstrap": "4.2.1",
     "bootstrap-colorpicker": "^2.5.3",
     "bootstrap-material-design": "^4.1.1",
-    "bson": "^4.0.1",
+    "bson": "^4.0.2",
     "chi-squared": "^1.1.0",
+    "clippyjs": "0.0.3",
+    "core-js": "^3.0.1",
     "crypto-api": "^0.8.3",
     "crypto-js": "^3.1.9-1",
     "ctph.js": "0.0.5",
-    "diff": "^3.5.0",
+    "d3": "^4.9.1",
+    "d3-hexbin": "^0.2.2",
+    "diff": "^4.0.1",
     "es6-promisify": "^6.0.1",
-    "escodegen": "^1.11.0",
+    "escodegen": "^1.11.1",
     "esmangle": "^1.0.1",
     "esprima": "^4.0.1",
     "exif-parser": "^0.1.12",
-    "file-saver": "^2.0.0",
+    "file-saver": "^2.0.1",
     "geodesy": "^1.1.3",
-    "highlight.js": "^9.13.1",
-    "jimp": "^0.6.0",
-    "jquery": "^3.3.1",
+    "highlight.js": "^9.15.6",
+    "jimp": "^0.6.1",
+    "jquery": "3.3.1",
     "js-crc": "^0.2.0",
     "js-sha3": "^0.8.0",
     "jsesc": "^2.5.2",
-    "jsonpath": "^1.0.0",
-    "jsonwebtoken": "^8.4.0",
-    "jsqr": "^1.1.1",
+    "jsonpath": "^1.0.1",
+    "jsonwebtoken": "^8.5.1",
+    "jsqr": "^1.2.0",
     "jsrsasign": "8.0.12",
-    "kbpgp": "^2.0.82",
+    "kbpgp": "2.1.0",
     "libyara-wasm": "0.0.12",
     "lodash": "^4.17.11",
     "loglevel": "^1.6.1",
     "loglevel-message-prefix": "^3.0.0",
-    "moment": "^2.23.0",
+    "moment": "^2.24.0",
     "moment-timezone": "^0.5.23",
     "ngeohash": "^0.6.3",
-    "node-forge": "^0.7.6",
+    "node-forge": "^0.8.2",
     "node-md6": "^0.1.0",
+    "nodom": "^2.2.0",
     "notepack.io": "^2.2.0",
     "nwmatcher": "^1.4.4",
     "otp": "^0.1.3",
-    "popper.js": "^1.14.6",
+    "popper.js": "^1.15.0",
     "qr-image": "^3.2.0",
     "scryptsy": "^2.0.0",
     "snackbarjs": "^1.1.0",
-    "sortablejs": "^1.8.0-rc1",
+    "sortablejs": "^1.8.4",
     "split.js": "^1.5.10",
     "ssdeep.js": "0.0.2",
     "ua-parser-js": "^0.7.19",

src/core/ChefWorker.js

@@ -6,7 +6,6 @@
  * @license Apache-2.0
  */
 
-import "babel-polyfill";
 import Chef from "./Chef";
 import OperationConfig from "./config/OperationConfig.json";
 import OpModules from "./config/modules/OpModules";
@@ -179,7 +178,7 @@ self.loadRequiredModules = function(recipeConfig) {
         if (!OpModules.hasOwnProperty(module)) {
             log.info(`Loading ${module} module`);
             self.sendStatusMessage(`Loading ${module} module`);
-            self.importScripts(`${self.docURL}/${module}.js`);
+            self.importScripts(`${self.docURL}/modules/${module}.js`);
             self.sendStatusMessage("");
         }
     });

src/core/Dish.mjs

@@ -21,8 +21,8 @@ class Dish {
      * @param {Dish} [dish=null] - A dish to clone
      */
     constructor(dish=null) {
-        this.value = [];
-        this.type = Dish.BYTE_ARRAY;
+        this.value = new ArrayBuffer(0);
+        this.type = Dish.ARRAY_BUFFER;
 
         if (dish &&
             dish.hasOwnProperty("value") &&
@@ -149,78 +149,75 @@ class Dish {
      */
     async _translate(toType, notUTF8=false) {
         log.debug(`Translating Dish from ${Dish.enumLookup(this.type)} to ${Dish.enumLookup(toType)}`);
-        const byteArrayToStr = notUTF8 ? Utils.byteArrayToChars : Utils.byteArrayToUtf8;
 
-        // Convert data to intermediate byteArray type
+        // Convert data to intermediate ArrayBuffer type
         try {
             switch (this.type) {
                 case Dish.STRING:
-                    this.value = this.value ? Utils.strToByteArray(this.value) : [];
+                    this.value = this.value ? Utils.strToArrayBuffer(this.value) : new ArrayBuffer;
                     break;
                 case Dish.NUMBER:
-                    this.value = typeof this.value === "number" ? Utils.strToByteArray(this.value.toString()) : [];
+                    this.value = typeof this.value === "number" ? Utils.strToArrayBuffer(this.value.toString()) : new ArrayBuffer;
                     break;
                 case Dish.HTML:
-                    this.value = this.value ? Utils.strToByteArray(Utils.unescapeHtml(Utils.stripHtmlTags(this.value, true))) : [];
+                    this.value = this.value ? Utils.strToArrayBuffer(Utils.unescapeHtml(Utils.stripHtmlTags(this.value, true))) : new ArrayBuffer;
                     break;
-                case Dish.ARRAY_BUFFER:
-                    // Array.from() would be nicer here, but it's slightly slower
-                    this.value = Array.prototype.slice.call(new Uint8Array(this.value));
+                case Dish.BYTE_ARRAY:
+                    this.value = new Uint8Array(this.value).buffer;
                     break;
                 case Dish.BIG_NUMBER:
-                    this.value = BigNumber.isBigNumber(this.value) ? Utils.strToByteArray(this.value.toFixed()) : [];
+                    this.value = BigNumber.isBigNumber(this.value) ? Utils.strToArrayBuffer(this.value.toFixed()) : new ArrayBuffer;
                     break;
                 case Dish.JSON:
-                    this.value = this.value ? Utils.strToByteArray(JSON.stringify(this.value, null, 4)) : [];
+                    this.value = this.value ? Utils.strToArrayBuffer(JSON.stringify(this.value, null, 4)) : new ArrayBuffer;
                     break;
                 case Dish.FILE:
-                    this.value = await Utils.readFile(this.value);
-                    this.value = Array.prototype.slice.call(this.value);
+                    this.value = (await Utils.readFile(this.value)).buffer;
                     break;
                 case Dish.LIST_FILE:
                     this.value = await Promise.all(this.value.map(async f => Utils.readFile(f)));
-                    this.value = this.value.map(b => Array.prototype.slice.call(b));
-                    this.value = [].concat.apply([], this.value);
+                    this.value = concatenateTypedArrays(...this.value).buffer;
                     break;
                 default:
                     break;
             }
         } catch (err) {
-            throw new DishError(`Error translating from ${Dish.enumLookup(this.type)} to byteArray: ${err}`);
+            throw new DishError(`Error translating from ${Dish.enumLookup(this.type)} to ArrayBuffer: ${err}`);
         }
 
-        this.type = Dish.BYTE_ARRAY;
+        this.type = Dish.ARRAY_BUFFER;
 
-        // Convert from byteArray to toType
+        // Convert from ArrayBuffer to toType
         try {
             switch (toType) {
                 case Dish.STRING:
                 case Dish.HTML:
-                    this.value = this.value ? byteArrayToStr(this.value) : "";
+                    this.value = this.value ? Utils.arrayBufferToStr(this.value, !notUTF8) : "";
                     this.type = Dish.STRING;
                     break;
                 case Dish.NUMBER:
-                    this.value = this.value ? parseFloat(byteArrayToStr(this.value)) : 0;
+                    this.value = this.value ? parseFloat(Utils.arrayBufferToStr(this.value, !notUTF8)) : 0;
                     this.type = Dish.NUMBER;
                     break;
-                case Dish.ARRAY_BUFFER:
-                    this.value = new Uint8Array(this.value).buffer;
+                case Dish.BYTE_ARRAY:
+                    this.value = Array.prototype.slice.call(new Uint8Array(this.value));
                     this.type = Dish.ARRAY_BUFFER;
                     break;
                 case Dish.BIG_NUMBER:
                     try {
-                        this.value = new BigNumber(byteArrayToStr(this.value));
+                        this.value = new BigNumber(Utils.arrayBufferToStr(this.value, !notUTF8));
                     } catch (err) {
                         this.value = new BigNumber(NaN);
                     }
                     this.type = Dish.BIG_NUMBER;
                     break;
                 case Dish.JSON:
-                    this.value = JSON.parse(byteArrayToStr(this.value));
+                    this.value = JSON.parse(Utils.arrayBufferToStr(this.value, !notUTF8));
                     this.type = Dish.JSON;
                     break;
                 case Dish.FILE:
                     this.value = new File(this.value, "unknown");
+                    this.type = Dish.FILE;
                     break;
                 case Dish.LIST_FILE:
                     this.value = [new File(this.value, "unknown")];
@@ -230,7 +227,7 @@ class Dish {
                     break;
             }
         } catch (err) {
-            throw new DishError(`Error translating from byteArray to ${Dish.enumLookup(toType)}: ${err}`);
+            throw new DishError(`Error translating from ArrayBuffer to ${Dish.enumLookup(toType)}: ${err}`);
         }
     }
 
@@ -374,6 +371,26 @@ class Dish {
 
 }
 
+/**
+ * Concatenates a list of Uint8Arrays together
+ *
+ * @param {Uint8Array[]} arrays
+ * @returns {Uint8Array}
+ */
+function concatenateTypedArrays(...arrays) {
+    let totalLength = 0;
+    for (const arr of arrays) {
+        totalLength += arr.length;
+    }
+    const result = new Uint8Array(totalLength);
+    let offset = 0;
+    for (const arr of arrays) {
+        result.set(arr, offset);
+        offset += arr.length;
+    }
+    return result;
+}
+
 
 /**
  * Dish data type enum for byte arrays.

src/core/Utils.mjs

@@ -367,6 +367,61 @@ class Utils {
     }
 
 
+    /**
+     * Converts a string to an ArrayBuffer.
+     * Treats the string as UTF-8 if any values are over 255.
+     *
+     * @param {string} str
+     * @returns {ArrayBuffer}
+     *
+     * @example
+     * // returns [72,101,108,108,111]
+     * Utils.strToArrayBuffer("Hello");
+     *
+     * // returns [228,189,160,229,165,189]
+     * Utils.strToArrayBuffer("你好");
+     */
+    static strToArrayBuffer(str) {
+        const arr = new Uint8Array(str.length);
+        let i = str.length, b;
+        while (i--) {
+            b = str.charCodeAt(i);
+            arr[i] = b;
+            // If any of the bytes are over 255, read as UTF-8
+            if (b > 255) return Utils.strToUtf8ArrayBuffer(str);
+        }
+        return arr.buffer;
+    }
+
+
+    /**
+     * Converts a string to a UTF-8 ArrayBuffer.
+     *
+     * @param {string} str
+     * @returns {ArrayBuffer}
+     *
+     * @example
+     * // returns [72,101,108,108,111]
+     * Utils.strToUtf8ArrayBuffer("Hello");
+     *
+     * // returns [228,189,160,229,165,189]
+     * Utils.strToUtf8ArrayBuffer("你好");
+     */
+    static strToUtf8ArrayBuffer(str) {
+        const utf8Str = utf8.encode(str);
+
+        if (str.length !== utf8Str.length) {
+            if (ENVIRONMENT_IS_WORKER()) {
+                self.setOption("attemptHighlight", false);
+            } else if (ENVIRONMENT_IS_WEB()) {
+                window.app.options.attemptHighlight = false;
+            }
+        }
+
+        return Utils.strToArrayBuffer(utf8Str);
+    }
+
+
     /**
      * Converts a string to a byte array.
      * Treats the string as UTF-8 if any values are over 255.
@@ -459,7 +514,7 @@ class Utils {
     /**
      * Attempts to convert a byte array to a UTF-8 string.
      *
-     * @param {byteArray} byteArray
+     * @param {byteArray|Uint8Array} byteArray
      * @returns {string}
      *
      * @example
@@ -505,6 +560,7 @@ class Utils {
     static byteArrayToChars(byteArray) {
         if (!byteArray) return "";
         let str = "";
+        // String concatenation appears to be faster than an array join
         for (let i = 0; i < byteArray.length;) {
             str += String.fromCharCode(byteArray[i++]);
         }
@@ -524,8 +580,8 @@ class Utils {
      * Utils.arrayBufferToStr(Uint8Array.from([104,101,108,108,111]).buffer);
      */
     static arrayBufferToStr(arrayBuffer, utf8=true) {
-        const byteArray = Array.prototype.slice.call(new Uint8Array(arrayBuffer));
-        return utf8 ? Utils.byteArrayToUtf8(byteArray) : Utils.byteArrayToChars(byteArray);
+        const arr = new Uint8Array(arrayBuffer);
+        return utf8 ? Utils.byteArrayToUtf8(arr) : Utils.byteArrayToChars(arr);
     }
 
 
@@ -1023,9 +1079,11 @@ class Utils {
     static charRep(token) {
         return {
             "Space":         " ",
+            "Percent":       "%",
             "Comma":         ",",
             "Semi-colon":    ";",
             "Colon":         ":",
+            "Tab":           "\t",
             "Line feed":     "\n",
             "CRLF":          "\r\n",
             "Forward slash": "/",
@@ -1047,6 +1105,7 @@ class Utils {
     static regexRep(token) {
         return {
             "Space":         /\s+/g,
+            "Percent":       /%/g,
             "Comma":         /,/g,
             "Semi-colon":    /;/g,
             "Colon":         /:/g,

src/core/config/Categories.json

@@ -169,6 +169,9 @@
             "Parse URI",
             "URL Encode",
             "URL Decode",
+            "Protobuf Decode",
+            "VarInt Encode",
+            "VarInt Decode",
             "Format MAC addresses",
             "Change IP format",
             "Group IP addresses",
@@ -297,6 +300,8 @@
             "HAS-160",
             "Whirlpool",
             "Snefru",
+            "BLAKE2b",
+            "BLAKE2s",
             "SSDEEP",
             "CTPH",
             "Compare SSDEEP hashes",
@@ -378,7 +383,11 @@
             "Image Filter",
             "Contain Image",
             "Cover Image",
-            "Image Hue/Saturation/Lightness"
+            "Image Hue/Saturation/Lightness",
+            "Hex Density chart",
+            "Scatter chart",
+            "Series chart",
+            "Heatmap chart"
         ]
     },
     {
@@ -395,6 +404,7 @@
             "Generate QR Code",
             "Parse QR Code",
             "Haversine distance",
+            "HTML To Text",
             "Generate Lorem Ipsum",
             "Numberwang",
             "XKCD Random Number"

src/core/lib/Charts.mjs

@@ -0,0 +1,178 @@
+/**
+ * @author tlwr [toby@toby.codes]
+ * @author Matt C [me@mitt.dev]
+ * @copyright Crown Copyright 2019
+ * @license Apache-2.0
+ */
+
+import OperationError from "../errors/OperationError";
+
+/**
+ * @constant
+ * @default
+ */
+export const RECORD_DELIMITER_OPTIONS = ["Line feed", "CRLF"];
+
+
+/**
+ * @constant
+ * @default
+ */
+export const FIELD_DELIMITER_OPTIONS = ["Space", "Comma", "Semi-colon", "Colon", "Tab"];
+
+
+/**
+ * Default from colour
+ *
+ * @constant
+ * @default
+ */
+export const COLOURS = {
+    min: "white",
+    max: "black"
+};
+
+
+/**
+ * Gets values from input for a plot.
+ *
+ * @param {string} input
+ * @param {string} recordDelimiter
+ * @param {string} fieldDelimiter
+ * @param {boolean} columnHeadingsAreIncluded - whether we should skip the first record
+ * @param {number} length
+ * @returns {Object[]}
+ */
+export function getValues(input, recordDelimiter, fieldDelimiter, columnHeadingsAreIncluded, length) {
+    let headings;
+    const values = [];
+
+    input
+        .split(recordDelimiter)
+        .forEach((row, rowIndex) => {
+            const split = row.split(fieldDelimiter);
+            if (split.length !== length) throw new OperationError(`Each row must have length ${length}.`);
+
+            if (columnHeadingsAreIncluded && rowIndex === 0) {
+                headings = split;
+            } else {
+                values.push(split);
+            }
+        });
+    return { headings, values };
+}
+
+
+/**
+ * Gets values from input for a scatter plot.
+ *
+ * @param {string} input
+ * @param {string} recordDelimiter
+ * @param {string} fieldDelimiter
+ * @param {boolean} columnHeadingsAreIncluded - whether we should skip the first record
+ * @returns {Object[]}
+ */
+export function getScatterValues(input, recordDelimiter, fieldDelimiter, columnHeadingsAreIncluded) {
+    let { headings, values } = getValues(
+        input,
+        recordDelimiter,
+        fieldDelimiter,
+        columnHeadingsAreIncluded,
+        2
+    );
+
+    if (headings) {
+        headings = {x: headings[0], y: headings[1]};
+    }
+
+    values = values.map(row => {
+        const x = parseFloat(row[0], 10),
+            y = parseFloat(row[1], 10);
+
+        if (Number.isNaN(x)) throw new OperationError("Values must be numbers in base 10.");
+        if (Number.isNaN(y)) throw new OperationError("Values must be numbers in base 10.");
+
+        return [x, y];
+    });
+
+    return { headings, values };
+}
+
+
+/**
+ * Gets values from input for a scatter plot with colour from the third column.
+ *
+ * @param {string} input
+ * @param {string} recordDelimiter
+ * @param {string} fieldDelimiter
+ * @param {boolean} columnHeadingsAreIncluded - whether we should skip the first record
+ * @returns {Object[]}
+ */
+export function getScatterValuesWithColour(input, recordDelimiter, fieldDelimiter, columnHeadingsAreIncluded) {
+    let { headings, values } = getValues(
+        input,
+        recordDelimiter, fieldDelimiter,
+        columnHeadingsAreIncluded,
+        3
+    );
+
+    if (headings) {
+        headings = {x: headings[0], y: headings[1]};
+    }
+
+    values = values.map(row => {
+        const x = parseFloat(row[0], 10),
+            y = parseFloat(row[1], 10),
+            colour = row[2];
+
+        if (Number.isNaN(x)) throw new OperationError("Values must be numbers in base 10.");
+        if (Number.isNaN(y)) throw new OperationError("Values must be numbers in base 10.");
+
+        return [x, y, colour];
+    });
+
+    return { headings, values };
+}
+
+/**
+ * Gets values from input for a time series plot.
+ *
+ * @param {string} input
+ * @param {string} recordDelimiter
+ * @param {string} fieldDelimiter
+ * @param {boolean} columnHeadingsAreIncluded - whether we should skip the first record
+ * @returns {Object[]}
+ */
+export function getSeriesValues(input, recordDelimiter, fieldDelimiter, columnHeadingsAreIncluded) {
+    const { values } = getValues(
+        input,
+        recordDelimiter, fieldDelimiter,
+        false,
+        3
+    );
+
+    let xValues = new Set();
+    const series = {};
+
+    values.forEach(row => {
+        const serie = row[0],
+            xVal = row[1],
+            val = parseFloat(row[2], 10);
+
+        if (Number.isNaN(val)) throw new OperationError("Values must be numbers in base 10.");
+
+        xValues.add(xVal);
+        if (typeof series[serie] === "undefined") series[serie] = {};
+        series[serie][xVal] = val;
+    });
+
+    xValues = new Array(...xValues);
+
+    const seriesList = [];
+    for (const seriesName in series) {
+        const serie = series[seriesName];
+        seriesList.push({name: seriesName, data: serie});
+    }
+
+    return { xValues, series: seriesList };
+}

src/core/lib/Hex.mjs

@@ -100,7 +100,7 @@ export function fromHex(data, delim="Auto", byteLen=2) {
 /**
  * To Hexadecimal delimiters.
  */
-export const TO_HEX_DELIM_OPTIONS = ["Space", "Comma", "Semi-colon", "Colon", "Line feed", "CRLF", "0x", "\\x", "None"];
+export const TO_HEX_DELIM_OPTIONS = ["Space", "Percent", "Comma", "Semi-colon", "Colon", "Line feed", "CRLF", "0x", "\\x", "None"];
 
 
 /**

src/core/lib/Protobuf.mjs

@@ -0,0 +1,285 @@
+import Utils from "../Utils";
+
+/**
+ * Protobuf lib. Contains functions to decode protobuf serialised
+ * data without a schema or .proto file.
+ *
+ * Provides utility functions to encode and decode variable length
+ * integers (varint).
+ *
+ * @author GCHQ Contributor [3]
+ * @copyright Crown Copyright 2019
+ * @license Apache-2.0
+ */
+class Protobuf {
+
+    /**
+     * Protobuf constructor
+     *
+     * @param {byteArray} data
+     */
+    constructor(data) {
+        // Check we have a byteArray
+        if (data instanceof Array) {
+            this.data = data;
+        } else {
+            throw new Error("Protobuf input must be a byteArray");
+        }
+
+        // Set up masks
+        this.TYPE = 0x07;
+        this.NUMBER = 0x78;
+        this.MSB = 0x80;
+        this.VALUE = 0x7f;
+
+        // Declare offset and length
+        this.offset = 0;
+        this.LENGTH = data.length;
+    }
+
+    // Public Functions
+
+    /**
+     * Encode a varint from a number
+     *
+     * @param {number} number
+     * @returns {byteArray}
+     */
+    static varIntEncode(number) {
+        const MSB = 0x80,
+            VALUE = 0x7f,
+            MSBALL = ~VALUE,
+            INT = Math.pow(2, 31);
+        const out = [];
+        let offset = 0;
+
+        while (number >= INT) {
+            out[offset++] = (number & 0xff) | MSB;
+            number /= 128;
+        }
+        while (number & MSBALL) {
+            out[offset++] = (number & 0xff) | MSB;
+            number >>>= 7;
+        }
+        out[offset] = number | 0;
+        return out;
+    }
+
+    /**
+     * Decode a varint from the byteArray
+     *
+     * @param {byteArray} input
+     * @returns {number}
+     */
+    static varIntDecode(input) {
+        const pb = new Protobuf(input);
+        return pb._varInt();
+    }
+
+    /**
+     * Parse Protobuf data
+     *
+     * @param {byteArray} input
+     * @returns {Object}
+     */
+    static decode(input) {
+        const pb = new Protobuf(input);
+        return pb._parse();
+    }
+
+    // Private Class Functions
+
+    /**
+     * Main private parsing function
+     *
+     * @private
+     * @returns {Object}
+     */
+    _parse() {
+        let object = {};
+        // Continue reading whilst we still have data
+        while (this.offset < this.LENGTH) {
+            const field = this._parseField();
+            object = this._addField(field, object);
+        }
+        // Throw an error if we have gone beyond the end of the data
+        if (this.offset > this.LENGTH) {
+            throw new Error("Exhausted Buffer");
+        }
+        return object;
+    }
+
+    /**
+     * Add a field read from the protobuf data into the Object. As
+     * protobuf fields can appear multiple times, if the field already
+     * exists we need to add the new field into an array of fields
+     * for that key.
+     *
+     * @private
+     * @param {Object} field
+     * @param {Object} object
+     * @returns {Object}
+     */
+    _addField(field, object) {
+        // Get the field key/values
+        const key = field.key;
+        const value = field.value;
+        object[key] = object.hasOwnProperty(key) ?
+            object[key] instanceof Array ?
+                object[key].concat([value]) :
+                [object[key], value] :
+            value;
+        return object;
+    }
+
+    /**
+     * Parse a field and return the Object read from the record
+     *
+     * @private
+     * @returns {Object}
+     */
+    _parseField() {
+        // Get the field headers
+        const header = this._fieldHeader();
+        const type = header.type;
+        const key = header.key;
+        switch (type) {
+            // varint
+            case 0:
+                return { "key": key, "value": this._varInt() };
+            // fixed 64
+            case 1:
+                return { "key": key, "value": this._uint64() };
+            // length delimited
+            case 2:
+                return { "key": key, "value": this._lenDelim() };
+            // fixed 32
+            case 5:
+                return { "key": key, "value": this._uint32() };
+            // unknown type
+            default:
+                throw new Error("Unknown type 0x" + type.toString(16));
+        }
+    }
+
+    /**
+     * Parse the field header and return the type and key
+     *
+     * @private
+     * @returns {Object}
+     */
+    _fieldHeader() {
+        // Make sure we call type then number to preserve offset
+        return { "type": this._fieldType(), "key": this._fieldNumber() };
+    }
+
+    /**
+     * Parse the field type from the field header. Type is stored in the
+     * lower 3 bits of the tag byte. This does not move the offset on as
+     * we need to read the field number from the tag byte too.
+     *
+     * @private
+     * @returns {number}
+     */
+    _fieldType() {
+        // Field type stored in lower 3 bits of tag byte
+        return this.data[this.offset] & this.TYPE;
+    }
+
+    /**
+     * Parse the field number (i.e. the key) from the field header. The
+     * field number is stored in the upper 5 bits of the tag byte - but
+     * is also varint encoded so the follow on bytes may need to be read
+     * when field numbers are > 15.
+     *
+     * @private
+     * @returns {number}
+     */
+    _fieldNumber() {
+        let shift = -3;
+        let fieldNumber = 0;
+        do {
+            fieldNumber += shift < 28 ?
+                shift === -3 ?
+                    (this.data[this.offset] & this.NUMBER) >> -shift :
+                    (this.data[this.offset] & this.VALUE) << shift :
+                (this.data[this.offset] & this.VALUE) * Math.pow(2, shift);
+            shift += 7;
+        } while ((this.data[this.offset++] & this.MSD) === this.MSB);
+        return fieldNumber;
+    }
+
+    // Field Parsing Functions
+
+    /**
+     * Read off a varint from the data
+     *
+     * @private
+     * @returns {number}
+     */
+    _varInt() {
+        let value = 0;
+        let shift = 0;
+        // Keep reading while upper bit set
+        do {
+            value += shift < 28 ?
+                (this.data[this.offset] & this.VALUE) << shift :
+                (this.data[this.offset] & this.VALUE) * Math.pow(2, shift);
+            shift += 7;
+        } while ((this.data[this.offset++] & this.MSB) === this.MSB);
+        return value;
+    }
+
+    /**
+     * Read off a 64 bit unsigned integer from the data
+     *
+     * @private
+     * @returns {number}
+     */
+    _uint64() {
+        // Read off a Uint64
+        let num = this.data[this.offset++] * 0x1000000 + (this.data[this.offset++] << 16) + (this.data[this.offset++] << 8) + this.data[this.offset++];
+        num = num * 0x100000000 + this.data[this.offset++] * 0x1000000 + (this.data[this.offset++] << 16) + (this.data[this.offset++] << 8) + this.data[this.offset++];
+        return num;
+    }
+
+    /**
+     * Read off a length delimited field from the data
+     *
+     * @private
+     * @returns {Object|string}
+     */
+    _lenDelim() {
+        // Read off the field length
+        const length = this._varInt();
+        const fieldBytes = this.data.slice(this.offset, this.offset + length);
+        let field;
+        try {
+            // Attempt to parse as a new Protobuf Object
+            const pbObject = new Protobuf(fieldBytes);
+            field = pbObject._parse();
+        } catch (err) {
+            // Otherwise treat as bytes
+            field = Utils.byteArrayToChars(fieldBytes);
+        }
+        // Move the offset and return the field
+        this.offset += length;
+        return field;
+    }
+
+    /**
+     * Read a 32 bit unsigned integer from the data
+     *
+     * @private
+     * @returns {number}
+     */
+    _uint32() {
+        // Use a dataview to read off the integer
+        const dataview = new DataView(new Uint8Array(this.data.slice(this.offset, this.offset + 4)).buffer);
+        const value = dataview.getUint32(0);
+        this.offset += 4;
+        return value;
+    }
+}
+
+export default Protobuf;

src/core/operations/BLAKE2b.mjs

@@ -0,0 +1,79 @@
+/**
+ * @author h345983745
+ * @copyright Crown Copyright 2019
+ * @license Apache-2.0
+ */
+
+import Operation from "../Operation";
+import blakejs from "blakejs";
+import OperationError from "../errors/OperationError";
+import Utils from "../Utils";
+import { toBase64 } from "../lib/Base64";
+
+/**
+ * BLAKE2b operation
+ */
+class BLAKE2b extends Operation {
+
+    /**
+     * BLAKE2b constructor
+     */
+    constructor() {
+        super();
+
+        this.name = "BLAKE2b";
+        this.module = "Hashing";
+        this.description = `Performs BLAKE2b hashing on the input.  
+        <br><br> BLAKE2b is a flavour of the BLAKE cryptographic hash function that is optimized for 64-bit platforms and produces digests of any size between 1 and 64 bytes.
+        <br><br> Supports the use of an optional key.`;
+        this.infoURL = "https://wikipedia.org/wiki/BLAKE_(hash_function)#BLAKE2b_algorithm";
+        this.inputType = "ArrayBuffer";
+        this.outputType = "string";
+        this.args = [
+            {
+                "name": "Size",
+                "type": "option",
+                "value": ["512", "384", "256", "160", "128"]
+            }, {
+                "name": "Output Encoding",
+                "type": "option",
+                "value": ["Hex", "Base64", "Raw"]
+            }, {
+                "name": "Key",
+                "type": "toggleString",
+                "value": "",
+                "toggleValues": ["UTF8", "Decimal", "Base64", "Hex", "Latin1"]
+            }
+        ];
+    }
+
+    /**
+     * @param {ArrayBuffer} input
+     * @param {Object[]} args
+     * @returns {string} The input having been hashed with BLAKE2b in the encoding format speicifed.
+     */
+    run(input, args) {
+        const [outSize, outFormat] = args;
+        let key = Utils.convertToByteArray(args[2].string || "", args[2].option);
+        if (key.length === 0) {
+            key = null;
+        } else if (key.length > 64) {
+            throw new OperationError(["Key cannot be greater than 64 bytes", "It is currently " + key.length + " bytes."].join("\n"));
+        }
+
+        input = new Uint8Array(input);
+        switch (outFormat) {
+            case "Hex":
+                return blakejs.blake2bHex(input, key, outSize / 8);
+            case "Base64":
+                return toBase64(blakejs.blake2b(input, key, outSize / 8));
+            case "Raw":
+                return Utils.arrayBufferToStr(blakejs.blake2b(input, key, outSize / 8).buffer);
+            default:
+                return new OperationError("Unsupported Output Type");
+        }
+    }
+
+}
+
+export default BLAKE2b;

src/core/operations/BLAKE2s.mjs

@@ -0,0 +1,80 @@
+/**
+ * @author h345983745
+ * @copyright Crown Copyright 2019
+ * @license Apache-2.0
+ */
+
+import Operation from "../Operation";
+import blakejs from "blakejs";
+import OperationError from "../errors/OperationError";
+import Utils from "../Utils";
+import { toBase64 } from "../lib/Base64";
+
+/**
+ * BLAKE2s Operation
+ */
+class BLAKE2s extends Operation {
+
+    /**
+     * BLAKE2s constructor
+     */
+    constructor() {
+        super();
+
+        this.name = "BLAKE2s";
+        this.module = "Hashing";
+        this.description = `Performs BLAKE2s hashing on the input.  
+        <br><br>BLAKE2s is a flavour of the BLAKE cryptographic hash function that is optimized for 8- to 32-bit platforms and produces digests of any size between 1 and 32 bytes.
+        <br><br>Supports the use of an optional key.`;
+        this.infoURL = "https://wikipedia.org/wiki/BLAKE_(hash_function)#BLAKE2";
+        this.inputType = "ArrayBuffer";
+        this.outputType = "string";
+        this.args = [
+            {
+                "name": "Size",
+                "type": "option",
+                "value": ["256", "160", "128"]
+            }, {
+                "name": "Output Encoding",
+                "type": "option",
+                "value": ["Hex", "Base64", "Raw"]
+            },
+            {
+                "name": "Key",
+                "type": "toggleString",
+                "value": "",
+                "toggleValues": ["UTF8", "Decimal", "Base64", "Hex", "Latin1"]
+            }
+        ];
+    }
+
+    /**
+     * @param {ArrayBuffer} input
+     * @param {Object[]} args
+     * @returns {string} The input having been hashed with BLAKE2s in the encoding format speicifed.
+     */
+    run(input, args) {
+        const [outSize, outFormat] = args;
+        let key = Utils.convertToByteArray(args[2].string || "", args[2].option);
+        if (key.length === 0) {
+            key = null;
+        } else if (key.length > 32) {
+            throw new OperationError(["Key cannot be greater than 32 bytes", "It is currently " + key.length + " bytes."].join("\n"));
+        }
+
+        input = new Uint8Array(input);
+        switch (outFormat) {
+            case "Hex":
+                return blakejs.blake2sHex(input, key, outSize / 8);
+            case "Base64":
+                return toBase64(blakejs.blake2s(input, key, outSize / 8));
+            case "Raw":
+                return Utils.arrayBufferToStr(blakejs.blake2s(input, key, outSize / 8).buffer);
+            default:
+                return new OperationError("Unsupported Output Type");
+        }
+    }
+
+}
+
+export default BLAKE2s;

src/core/operations/ExtractFiles.mjs

@@ -23,7 +23,7 @@ class ExtractFiles extends Operation {
 
         this.name = "Extract Files";
         this.module = "Default";
-        this.description = "TODO";
+        this.description = "Performs file carving to attempt to extract files from the input.<br><br>This operation is currently capable of carving out the following formats:<ul><li>JPG</li><li>EXE</li><li>ZIP</li><li>PDF</li><li>PNG</li><li>BMP</li><li>FLV</li><li>RTF</li><li>DOCX, PPTX, XLSX</li><li>EPUB</li><li>GZIP</li><li>ZLIB</li><li>ELF, BIN, AXF, O, PRX, SO</li></ul>";
         this.infoURL = "https://forensicswiki.org/wiki/File_Carving";
         this.inputType = "ArrayBuffer";
         this.outputType = "List<File>";

src/core/operations/GenerateAllHashes.mjs

@@ -28,6 +28,8 @@ import Fletcher64Checksum from "./Fletcher64Checksum";
 import Adler32Checksum from "./Adler32Checksum";
 import CRC16Checksum from "./CRC16Checksum";
 import CRC32Checksum from "./CRC32Checksum";
+import BLAKE2b from "./BLAKE2b";
+import BLAKE2s from "./BLAKE2s";
 
 /**
  * Generate all hashes operation
@@ -86,6 +88,14 @@ class GenerateAllHashes extends Operation {
                 "\nWhirlpool-0: " + (new Whirlpool()).run(arrayBuffer, ["Whirlpool-0"]) +
                 "\nWhirlpool-T: " + (new Whirlpool()).run(arrayBuffer, ["Whirlpool-T"]) +
                 "\nWhirlpool:   " + (new Whirlpool()).run(arrayBuffer, ["Whirlpool"]) +
+                "\nBLAKE2b-128: " + (new BLAKE2b).run(arrayBuffer, ["128", "Hex", {string: "", option: "UTF8"}]) +
+                "\nBLAKE2b-160: " + (new BLAKE2b).run(arrayBuffer, ["160", "Hex", {string: "", option: "UTF8"}]) +
+                "\nBLAKE2b-256: " + (new BLAKE2b).run(arrayBuffer, ["256", "Hex", {string: "", option: "UTF8"}]) +
+                "\nBLAKE2b-384: " + (new BLAKE2b).run(arrayBuffer, ["384", "Hex", {string: "", option: "UTF8"}]) +
+                "\nBLAKE2b-512: " + (new BLAKE2b).run(arrayBuffer, ["512", "Hex", {string: "", option: "UTF8"}]) +
+                "\nBLAKE2s-128: " + (new BLAKE2s).run(arrayBuffer, ["128", "Hex", {string: "", option: "UTF8"}]) +
+                "\nBLAKE2s-160: " + (new BLAKE2s).run(arrayBuffer, ["160", "Hex", {string: "", option: "UTF8"}]) +
+                "\nBLAKE2s-256: " + (new BLAKE2s).run(arrayBuffer, ["256", "Hex", {string: "", option: "UTF8"}]) +
                 "\nSSDEEP:      " + (new SSDEEP()).run(str) +
                 "\nCTPH:        " + (new CTPH()).run(str) +
                 "\n\nChecksums:" +

src/core/operations/HTMLToText.mjs

@@ -0,0 +1,41 @@
+/**
+ * @author tlwr [toby@toby.codes]
+ * @author Matt C [me@mitt.dev]
+ * @copyright Crown Copyright 2019
+ * @license Apache-2.0
+ */
+
+import Operation from "../Operation";
+
+/**
+ * HTML To Text operation
+ */
+class HTMLToText extends Operation {
+
+    /**
+     * HTMLToText constructor
+     */
+    constructor() {
+        super();
+
+        this.name = "HTML To Text";
+        this.module = "Default";
+        this.description = "Converts an HTML output from an operation to a readable string instead of being rendered in the DOM.";
+        this.infoURL = "";
+        this.inputType = "html";
+        this.outputType = "string";
+        this.args = [];
+    }
+
+    /**
+     * @param {html} input
+     * @param {Object[]} args
+     * @returns {string}
+     */
+    run(input, args) {
+        return input;
+    }
+
+}
+
+export default HTMLToText;

src/core/operations/HeatmapChart.mjs

@@ -0,0 +1,266 @@
+/**
+ * @author tlwr [toby@toby.codes]
+ * @author Matt C [me@mitt.dev]
+ * @copyright Crown Copyright 2019
+ * @license Apache-2.0
+ */
+
+import * as d3temp from "d3";
+import * as nodomtemp from "nodom";
+import { getScatterValues, RECORD_DELIMITER_OPTIONS, COLOURS, FIELD_DELIMITER_OPTIONS } from "../lib/Charts";
+
+import Operation from "../Operation";
+import OperationError from "../errors/OperationError";
+import Utils from "../Utils";
+
+const d3 = d3temp.default ? d3temp.default : d3temp;
+const nodom = nodomtemp.default ? nodomtemp.default: nodomtemp;
+
+/**
+ * Heatmap chart operation
+ */
+class HeatmapChart extends Operation {
+
+    /**
+     * HeatmapChart constructor
+     */
+    constructor() {
+        super();
+
+        this.name = "Heatmap chart";
+        this.module = "Charts";
+        this.description = "A heatmap is a graphical representation of data where the individual values contained in a matrix are represented as colors.";
+        this.infoURL = "https://wikipedia.org/wiki/Heat_map";
+        this.inputType = "string";
+        this.outputType = "html";
+        this.args = [
+            {
+                name: "Record delimiter",
+                type: "option",
+                value: RECORD_DELIMITER_OPTIONS,
+            },
+            {
+                name: "Field delimiter",
+                type: "option",
+                value: FIELD_DELIMITER_OPTIONS,
+            },
+            {
+                name: "Number of vertical bins",
+                type: "number",
+                value: 25,
+            },
+            {
+                name: "Number of horizontal bins",
+                type: "number",
+                value: 25,
+            },
+            {
+                name: "Use column headers as labels",
+                type: "boolean",
+                value: true,
+            },
+            {
+                name: "X label",
+                type: "string",
+                value: "",
+            },
+            {
+                name: "Y label",
+                type: "string",
+                value: "",
+            },
+            {
+                name: "Draw bin edges",
+                type: "boolean",
+                value: false,
+            },
+            {
+                name: "Min colour value",
+                type: "string",
+                value: COLOURS.min,
+            },
+            {
+                name: "Max colour value",
+                type: "string",
+                value: COLOURS.max,
+            },
+        ];
+    }
+
+    /**
+     * Heatmap chart operation.
+     *
+     * @param {string} input
+     * @param {Object[]} args
+     * @returns {html}
+     */
+    run(input, args) {
+        const recordDelimiter = Utils.charRep(args[0]),
+            fieldDelimiter = Utils.charRep(args[1]),
+            vBins = args[2],
+            hBins = args[3],
+            columnHeadingsAreIncluded = args[4],
+            drawEdges = args[7],
+            minColour = args[8],
+            maxColour = args[9],
+            dimension = 500;
+        if (vBins <= 0) throw new OperationError("Number of vertical bins must be greater than 0");
+        if (hBins <= 0) throw new OperationError("Number of horizontal bins must be greater than 0");
+
+        let xLabel = args[5],
+            yLabel = args[6];
+        const { headings, values } = getScatterValues(
+            input,
+            recordDelimiter,
+            fieldDelimiter,
+            columnHeadingsAreIncluded
+        );
+
+        if (headings) {
+            xLabel = headings.x;
+            yLabel = headings.y;
+        }
+
+        const document = new nodom.Document();
+        let svg = document.createElement("svg");
+        svg = d3.select(svg)
+            .attr("width", "100%")
+            .attr("height", "100%")
+            .attr("viewBox", `0 0 ${dimension} ${dimension}`);
+
+        const margin = {
+                top: 10,
+                right: 0,
+                bottom: 40,
+                left: 30,
+            },
+            width = dimension - margin.left - margin.right,
+            height = dimension - margin.top - margin.bottom,
+            binWidth = width / hBins,
+            binHeight = height/ vBins,
+            marginedSpace = svg.append("g")
+                .attr("transform", "translate(" + margin.left + "," + margin.top + ")");
+
+        const bins = this.getHeatmapPacking(values, vBins, hBins),
+            maxCount = Math.max(...bins.map(row => {
+                const lengths = row.map(cell => cell.length);
+                return Math.max(...lengths);
+            }));
+
+        const xExtent = d3.extent(values, d => d[0]),
+            yExtent = d3.extent(values, d => d[1]);
+
+        const xAxis = d3.scaleLinear()
+            .domain(xExtent)
+            .range([0, width]);
+        const yAxis = d3.scaleLinear()
+            .domain(yExtent)
+            .range([height, 0]);
+
+        const colour = d3.scaleSequential(d3.interpolateLab(minColour, maxColour))
+            .domain([0, maxCount]);
+
+        marginedSpace.append("clipPath")
+            .attr("id", "clip")
+            .append("rect")
+            .attr("width", width)
+            .attr("height", height);
+
+        marginedSpace.append("g")
+            .attr("class", "bins")
+            .attr("clip-path", "url(#clip)")
+            .selectAll("g")
+            .data(bins)
+            .enter()
+            .append("g")
+            .selectAll("rect")
+            .data(d => d)
+            .enter()
+            .append("rect")
+            .attr("x", (d) => binWidth * d.x)
+            .attr("y", (d) => (height - binHeight * (d.y + 1)))
+            .attr("width", binWidth)
+            .attr("height", binHeight)
+            .attr("fill", (d) => colour(d.length))
+            .attr("stroke", drawEdges ? "rgba(0, 0, 0, 0.5)" : "none")
+            .attr("stroke-width", drawEdges ? "0.5" : "none")
+            .append("title")
+            .text(d => {
+                const count = d.length,
+                    perc = 100.0 * d.length / values.length,
+                    tooltip = `Count: ${count}\n
+                               Percentage: ${perc.toFixed(2)}%\n
+                    `.replace(/\s{2,}/g, "\n");
+                return tooltip;
+            });
+
+        marginedSpace.append("g")
+            .attr("class", "axis axis--y")
+            .call(d3.axisLeft(yAxis).tickSizeOuter(-width));
+
+        svg.append("text")
+            .attr("transform", "rotate(-90)")
+            .attr("y", -margin.left)
+            .attr("x", -(height / 2))
+            .attr("dy", "1em")
+            .style("text-anchor", "middle")
+            .text(yLabel);
+
+        marginedSpace.append("g")
+            .attr("class", "axis axis--x")
+            .attr("transform", "translate(0," + height + ")")
+            .call(d3.axisBottom(xAxis).tickSizeOuter(-height));
+
+        svg.append("text")
+            .attr("x", width / 2)
+            .attr("y", dimension)
+            .style("text-anchor", "middle")
+            .text(xLabel);
+
+        return svg._groups[0][0].outerHTML;
+    }
+
+    /**
+     * Packs a list of x, y coordinates into a number of bins for use in a heatmap.
+     *
+     * @param {Object[]} points
+     * @param {number} number of vertical bins
+     * @param {number} number of horizontal bins
+     * @returns {Object[]} a list of bins (each bin is an Array) with x y coordinates, filled with the points
+     */
+    getHeatmapPacking(values, vBins, hBins) {
+        const xBounds = d3.extent(values, d => d[0]),
+            yBounds = d3.extent(values, d => d[1]),
+            bins = [];
+
+        if (xBounds[0] === xBounds[1]) throw "Cannot pack points. There is no difference between the minimum and maximum X coordinate.";
+        if (yBounds[0] === yBounds[1]) throw "Cannot pack points. There is no difference between the minimum and maximum Y coordinate.";
+
+        for (let y = 0; y < vBins; y++) {
+            bins.push([]);
+            for (let x = 0; x < hBins; x++) {
+                const item = [];
+                item.y = y;
+                item.x = x;
+
+                bins[y].push(item);
+            } // x
+        } // y
+
+        const epsilon = 0.000000001; // This is to clamp values that are exactly the maximum;
+
+        values.forEach(v => {
+            const fractionOfY = (v[1] - yBounds[0]) / ((yBounds[1] + epsilon) - yBounds[0]),
+                fractionOfX = (v[0] - xBounds[0]) / ((xBounds[1] + epsilon) - xBounds[0]),
+                y = Math.floor(vBins * fractionOfY),
+                x = Math.floor(hBins * fractionOfX);
+
+            bins[y][x].push({x: v[0], y: v[1]});
+        });
+
+        return bins;
+    }
+
+}
+
+export default HeatmapChart;

src/core/operations/HexDensityChart.mjs

@@ -0,0 +1,296 @@
+/**
+ * @author tlwr [toby@toby.codes]
+ * @author Matt C [me@mitt.dev]
+ * @copyright Crown Copyright 2019
+ * @license Apache-2.0
+ */
+
+import * as d3temp from "d3";
+import * as d3hexbintemp from "d3-hexbin";
+import * as nodomtemp from "nodom";
+import { getScatterValues, RECORD_DELIMITER_OPTIONS, COLOURS, FIELD_DELIMITER_OPTIONS } from "../lib/Charts";
+
+import Operation from "../Operation";
+import Utils from "../Utils";
+
+const d3 = d3temp.default ? d3temp.default : d3temp;
+const d3hexbin = d3hexbintemp.default ? d3hexbintemp.default : d3hexbintemp;
+const nodom = nodomtemp.default ? nodomtemp.default: nodomtemp;
+
+
+/**
+ * Hex Density chart operation
+ */
+class HexDensityChart extends Operation {
+
+    /**
+     * HexDensityChart constructor
+     */
+    constructor() {
+        super();
+
+        this.name = "Hex Density chart";
+        this.module = "Charts";
+        this.description = "Hex density charts are used in a similar way to scatter charts, however rather than rendering tens of thousands of points, it groups the points into a few hundred hexagons to show the distribution.";
+        this.inputType = "string";
+        this.outputType = "html";
+        this.args = [
+            {
+                name: "Record delimiter",
+                type: "option",
+                value: RECORD_DELIMITER_OPTIONS,
+            },
+            {
+                name: "Field delimiter",
+                type: "option",
+                value: FIELD_DELIMITER_OPTIONS,
+            },
+            {
+                name: "Pack radius",
+                type: "number",
+                value: 25,
+            },
+            {
+                name: "Draw radius",
+                type: "number",
+                value: 15,
+            },
+            {
+                name: "Use column headers as labels",
+                type: "boolean",
+                value: true,
+            },
+            {
+                name: "X label",
+                type: "string",
+                value: "",
+            },
+            {
+                name: "Y label",
+                type: "string",
+                value: "",
+            },
+            {
+                name: "Draw hexagon edges",
+                type: "boolean",
+                value: false,
+            },
+            {
+                name: "Min colour value",
+                type: "string",
+                value: COLOURS.min,
+            },
+            {
+                name: "Max colour value",
+                type: "string",
+                value: COLOURS.max,
+            },
+            {
+                name: "Draw empty hexagons within data boundaries",
+                type: "boolean",
+                value: false,
+            }
+        ];
+    }
+
+
+    /**
+     * Hex Bin chart operation.
+     *
+     * @param {string} input
+     * @param {Object[]} args
+     * @returns {html}
+     */
+    run(input, args) {
+        const recordDelimiter = Utils.charRep(args[0]),
+            fieldDelimiter = Utils.charRep(args[1]),
+            packRadius = args[2],
+            drawRadius = args[3],
+            columnHeadingsAreIncluded = args[4],
+            drawEdges = args[7],
+            minColour = args[8],
+            maxColour = args[9],
+            drawEmptyHexagons = args[10],
+            dimension = 500;
+
+        let xLabel = args[5],
+            yLabel = args[6];
+        const { headings, values } = getScatterValues(
+            input,
+            recordDelimiter,
+            fieldDelimiter,
+            columnHeadingsAreIncluded
+        );
+
+        if (headings) {
+            xLabel = headings.x;
+            yLabel = headings.y;
+        }
+
+        const document = new nodom.Document();
+        let svg = document.createElement("svg");
+        svg = d3.select(svg)
+            .attr("width", "100%")
+            .attr("height", "100%")
+            .attr("viewBox", `0 0 ${dimension} ${dimension}`);
+
+        const margin = {
+                top: 10,
+                right: 0,
+                bottom: 40,
+                left: 30,
+            },
+            width = dimension - margin.left - margin.right,
+            height = dimension - margin.top - margin.bottom,
+            marginedSpace = svg.append("g")
+                .attr("transform", "translate(" + margin.left + "," + margin.top + ")");
+
+        const hexbin = d3hexbin.hexbin()
+            .radius(packRadius)
+            .extent([0, 0], [width, height]);
+
+        const hexPoints = hexbin(values),
+            maxCount = Math.max(...hexPoints.map(b => b.length));
+
+        const xExtent = d3.extent(hexPoints, d => d.x),
+            yExtent = d3.extent(hexPoints, d => d.y);
+        xExtent[0] -= 2 * packRadius;
+        xExtent[1] += 3 * packRadius;
+        yExtent[0] -= 2 * packRadius;
+        yExtent[1] += 2 * packRadius;
+
+        const xAxis = d3.scaleLinear()
+            .domain(xExtent)
+            .range([0, width]);
+        const yAxis = d3.scaleLinear()
+            .domain(yExtent)
+            .range([height, 0]);
+
+        const colour = d3.scaleSequential(d3.interpolateLab(minColour, maxColour))
+            .domain([0, maxCount]);
+
+        marginedSpace.append("clipPath")
+            .attr("id", "clip")
+            .append("rect")
+            .attr("width", width)
+            .attr("height", height);
+
+        if (drawEmptyHexagons) {
+            marginedSpace.append("g")
+                .attr("class", "empty-hexagon")
+                .selectAll("path")
+                .data(this.getEmptyHexagons(hexPoints, packRadius))
+                .enter()
+                .append("path")
+                .attr("d", d => {
+                    return `M${xAxis(d.x)},${yAxis(d.y)} ${hexbin.hexagon(drawRadius)}`;
+                })
+                .attr("fill", (d) => colour(0))
+                .attr("stroke", drawEdges ? "black" : "none")
+                .attr("stroke-width", drawEdges ? "0.5" : "none")
+                .append("title")
+                .text(d => {
+                    const count = 0,
+                        perc = 0,
+                        tooltip = `Count: ${count}\n
+                                Percentage: ${perc.toFixed(2)}%\n
+                                Center: ${d.x.toFixed(2)}, ${d.y.toFixed(2)}\n
+                        `.replace(/\s{2,}/g, "\n");
+                    return tooltip;
+                });
+        }
+
+        marginedSpace.append("g")
+            .attr("class", "hexagon")
+            .attr("clip-path", "url(#clip)")
+            .selectAll("path")
+            .data(hexPoints)
+            .enter()
+            .append("path")
+            .attr("d", d => {
+                return `M${xAxis(d.x)},${yAxis(d.y)} ${hexbin.hexagon(drawRadius)}`;
+            })
+            .attr("fill", (d) => colour(d.length))
+            .attr("stroke", drawEdges ? "black" : "none")
+            .attr("stroke-width", drawEdges ? "0.5" : "none")
+            .append("title")
+            .text(d => {
+                const count = d.length,
+                    perc = 100.0 * d.length / values.length,
+                    CX = d.x,
+                    CY = d.y,
+                    xMin = Math.min(...d.map(d => d[0])),
+                    xMax = Math.max(...d.map(d => d[0])),
+                    yMin = Math.min(...d.map(d => d[1])),
+                    yMax = Math.max(...d.map(d => d[1])),
+                    tooltip = `Count: ${count}\n
+                               Percentage: ${perc.toFixed(2)}%\n
+                               Center: ${CX.toFixed(2)}, ${CY.toFixed(2)}\n
+                               Min X: ${xMin.toFixed(2)}\n
+                               Max X: ${xMax.toFixed(2)}\n
+                               Min Y: ${yMin.toFixed(2)}\n
+                               Max Y: ${yMax.toFixed(2)}
+                    `.replace(/\s{2,}/g, "\n");
+                return tooltip;
+            });
+
+        marginedSpace.append("g")
+            .attr("class", "axis axis--y")
+            .call(d3.axisLeft(yAxis).tickSizeOuter(-width));
+
+        svg.append("text")
+            .attr("transform", "rotate(-90)")
+            .attr("y", -margin.left)
+            .attr("x", -(height / 2))
+            .attr("dy", "1em")
+            .style("text-anchor", "middle")
+            .text(yLabel);
+
+        marginedSpace.append("g")
+            .attr("class", "axis axis--x")
+            .attr("transform", "translate(0," + height + ")")
+            .call(d3.axisBottom(xAxis).tickSizeOuter(-height));
+
+        svg.append("text")
+            .attr("x", width / 2)
+            .attr("y", dimension)
+            .style("text-anchor", "middle")
+            .text(xLabel);
+
+        return svg._groups[0][0].outerHTML;
+    }
+
+
+    /**
+     * Hex Bin chart operation.
+     *
+     * @param {Object[]} - centres
+     * @param {number} - radius
+     * @returns {Object[]}
+     */
+    getEmptyHexagons(centres, radius) {
+        const emptyCentres = [],
+            boundingRect = [d3.extent(centres, d => d.x), d3.extent(centres, d => d.y)],
+            hexagonCenterToEdge = Math.cos(2 * Math.PI / 12) * radius,
+            hexagonEdgeLength = Math.sin(2 * Math.PI / 12) * radius;
+        let indent = false;
+
+        for (let y = boundingRect[1][0]; y <= boundingRect[1][1] + radius; y += hexagonEdgeLength + radius) {
+            for (let x = boundingRect[0][0]; x <= boundingRect[0][1] + radius; x += 2 * hexagonCenterToEdge) {
+                let cx = x;
+                const cy = y;
+
+                if (indent && x >= boundingRect[0][1]) break;
+                if (indent) cx += hexagonCenterToEdge;
+
+                emptyCentres.push({x: cx, y: cy});
+            }
+            indent = !indent;
+        }
+
+        return emptyCentres;
+    }
+
+}
+
+export default HexDensityChart;

src/core/operations/JavaScriptParser.mjs

@@ -21,7 +21,7 @@ class JavaScriptParser extends Operation {
         this.name = "JavaScript Parser";
         this.module = "Code";
         this.description = "Returns an Abstract Syntax Tree for valid JavaScript code.";
-        this.infoURL = "https://en.wikipedia.org/wiki/Abstract_syntax_tree";
+        this.infoURL = "https://wikipedia.org/wiki/Abstract_syntax_tree";
         this.inputType = "string";
         this.outputType = "string";
         this.args = [

src/core/operations/PEMToHex.mjs

@@ -21,7 +21,7 @@ class PEMToHex extends Operation {
         this.name = "PEM to Hex";
         this.module = "PublicKey";
         this.description = "Converts PEM (Privacy Enhanced Mail) format to a hexadecimal DER (Distinguished Encoding Rules) string.";
-        this.infoURL = "https://en.wikipedia.org/wiki/X.690#DER_encoding";
+        this.infoURL = "https://wikipedia.org/wiki/X.690#DER_encoding";
         this.inputType = "string";
         this.outputType = "string";
         this.args = [];

src/core/operations/ProtobufDecode.mjs

@@ -0,0 +1,46 @@
+/**
+ * @author GCHQ Contributor [3]
+ * @copyright Crown Copyright 2019
+ * @license Apache-2.0
+ */
+
+import Operation from "../Operation";
+import OperationError from "../errors/OperationError";
+import Protobuf from "../lib/Protobuf";
+
+/**
+ * Protobuf Decode operation
+ */
+class ProtobufDecode extends Operation {
+
+    /**
+     * ProtobufDecode constructor
+     */
+    constructor() {
+        super();
+
+        this.name = "Protobuf Decode";
+        this.module = "Default";
+        this.description = "Decodes any Protobuf encoded data to a JSON representation of the data using the field number as the field key.";
+        this.infoURL = "https://wikipedia.org/wiki/Protocol_Buffers";
+        this.inputType = "byteArray";
+        this.outputType = "JSON";
+        this.args = [];
+    }
+
+    /**
+     * @param {byteArray} input
+     * @param {Object[]} args
+     * @returns {JSON}
+     */
+    run(input, args) {
+        try {
+            return Protobuf.decode(input);
+        } catch (err) {
+            throw new OperationError(err);
+        }
+    }
+
+}
+
+export default ProtobufDecode;

src/core/operations/RegularExpression.mjs

@@ -230,6 +230,7 @@ function regexHighlight (input, regex, displayTotal) {
         title = "",
         hl = 1,
         total = 0;
+    const captureGroups = [];
 
     output = input.replace(regex, (match, ...args) => {
         args.pop(); // Throw away full string
@@ -247,9 +248,15 @@ function regexHighlight (input, regex, displayTotal) {
         // Switch highlight
         hl = hl === 1 ? 2 : 1;
 
-        total++;
+        // Store highlighted match and replace with a placeholder
+        captureGroups.push(`<span class='hl${hl}' title='${title}'>${Utils.escapeHtml(match)}</span>`);
+        return `[cc_capture_group_${total++}]`;
+    });
 
-        return `<span class='hl${hl}' title='${title}'>${Utils.escapeHtml(match)}</span>`;
+    // Safely escape all remaining text, then replace placeholders
+    output = Utils.escapeHtml(output);
+    output = output.replace(/\[cc_capture_group_(\d+)\]/g, (_, i) => {
+        return captureGroups[i];
     });
 
     if (displayTotal)

src/core/operations/ScatterChart.mjs

@@ -0,0 +1,199 @@
+/**
+ * @author tlwr [toby@toby.codes]
+ * @author Matt C [me@mitt.dev]
+ * @copyright Crown Copyright 2019
+ * @license Apache-2.0
+ */
+
+import * as d3temp from "d3";
+import * as nodomtemp from "nodom";
+import { getScatterValues, getScatterValuesWithColour, RECORD_DELIMITER_OPTIONS, COLOURS, FIELD_DELIMITER_OPTIONS } from "../lib/Charts";
+
+import Operation from "../Operation";
+import Utils from "../Utils";
+
+const d3 = d3temp.default ? d3temp.default : d3temp;
+const nodom = nodomtemp.default ? nodomtemp.default: nodomtemp;
+
+/**
+ * Scatter chart operation
+ */
+class ScatterChart extends Operation {
+
+    /**
+     * ScatterChart constructor
+     */
+    constructor() {
+        super();
+
+        this.name = "Scatter chart";
+        this.module = "Charts";
+        this.description = "Plots two-variable data as single points on a graph.";
+        this.infoURL = "https://wikipedia.org/wiki/Scatter_plot";
+        this.inputType = "string";
+        this.outputType = "html";
+        this.args = [
+            {
+                name: "Record delimiter",
+                type: "option",
+                value: RECORD_DELIMITER_OPTIONS,
+            },
+            {
+                name: "Field delimiter",
+                type: "option",
+                value: FIELD_DELIMITER_OPTIONS,
+            },
+            {
+                name: "Use column headers as labels",
+                type: "boolean",
+                value: true,
+            },
+            {
+                name: "X label",
+                type: "string",
+                value: "",
+            },
+            {
+                name: "Y label",
+                type: "string",
+                value: "",
+            },
+            {
+                name: "Colour",
+                type: "string",
+                value: COLOURS.max,
+            },
+            {
+                name: "Point radius",
+                type: "number",
+                value: 10,
+            },
+            {
+                name: "Use colour from third column",
+                type: "boolean",
+                value: false,
+            }
+        ];
+    }
+
+    /**
+     * Scatter chart operation.
+     *
+     * @param {string} input
+     * @param {Object[]} args
+     * @returns {html}
+     */
+    run(input, args) {
+        const recordDelimiter = Utils.charRep(args[0]),
+            fieldDelimiter = Utils.charRep(args[1]),
+            columnHeadingsAreIncluded = args[2],
+            fillColour = args[5],
+            radius = args[6],
+            colourInInput = args[7],
+            dimension = 500;
+
+        let xLabel = args[3],
+            yLabel = args[4];
+
+        const dataFunction = colourInInput ? getScatterValuesWithColour : getScatterValues;
+        const { headings, values } = dataFunction(
+            input,
+            recordDelimiter,
+            fieldDelimiter,
+            columnHeadingsAreIncluded
+        );
+
+        if (headings) {
+            xLabel = headings.x;
+            yLabel = headings.y;
+        }
+
+        const document = new nodom.Document();
+        let svg = document.createElement("svg");
+        svg = d3.select(svg)
+            .attr("width", "100%")
+            .attr("height", "100%")
+            .attr("viewBox", `0 0 ${dimension} ${dimension}`);
+
+        const margin = {
+                top: 10,
+                right: 0,
+                bottom: 40,
+                left: 30,
+            },
+            width = dimension - margin.left - margin.right,
+            height = dimension - margin.top - margin.bottom,
+            marginedSpace = svg.append("g")
+                .attr("transform", "translate(" + margin.left + "," + margin.top + ")");
+
+        const xExtent = d3.extent(values, d => d[0]),
+            xDelta = xExtent[1] - xExtent[0],
+            yExtent = d3.extent(values, d => d[1]),
+            yDelta = yExtent[1] - yExtent[0],
+            xAxis = d3.scaleLinear()
+                .domain([xExtent[0] - (0.1 * xDelta), xExtent[1] + (0.1 * xDelta)])
+                .range([0, width]),
+            yAxis = d3.scaleLinear()
+                .domain([yExtent[0] - (0.1 * yDelta), yExtent[1] + (0.1 * yDelta)])
+                .range([height, 0]);
+
+        marginedSpace.append("clipPath")
+            .attr("id", "clip")
+            .append("rect")
+            .attr("width", width)
+            .attr("height", height);
+
+        marginedSpace.append("g")
+            .attr("class", "points")
+            .attr("clip-path", "url(#clip)")
+            .selectAll("circle")
+            .data(values)
+            .enter()
+            .append("circle")
+            .attr("cx", (d) => xAxis(d[0]))
+            .attr("cy", (d) => yAxis(d[1]))
+            .attr("r", d => radius)
+            .attr("fill", d => {
+                return colourInInput ? d[2] : fillColour;
+            })
+            .attr("stroke", "rgba(0, 0, 0, 0.5)")
+            .attr("stroke-width", "0.5")
+            .append("title")
+            .text(d => {
+                const x = d[0],
+                    y = d[1],
+                    tooltip = `X: ${x}\n
+                               Y: ${y}\n
+                    `.replace(/\s{2,}/g, "\n");
+                return tooltip;
+            });
+
+        marginedSpace.append("g")
+            .attr("class", "axis axis--y")
+            .call(d3.axisLeft(yAxis).tickSizeOuter(-width));
+
+        svg.append("text")
+            .attr("transform", "rotate(-90)")
+            .attr("y", -margin.left)
+            .attr("x", -(height / 2))
+            .attr("dy", "1em")
+            .style("text-anchor", "middle")
+            .text(yLabel);
+
+        marginedSpace.append("g")
+            .attr("class", "axis axis--x")
+            .attr("transform", "translate(0," + height + ")")
+            .call(d3.axisBottom(xAxis).tickSizeOuter(-height));
+
+        svg.append("text")
+            .attr("x", width / 2)
+            .attr("y", dimension)
+            .style("text-anchor", "middle")
+            .text(xLabel);
+
+        return svg._groups[0][0].outerHTML;
+    }
+
+}
+
+export default ScatterChart;

src/core/operations/SeriesChart.mjs

@@ -0,0 +1,227 @@
+/**
+ * @author tlwr [toby@toby.codes]
+ * @author Matt C [me@mitt.dev]
+ * @copyright Crown Copyright 2019
+ * @license Apache-2.0
+ */
+
+import * as d3temp from "d3";
+import * as nodomtemp from "nodom";
+import { getSeriesValues, RECORD_DELIMITER_OPTIONS, FIELD_DELIMITER_OPTIONS } from "../lib/Charts";
+
+import Operation from "../Operation";
+import Utils from "../Utils";
+
+const d3 = d3temp.default ? d3temp.default : d3temp;
+const nodom = nodomtemp.default ? nodomtemp.default: nodomtemp;
+
+/**
+ * Series chart operation
+ */
+class SeriesChart extends Operation {
+
+    /**
+     * SeriesChart constructor
+     */
+    constructor() {
+        super();
+
+        this.name = "Series chart";
+        this.module = "Charts";
+        this.description = "A time series graph is a line graph of repeated measurements taken over regular time intervals.";
+        this.inputType = "string";
+        this.outputType = "html";
+        this.args = [
+            {
+                name: "Record delimiter",
+                type: "option",
+                value: RECORD_DELIMITER_OPTIONS,
+            },
+            {
+                name: "Field delimiter",
+                type: "option",
+                value: FIELD_DELIMITER_OPTIONS,
+            },
+            {
+                name: "X label",
+                type: "string",
+                value: "",
+            },
+            {
+                name: "Point radius",
+                type: "number",
+                value: 1,
+            },
+            {
+                name: "Series colours",
+                type: "string",
+                value: "mediumseagreen, dodgerblue, tomato",
+            },
+        ];
+    }
+
+    /**
+     * Series chart operation.
+     *
+     * @param {string} input
+     * @param {Object[]} args
+     * @returns {html}
+     */
+    run(input, args) {
+        const recordDelimiter = Utils.charRep(args[0]),
+            fieldDelimiter = Utils.charRep(args[1]),
+            xLabel = args[2],
+            pipRadius = args[3],
+            seriesColours = args[4].split(","),
+            svgWidth = 500,
+            interSeriesPadding = 20,
+            xAxisHeight = 50,
+            seriesLabelWidth = 50,
+            seriesHeight = 100,
+            seriesWidth = svgWidth - seriesLabelWidth - interSeriesPadding;
+
+        const { xValues, series } = getSeriesValues(input, recordDelimiter, fieldDelimiter),
+            allSeriesHeight = Object.keys(series).length * (interSeriesPadding + seriesHeight),
+            svgHeight = allSeriesHeight + xAxisHeight + interSeriesPadding;
+
+        const document = new nodom.Document();
+        let svg = document.createElement("svg");
+        svg = d3.select(svg)
+            .attr("width", "100%")
+            .attr("height", "100%")
+            .attr("viewBox", `0 0 ${svgWidth} ${svgHeight}`);
+
+        const xAxis = d3.scalePoint()
+            .domain(xValues)
+            .range([0, seriesWidth]);
+
+        svg.append("g")
+            .attr("class", "axis axis--x")
+            .attr("transform", `translate(${seriesLabelWidth}, ${xAxisHeight})`)
+            .call(
+                d3.axisTop(xAxis).tickValues(xValues.filter((x, i) => {
+                    return [0, Math.round(xValues.length / 2), xValues.length -1].indexOf(i) >= 0;
+                }))
+            );
+
+        svg.append("text")
+            .attr("x", svgWidth / 2)
+            .attr("y", xAxisHeight / 2)
+            .style("text-anchor", "middle")
+            .text(xLabel);
+
+        const tooltipText = {},
+            tooltipAreaWidth = seriesWidth / xValues.length;
+
+        xValues.forEach(x => {
+            const tooltip = [];
+
+            series.forEach(serie => {
+                const y = serie.data[x];
+                if (typeof y === "undefined") return;
+
+                tooltip.push(`${serie.name}: ${y}`);
+            });
+
+            tooltipText[x] = tooltip.join("\n");
+        });
+
+        const chartArea = svg.append("g")
+            .attr("transform", `translate(${seriesLabelWidth}, ${xAxisHeight})`);
+
+        chartArea
+            .append("g")
+            .selectAll("rect")
+            .data(xValues)
+            .enter()
+            .append("rect")
+            .attr("x", x => {
+                return xAxis(x) - (tooltipAreaWidth / 2);
+            })
+            .attr("y", 0)
+            .attr("width", tooltipAreaWidth)
+            .attr("height", allSeriesHeight)
+            .attr("stroke", "none")
+            .attr("fill", "transparent")
+            .append("title")
+            .text(x => {
+                return `${x}\n
+                    --\n
+                    ${tooltipText[x]}\n
+                `.replace(/\s{2,}/g, "\n");
+            });
+
+        const yAxesArea = svg.append("g")
+            .attr("transform", `translate(0, ${xAxisHeight})`);
+
+        series.forEach((serie, seriesIndex) => {
+            const yExtent = d3.extent(Object.values(serie.data)),
+                yAxis = d3.scaleLinear()
+                    .domain(yExtent)
+                    .range([seriesHeight, 0]);
+
+            const seriesGroup = chartArea
+                .append("g")
+                .attr("transform", `translate(0, ${seriesHeight * seriesIndex + interSeriesPadding * (seriesIndex + 1)})`);
+
+            let path = "";
+            xValues.forEach((x, xIndex) => {
+                let nextX = xValues[xIndex + 1],
+                    y = serie.data[x],
+                    nextY= serie.data[nextX];
+
+                if (typeof y === "undefined" || typeof nextY === "undefined") return;
+
+                x = xAxis(x); nextX = xAxis(nextX);
+                y = yAxis(y); nextY = yAxis(nextY);
+
+                path += `M ${x} ${y} L ${nextX} ${nextY} z `;
+            });
+
+            seriesGroup
+                .append("path")
+                .attr("d", path)
+                .attr("fill", "none")
+                .attr("stroke", seriesColours[seriesIndex % seriesColours.length])
+                .attr("stroke-width", "1");
+
+            xValues.forEach(x => {
+                const y = serie.data[x];
+                if (typeof y === "undefined") return;
+
+                seriesGroup
+                    .append("circle")
+                    .attr("cx", xAxis(x))
+                    .attr("cy", yAxis(y))
+                    .attr("r", pipRadius)
+                    .attr("fill", seriesColours[seriesIndex % seriesColours.length])
+                    .append("title")
+                    .text(d => {
+                        return `${x}\n
+                            --\n
+                            ${tooltipText[x]}\n
+                        `.replace(/\s{2,}/g, "\n");
+                    });
+            });
+
+            yAxesArea
+                .append("g")
+                .attr("transform", `translate(${seriesLabelWidth - interSeriesPadding}, ${seriesHeight * seriesIndex + interSeriesPadding * (seriesIndex + 1)})`)
+                .attr("class", "axis axis--y")
+                .call(d3.axisLeft(yAxis).ticks(5));
+
+            yAxesArea
+                .append("g")
+                .attr("transform", `translate(0, ${seriesHeight / 2 + seriesHeight * seriesIndex + interSeriesPadding * (seriesIndex + 1)})`)
+                .append("text")
+                .style("text-anchor", "middle")
+                .attr("transform", "rotate(-90)")
+                .text(serie.name);
+        });
+
+        return svg._groups[0][0].outerHTML;
+    }
+
+}
+
+export default SeriesChart;

src/core/operations/TextEncodingBruteForce.mjs

@@ -79,7 +79,7 @@ class TextEncodingBruteForce extends Operation {
         let table = "<table class='table table-hover table-sm table-bordered table-nonfluid'><tr><th>Encoding</th><th>Value</th></tr>";
 
         for (const enc in encodings) {
-            const value = Utils.printable(encodings[enc], true);
+            const value = Utils.escapeHtml(Utils.printable(encodings[enc], true));
             table += `<tr><td>${enc}</td><td>${value}</td></tr>`;
         }
 

src/core/operations/VarIntDecode.mjs

@@ -0,0 +1,46 @@
+/**
+ * @author GCHQ Contributor [3]
+ * @copyright Crown Copyright 2019
+ * @license Apache-2.0
+ */
+
+import Operation from "../Operation";
+import OperationError from "../errors/OperationError";
+import Protobuf from "../lib/Protobuf";
+
+/**
+ * VarInt Decode operation
+ */
+class VarIntDecode extends Operation {
+
+    /**
+     * VarIntDecode constructor
+     */
+    constructor() {
+        super();
+
+        this.name = "VarInt Decode";
+        this.module = "Default";
+        this.description = "Decodes a VarInt encoded integer. VarInt is an efficient way of encoding variable length integers and is commonly used with Protobuf.";
+        this.infoURL = "https://developers.google.com/protocol-buffers/docs/encoding#varints";
+        this.inputType = "byteArray";
+        this.outputType = "number";
+        this.args = [];
+    }
+
+    /**
+     * @param {byteArray} input
+     * @param {Object[]} args
+     * @returns {number}
+     */
+    run(input, args) {
+        try {
+            return Protobuf.varIntDecode(input);
+        } catch (err) {
+            throw new OperationError(err);
+        }
+    }
+
+}
+
+export default VarIntDecode;

src/core/operations/VarIntEncode.mjs

@@ -0,0 +1,46 @@
+/**
+ * @author GCHQ Contributor [3]
+ * @copyright Crown Copyright 2019
+ * @license Apache-2.0
+ */
+
+import Operation from "../Operation";
+import OperationError from "../errors/OperationError";
+import Protobuf from "../lib/Protobuf";
+
+/**
+ * VarInt Encode operation
+ */
+class VarIntEncode extends Operation {
+
+    /**
+     * VarIntEncode constructor
+     */
+    constructor() {
+        super();
+
+        this.name = "VarInt Encode";
+        this.module = "Default";
+        this.description = "Encodes a Vn integer as a VarInt. VarInt is an efficient way of encoding variable length integers and is commonly used with Protobuf.";
+        this.infoURL = "https://developers.google.com/protocol-buffers/docs/encoding#varints";
+        this.inputType = "number";
+        this.outputType = "byteArray";
+        this.args = [];
+    }
+
+    /**
+     * @param {number} input
+     * @param {Object[]} args
+     * @returns {byteArray}
+     */
+    run(input, args) {
+        try {
+            return Protobuf.varIntEncode(input);
+        } catch (err) {
+            throw new OperationError(err);
+        }
+    }
+
+}
+
+export default VarIntEncode;

src/node/index.mjs

@@ -5,7 +5,6 @@
  * @copyright Crown Copyright 2017
  * @license Apache-2.0
  */
-import "babel-polyfill";
 
 // Define global environment functions
 global.ENVIRONMENT_IS_WORKER = function() {

src/web/ControlsWaiter.mjs

@@ -338,7 +338,7 @@ class ControlsWaiter {
         const saveLink = this.generateStateUrl(true, true, null, "https://gchq.github.io/CyberChef/");
 
         if (reportBugInfo) {
-            reportBugInfo.innerHTML = `* Version: ${PKG_VERSION + (typeof INLINE === "undefined" ? "" : "s")}
+            reportBugInfo.innerHTML = `* Version: ${PKG_VERSION}
 * Compile time: ${COMPILE_TIME}
 * User-Agent:
 ${navigator.userAgent}

src/web/SeasonalWaiter.mjs

@@ -4,6 +4,10 @@
  * @license Apache-2.0
  */
 
+import clippy from "clippyjs";
+import "./static/clippy_assets/agents/Clippy/agent.js";
+import clippyMap from "./static/clippy_assets/agents/Clippy/map.png";
+
 /**
  * Waiter to handle seasonal events and easter eggs.
  */
@@ -18,6 +22,8 @@ class SeasonalWaiter {
     constructor(app, manager) {
         this.app = app;
         this.manager = manager;
+
+        this.clippyAgent = null;
     }
 
 
@@ -28,6 +34,14 @@ class SeasonalWaiter {
         // Konami code
         this.kkeys = [];
         window.addEventListener("keydown", this.konamiCodeListener.bind(this));
+
+        // Clippy
+        const now = new Date();
+        if (now.getMonth() === 3 && now.getDate() === 1) {
+            this.addClippyOption();
+            this.manager.addDynamicListener(".option-item #clippy", "change", this.setupClippy, this);
+            this.setupClippy();
+        }
     }
 
 
@@ -51,6 +65,285 @@ class SeasonalWaiter {
         }
     }
 
+    /**
+     * Creates an option in the Options menu for turning Clippy on or off
+     */
+    addClippyOption() {
+        const optionsBody = document.getElementById("options-body"),
+            optionItem = document.createElement("span");
+
+        optionItem.className = "bmd-form-group is-filled";
+        optionItem.innerHTML = `<div class="checkbox option-item">
+            <label for="clippy">
+                <input type="checkbox" option="clippy" id="clippy" checked="">
+                Use the Clippy helper
+            </label>
+        </div>`;
+        optionsBody.appendChild(optionItem);
+
+        if (!this.app.options.hasOwnProperty("clippy")) {
+            this.app.options.clippy = true;
+        }
+
+        this.manager.options.load();
+    }
+
+    /**
+     * Sets up Clippy for April Fools Day
+     */
+    setupClippy() {
+        // Destroy any previous agents
+        if (this.clippyAgent) {
+            this.clippyAgent.closeBalloonImmediately();
+            this.clippyAgent.hide();
+        }
+
+        if (!this.app.options.clippy) {
+            if (this.clippyTimeouts) this.clippyTimeouts.forEach(t => clearTimeout(t));
+            return;
+        }
+
+        // Set base path to # to prevent external network requests
+        const clippyAssets = "#";
+        // Shim the library to prevent external network requests
+        shimClippy(clippy);
+
+        const self = this;
+        clippy.load("Clippy", (agent) => {
+            shimClippyAgent(agent);
+            self.clippyAgent = agent;
+            agent.show();
+            agent.speak("Hello, I'm Clippy, your personal cyber assistant!");
+        }, undefined, clippyAssets);
+
+        // Watch for the Auto Magic button appearing
+        const magic = document.getElementById("magic");
+        const observer = new MutationObserver((mutationsList, observer) => {
+            // Read in message and recipe
+            let msg, recipe;
+            for (const mutation of mutationsList) {
+                if (mutation.attributeName === "data-original-title") {
+                    msg = magic.getAttribute("data-original-title");
+                }
+                if (mutation.attributeName === "data-recipe") {
+                    recipe = magic.getAttribute("data-recipe");
+                }
+            }
+
+            // Close balloon if it is currently showing a magic hint
+            const balloon = self.clippyAgent._balloon._balloon;
+            if (balloon.is(":visible") && balloon.text().indexOf("That looks like encoded data") >= 0) {
+                self.clippyAgent._balloon.hide(true);
+                this.clippyAgent._balloon._hidden = true;
+            }
+
+            // If a recipe was found, get Clippy to tell the user
+            if (recipe) {
+                recipe = this.manager.controls.generateStateUrl(true, true, JSON.parse(recipe));
+                msg = `That looks like encoded data!<br><br>${msg}<br><br>Click <a class="clippyMagicRecipe" href="${recipe}">here</a> to load this recipe.`;
+
+                // Stop current balloon activity immediately and trigger speak again
+                this.clippyAgent.closeBalloonImmediately();
+                self.clippyAgent.speak(msg, true);
+                // self.clippyAgent._queue.next();
+            }
+        });
+        observer.observe(document.getElementById("magic"), {attributes: true});
+
+        // Play animations for various things
+        this.manager.addListeners("#search", "click", () => {
+            this.clippyAgent.play("Searching");
+        }, this);
+        this.manager.addListeners("#save,#save-to-file", "click", () => {
+            this.clippyAgent.play("Save");
+        }, this);
+        this.manager.addListeners("#clr-recipe,#clr-io", "click", () => {
+            this.clippyAgent.play("EmptyTrash");
+        }, this);
+        this.manager.addListeners("#bake", "click", e => {
+            if (e.target.closest("button").textContent.toLowerCase().indexOf("bake") >= 0) {
+                this.clippyAgent.play("Thinking");
+            } else {
+                this.clippyAgent.play("EmptyTrash");
+            }
+            this.clippyAgent._queue.clear();
+        }, this);
+        this.manager.addListeners("#input-text", "keydown", () => {
+            this.clippyAgent.play("Writing");
+            this.clippyAgent._queue.clear();
+        }, this);
+        this.manager.addDynamicListener("a.clippyMagicRecipe", "click", (e) => {
+            this.clippyAgent.play("Congratulate");
+        }, this);
+
+        this.clippyTimeouts = [];
+        // Show challenge after timeout
+        this.clippyTimeouts.push(setTimeout(() => {
+            const hex = "1f 8b 08 00 ae a1 9b 5c 00 ff 05 40 a1 12 00 10 0c fd 26 61 5b 76 aa 9d 26 a8 02 02 37 84 f7 fb bb c5 a4 5f 22 c6 09 e5 6e c5 4c 2d 3f e9 30 a6 ea 41 a2 f2 ac 1c 00 00 00";
+            self.clippyAgent.speak(`How about a fun challenge?<br><br>Try decoding this (click to load):<br><a href="#recipe=[]&input=${encodeURIComponent(btoa(hex))}">${hex}</a>`, true);
+            self.clippyAgent.play("GetAttention");
+        }, 1 * 60 * 1000));
+
+        this.clippyTimeouts.push(setTimeout(() => {
+            self.clippyAgent.speak("<i>Did you know?</i><br><br>You can load files into CyberChef up to around 500MB using drag and drop or the load file button.", 15000);
+            self.clippyAgent.play("Wave");
+        }, 2 * 60 * 1000));
+
+        this.clippyTimeouts.push(setTimeout(() => {
+            self.clippyAgent.speak("<i>Did you know?</i><br><br>You can use the 'Fork' operation to split up your input and run the recipe over each branch separately.<br><br><a class='clippyMagicRecipe' href=\"#recipe=Fork('%5C%5Cn','%5C%5Cn',false)From_UNIX_Timestamp('Seconds%20(s)')&amp;input=OTc4MzQ2ODAwCjEwMTI2NTEyMDAKMTA0NjY5NjQwMAoxMDgxMDg3MjAwCjExMTUzMDUyMDAKMTE0OTYwOTYwMA\">Here's an example</a>.", 15000);
+            self.clippyAgent.play("Print");
+        }, 3 * 60 * 1000));
+
+        this.clippyTimeouts.push(setTimeout(() => {
+            self.clippyAgent.speak("<i>Did you know?</i><br><br>The 'Magic' operation uses a number of methods to detect encoded data and the operations which can be used to make sense of it. A technical description of these methods can be found <a href=\"https://github.com/gchq/CyberChef/wiki/Automatic-detection-of-encoded-data-using-CyberChef-Magic\">here</a>.", 15000);
+            self.clippyAgent.play("Alert");
+        }, 4 * 60 * 1000));
+
+        this.clippyTimeouts.push(setTimeout(() => {
+            self.clippyAgent.speak("<i>Did you know?</i><br><br>You can use parts of the input as arguments to operations.<br><br><a class='clippyMagicRecipe' href=\"#recipe=Register('key%3D(%5B%5C%5Cda-f%5D*)',true,false)Find_/_Replace(%7B'option':'Regex','string':'.*data%3D(.*)'%7D,'$1',true,false,true)RC4(%7B'option':'Hex','string':'$R0'%7D,'Hex','Latin1')&amp;input=aHR0cDovL21hbHdhcmV6LmJpei9iZWFjb24ucGhwP2tleT0wZTkzMmE1YyZkYXRhPThkYjdkNWViZTM4NjYzYTU0ZWNiYjMzNGUzZGIxMQ\">Click here for an example</a>.", 15000);
+            self.clippyAgent.play("CheckingSomething");
+        }, 5 * 60 * 1000));
+    }
+
+}
+
+
+/**
+ * Shims various ClippyJS functions to modify behaviour.
+ *
+ * @param {Clippy} clippy - The Clippy library
+ */
+function shimClippy(clippy) {
+    // Shim _loadSounds so that it doesn't actually try to load any sounds
+    clippy.load._loadSounds = function _loadSounds (name, path) {
+        let dfd = clippy.load._sounds[name];
+        if (dfd) return dfd;
+
+        // set dfd if not defined
+        dfd = clippy.load._sounds[name] = $.Deferred();
+
+        // Resolve immediately without loading
+        dfd.resolve({});
+
+        return dfd.promise();
+    };
+
+    // Shim _loadMap so that it uses the local copy
+    clippy.load._loadMap = function _loadMap (path) {
+        let dfd = clippy.load._maps[path];
+        if (dfd) return dfd;
+
+        // set dfd if not defined
+        dfd = clippy.load._maps[path] = $.Deferred();
+
+        const src = clippyMap;
+        const img = new Image();
+
+        img.onload = dfd.resolve;
+        img.onerror = dfd.reject;
+
+        // start loading the map;
+        img.setAttribute("src", src);
+
+        return dfd.promise();
+    };
+
+    // Make sure we don't request the remote map
+    clippy.Animator.prototype._setupElement = function _setupElement (el) {
+        const frameSize = this._data.framesize;
+        el.css("display", "none");
+        el.css({ width: frameSize[0], height: frameSize[1] });
+        el.css("background", "url('" + clippyMap + "') no-repeat");
+
+        return el;
+    };
+}
+
+/**
+ * Shims various ClippyJS Agent functions to modify behaviour.
+ *
+ * @param {Agent} agent - The Clippy Agent
+ */
+function shimClippyAgent(agent) {
+    // Turn off all sounds
+    agent._animator._playSound = () => {};
+
+    // Improve speak function to support HTML markup
+    const self = agent._balloon;
+    agent._balloon.speak = (complete, text, hold) => {
+        self._hidden = false;
+        self.show();
+        const c = self._content;
+        // set height to auto
+        c.height("auto");
+        c.width("auto");
+        // add the text
+        c.html(text);
+        // set height
+        c.height(c.height());
+        c.width(c.width());
+        c.text("");
+        self.reposition();
+
+        self._complete = complete;
+        self._sayWords(text, hold, complete);
+        if (hold) agent._queue.next();
+    };
+
+    // Improve the _sayWords function to allow HTML and support timeouts
+    agent._balloon.WORD_SPEAK_TIME = 60;
+    agent._balloon._sayWords = (text, hold, complete) => {
+        self._active = true;
+        self._hold = hold;
+        const words = text.split(/[^\S-]/);
+        const time = self.WORD_SPEAK_TIME;
+        const el = self._content;
+        let idx = 1;
+        clearTimeout(self.holdTimeout);
+
+        self._addWord = $.proxy(function () {
+            if (!self._active) return;
+            if (idx > words.length) {
+                delete self._addWord;
+                self._active = false;
+                if (!self._hold) {
+                    complete();
+                    self.hide();
+                } else if (typeof hold === "number") {
+                    self.holdTimeout = setTimeout(() => {
+                        self._hold = false;
+                        complete();
+                        self.hide();
+                    }, hold);
+                }
+            } else {
+                el.html(words.slice(0, idx).join(" "));
+                idx++;
+                self._loop = window.setTimeout($.proxy(self._addWord, self), time);
+            }
+        }, self);
+
+        self._addWord();
+    };
+
+    // Add break-word to balloon CSS
+    agent._balloon._balloon.css("word-break", "break-word");
+
+    // Close the balloon on click (unless it was a link)
+    agent._balloon._balloon.click(e => {
+        if (e.target.nodeName !== "A") {
+            agent._balloon.hide(true);
+            agent._balloon._hidden = true;
+        }
+    });
+
+    // Add function to immediately close the balloon even if it is currently doing something
+    agent.closeBalloonImmediately = () => {
+        agent._queue.clear();
+        agent._balloon.hide(true);
+        agent._balloon._hidden = true;
+        agent._queue.next();
+    };
 }
 
 export default SeasonalWaiter;

src/web/html/index.html

@@ -131,13 +131,6 @@
             };
             window.addEventListener("error", loadingErrorHandler);
         </script>
-        <% if (htmlWebpackPlugin.options.inline) { %>
-            <meta name="robots" content="noindex" />
-        <% } else { %>
-            <script type="application/ld+json">
-                <% print(JSON.stringify(require("../static/structuredData.json"))); %>
-            </script>
-        <% } %>
     </head>
     <body>
         <!-- Preloader overlay -->
@@ -153,11 +146,7 @@
         <div id="content-wrapper">
             <div id="banner" class="row">
                 <div class="col" style="text-align: left; padding-left: 10px;">
-                    <% if (htmlWebpackPlugin.options.inline) { %>
-                        <span>Version <%= htmlWebpackPlugin.options.version %></span>
-                    <% } else { %>
-                        <a href="cyberchef.htm" download>Download CyberChef <i class="material-icons">file_download</i></a>
-                    <% } %>
+                    <a href="CyberChef_v<%= htmlWebpackPlugin.options.version %>.zip" download>Download CyberChef <i class="material-icons">file_download</i></a>
                 </div>
                 <div class="col-md-6" id="notice-wrapper">
                     <span id="notice">
@@ -591,7 +580,7 @@
                                     What sort of things can I do with CyberChef?
                                 </a>
                                 <div class="collapse" id="faq-examples">
-                                    <p>There are around 200 operations in CyberChef allowing you to carry out simple and complex tasks easily. Here are some examples:</p>
+                                    <p>There are around 300 operations in CyberChef allowing you to carry out simple and complex tasks easily. Here are some examples:</p>
                                     <ul>
                                         <li><a href="#recipe=From_Base64('A-Za-z0-9%2B/%3D',true)&input=VTI4Z2JHOXVaeUJoYm1RZ2RHaGhibXR6SUdadmNpQmhiR3dnZEdobElHWnBjMmd1">Decode a Base64-encoded string</a></li>
                                         <li><a href="#recipe=Translate_DateTime_Format('Standard%20date%20and%20time','DD/MM/YYYY%20HH:mm:ss','UTC','dddd%20Do%20MMMM%20YYYY%20HH:mm:ss%20Z%20z','Australia/Queensland')&input=MTUvMDYvMjAxNSAyMDo0NTowMA">Convert a date and time to a different time zone</a></li>

src/web/index.js

@@ -8,7 +8,6 @@
 import "./stylesheets/index.js";
 
 // Libs
-import "babel-polyfill";
 import "arrive";
 import "snackbarjs";
 import "bootstrap-material-design";

src/web/static/clippy_assets/clippy.css

@@ -0,0 +1,62 @@
+.clippy, .clippy-balloon {
+    position: fixed;
+    z-index: 1000;
+    cursor: pointer;
+}
+
+.clippy-balloon {
+
+    background: #FFC;
+    color: black;
+    padding: 8px;
+    border: 1px solid black;
+    border-radius: 5px;
+
+}
+
+.clippy-content {
+    max-width: 200px;
+    min-width: 120px;
+    font-family: "Microsoft Sans", sans-serif;
+    font-size: 10pt;
+}
+
+.clippy-tip {
+    width: 10px;
+    height: 16px;
+    background: url(data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAABQAAAAgCAMAAAAlvKiEAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAAAlQTFRF///MAAAA////52QwgAAAAAN0Uk5T//8A18oNQQAAAGxJREFUeNqs0kEOwCAIRFHn3//QTUU6xMyyxii+jQosrTPkyPEM6IN3FtzIRk1U4dFeKWQiH6pRRowMVKEmvronEynkwj0uZJgR22+YLopPSo9P34wJSamLSU7lSIWLJU7NkNomNlhqxUeAAQC+TQLZyEuJBwAAAABJRU5ErkJggg==) no-repeat;
+    position: absolute;
+}
+
+.clippy-top-left .clippy-tip {
+    top: 100%;
+    margin-top: 0px;
+    left: 100%;
+    margin-left: -50px;
+}
+
+.clippy-top-right .clippy-tip {
+    top: 100%;
+    margin-top: 0px;
+    left: 0;
+    margin-left: 50px;
+    background-position: -10px 0;
+
+}
+
+.clippy-bottom-right .clippy-tip {
+    top: 0;
+    margin-top: -16px;
+    left: 0;
+    margin-left: 50px;
+    background-position: -10px -16px;
+}
+
+.clippy-bottom-left .clippy-tip {
+    top: 0;
+    margin-top: -16px;
+    left: 100%;
+    margin-left: -50px;
+    background-position: 0px -16px;
+}
+

src/web/stylesheets/index.js

@@ -8,6 +8,7 @@
 
 /* Libraries */
 import "highlight.js/styles/vs.css";
+import "../static/clippy_assets/clippy.css";
 
 /* Frameworks */
 import "./vendors/bootstrap.scss";

tests/operations/index.mjs

@@ -10,7 +10,6 @@
  * @copyright Crown Copyright 2017
  * @license Apache-2.0
  */
-import "babel-polyfill";
 
 // Define global environment functions
 global.ENVIRONMENT_IS_WORKER = function() {
@@ -33,6 +32,7 @@ import "./tests/BitwiseOp";
 import "./tests/ByteRepr";
 import "./tests/CartesianProduct";
 import "./tests/CharEnc";
+import "./tests/Charts";
 import "./tests/Checksum";
 import "./tests/Ciphers";
 import "./tests/Code";
@@ -87,6 +87,9 @@ import "./tests/Enigma";
 import "./tests/Bombe";
 import "./tests/MultipleBombe";
 import "./tests/Typex";
+import "./tests/BLAKE2b";
+import "./tests/BLAKE2s";
+import "./tests/Protobuf";
 
 // Cannot test operations that use the File type yet
 //import "./tests/SplitColourChannels";

tests/operations/tests/BLAKE2b.mjs

@@ -0,0 +1,56 @@
+/**
+ * BitwiseOp tests
+ *
+ * @author h345983745
+ * @copyright Crown Copyright 2019
+ * @license Apache-2.0
+ */
+import TestRegister from "../TestRegister";
+
+TestRegister.addTests([
+    {
+        name: "BLAKE2b: 512 - Hello World",
+        input: "Hello World",
+        expectedOutput: "4386a08a265111c9896f56456e2cb61a64239115c4784cf438e36cc851221972da3fb0115f73cd02486254001f878ab1fd126aac69844ef1c1ca152379d0a9bd",
+        recipeConfig: [
+            { "op": "BLAKE2b",
+                "args": ["512", "Hex", {string: "", option: "UTF8"}] }
+        ]
+    },
+    {
+        name: "BLAKE2b: 384 - Hello World",
+        input: "Hello World",
+        expectedOutput: "4d388e82ca8f866e606b6f6f0be910abd62ad6e98c0adfc27cf35acf948986d5c5b9c18b6f47261e1e679eb98edf8e2d",
+        recipeConfig: [
+            { "op": "BLAKE2b",
+                "args": ["384", "Hex", {string: "", option: "UTF8"}] }
+        ]
+    },
+    {
+        name: "BLAKE2b: 256 - Hello World",
+        input: "Hello World",
+        expectedOutput: "1dc01772ee0171f5f614c673e3c7fa1107a8cf727bdf5a6dadb379e93c0d1d00",
+        recipeConfig: [
+            { "op": "BLAKE2b",
+                "args": ["256", "Hex", {string: "", option: "UTF8"}] }
+        ]
+    },
+    {
+        name: "BLAKE2b: 160 - Hello World",
+        input: "Hello World",
+        expectedOutput: "6a8489e6fd6e51fae12ab271ec7fc8134dd5d737",
+        recipeConfig: [
+            { "op": "BLAKE2b",
+                "args": ["160", "Hex", {string: "", option: "UTF8"}] }
+        ]
+    },
+    {
+        name: "BLAKE2b: Key Test",
+        input: "message data",
+        expectedOutput: "3d363ff7401e02026f4a4687d4863ced",
+        recipeConfig: [
+            { "op": "BLAKE2b",
+                "args": ["128", "Hex", {string: "pseudorandom key", option: "UTF8"}] }
+        ]
+    }
+]);

tests/operations/tests/BLAKE2s.mjs

@@ -0,0 +1,47 @@
+/**
+ * BitwiseOp tests
+ *
+ * @author h345983745
+ * @copyright Crown Copyright 2019
+ * @license Apache-2.0
+ */
+import TestRegister from "../TestRegister";
+
+TestRegister.addTests([
+    {
+        name: "BLAKE2s: 256 - Hello World",
+        input: "Hello World",
+        expectedOutput: "7706af019148849e516f95ba630307a2018bb7bf03803eca5ed7ed2c3c013513",
+        recipeConfig: [
+            { "op": "BLAKE2s",
+                "args": ["256", "Hex", {string: "", option: "UTF8"}] }
+        ]
+    },
+    {
+        name: "BLAKE2s: 160 - Hello World",
+        input: "Hello World",
+        expectedOutput: "0e4fcfc2ee0097ac1d72d70b595a39e09a3c7c7e",
+        recipeConfig: [
+            { "op": "BLAKE2s",
+                "args": ["160", "Hex", {string: "", option: "UTF8"}] }
+        ]
+    },
+    {
+        name: "BLAKE2s: 128 - Hello World",
+        input: "Hello World",
+        expectedOutput: "9964ee6f36126626bf864363edfa96f6",
+        recipeConfig: [
+            { "op": "BLAKE2s",
+                "args": ["128", "Hex", {string: "", option: "UTF8"}] }
+        ]
+    },
+    {
+        name: "BLAKE2s: Key Test",
+        input: "Hello World",
+        expectedOutput: "9964ee6f36126626bf864363edfa96f6",
+        recipeConfig: [
+            { "op": "BLAKE2s",
+                "args": ["128", "Hex", {string: "", option: "UTF8"}] }
+        ]
+    }
+]);

tests/operations/tests/Charts.mjs

@@ -0,0 +1,55 @@
+/**
+ * Chart tests.
+ *
+ * @author Matt C [me@mitt.dev]
+ * @copyright Crown Copyright 2019
+ * @license Apache-2.0
+ */
+import TestRegister from "../TestRegister";
+
+TestRegister.addTests([
+    {
+        name: "Scatter chart",
+        input: "100 100\n200 200\n300 300\n400 400\n500 500",
+        expectedMatch: /^<svg width/,
+        recipeConfig: [
+            {
+                "op": "Scatter chart",
+                "args": ["Line feed", "Space", false, "time", "stress", "black", 5, false]
+            }
+        ],
+    },
+    {
+        name: "Hex density chart",
+        input: "100 100\n200 200\n300 300\n400 400\n500 500",
+        expectedMatch: /^<svg width/,
+        recipeConfig: [
+            {
+                "op": "Hex Density chart",
+                "args": ["Line feed", "Space", 25, 15, true, "", "", true, "white", "black", true]
+            }
+        ],
+    },
+    {
+        name: "Series chart",
+        input: "100 100 100\n200 200 200\n300 300 300\n400 400 400\n500 500 500",
+        expectedMatch: /^<svg width/,
+        recipeConfig: [
+            {
+                "op": "Series chart",
+                "args": ["Line feed", "Space", "", 1, "mediumseagreen, dodgerblue, tomato"]
+            }
+        ],
+    },
+    {
+        name: "Heatmap chart",
+        input: "100 100\n200 200\n300 300\n400 400\n500 500",
+        expectedMatch: /^<svg width/,
+        recipeConfig: [
+            {
+                "op": "Heatmap chart",
+                "args": ["Line feed", "Space", 25, 25, true, "", "", false, "white", "black"]
+            }
+        ],
+    },
+]);

tests/operations/tests/Protobuf.mjs

@@ -0,0 +1,36 @@
+/**
+ * Protobuf tests.
+ *
+ * @author n1474335 [n1474335@gmail.com]
+ * @copyright Crown Copyright 2019
+ * @license Apache-2.0
+ */
+
+import TestRegister from "../TestRegister";
+
+TestRegister.addTests([
+    {
+        name: "Protobuf Decode",
+        input: "0d1c0000001203596f751a024d65202b2a0a0a066162633132331200",
+        expectedOutput: JSON.stringify({
+            "1": 469762048,
+            "2": "You",
+            "3": "Me",
+            "4": 43,
+            "5": {
+                "1": "abc123",
+                "2": {}
+            }
+        }, null, 4),
+        recipeConfig: [
+            {
+                "op": "From Hex",
+                "args": ["Auto"]
+            },
+            {
+                "op": "Protobuf Decode",
+                "args": []
+            }
+        ]
+    },
+]);

webpack.config.js

@@ -48,7 +48,7 @@ module.exports = {
             "process.browser": "true"
         }),
         new MiniCssExtractPlugin({
-            filename: "[name].css"
+            filename: "assets/[name].css"
         }),
     ],
     resolve: {
@@ -80,7 +80,12 @@ module.exports = {
             {
                 test: /\.css$/,
                 use: [
-                    MiniCssExtractPlugin.loader,
+                    {
+                        loader: MiniCssExtractPlugin.loader,
+                        options: {
+                            publicPath: "../"
+                        }
+                    },
                     "css-loader",
                     "postcss-loader",
                 ]
@@ -88,7 +93,12 @@ module.exports = {
             {
                 test: /\.scss$/,
                 use: [
-                    MiniCssExtractPlugin.loader,
+                    {
+                        loader: MiniCssExtractPlugin.loader,
+                        options: {
+                            publicPath: "../"
+                        }
+                    },
                     "css-loader",
                     "sass-loader",
                 ]
@@ -97,7 +107,9 @@ module.exports = {
                 test: /\.(ico|eot|ttf|woff|woff2)$/,
                 loader: "url-loader",
                 options: {
-                    limit: 10000
+                    limit: 10000,
+                    name: "[hash].[ext]",
+                    outputPath: "assets"
                 }
             },
             {
@@ -120,7 +132,9 @@ module.exports = {
                 exclude: /web\/static/,
                 loader: "url-loader",
                 options: {
-                    limit: 10000
+                    limit: 10000,
+                    name: "[hash].[ext]",
+                    outputPath: "assets"
                 }
             },
         ]
@@ -133,11 +147,15 @@ module.exports = {
         warningsFilter: [
             /source-map/,
             /dependency is an expression/,
-            /export 'default'/
+            /export 'default'/,
+            /Can't resolve 'sodium'/
         ],
     },
     node: {
-        fs: "empty"
+        fs: "empty",
+        "child_process": "empty",
+        net: "empty",
+        tls: "empty"
     },
     performance: {
         hints: false