Merge branch 'esm' of github.com:gchq/CyberChef into esm

.eslintignore

@@ -1,2 +1,2 @@
 src/core/vendor/**
-src/core/operations/legacy/**
+src/core/operations/legacy/**

Gruntfile.js

@@ -379,6 +379,9 @@ module.exports = function (grunt) {
             generateConfig: {
                 command: [
                     "echo '\n--- Regenerating config files. ---'",
+                    "mkdir -p src/core/config/modules",
+                    "echo 'export default {};\n' > src/core/config/modules/OpModules.mjs",
+                    "echo '[]\n' > src/core/config/OperationConfig.json",
                     "node --experimental-modules src/core/config/scripts/generateOpsIndex.mjs",
                     "node --experimental-modules src/core/config/scripts/generateConfig.mjs",
                     "echo '--- Config scripts finished. ---\n'"

package-lock.json

@@ -1897,6 +1897,14 @@
       "integrity": "sha1-tUc7M9yXxCTl2Y3IfVXU2KKci/I=",
       "dev": true
     },
+    "chi-squared": {
+      "version": "1.1.0",
+      "resolved": "https://registry.npmjs.org/chi-squared/-/chi-squared-1.1.0.tgz",
+      "integrity": "sha1-iShlz/qOCnIPkhv8nGNcGawqNG0=",
+      "requires": {
+        "gamma": "^1.0.0"
+      }
+    },
     "chokidar": {
       "version": "2.0.3",
       "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-2.0.3.tgz",
@@ -4157,8 +4165,8 @@
       "dev": true,
       "optional": true,
       "requires": {
-        "nan": "2.10.0",
-        "node-pre-gyp": "0.10.0"
+        "nan": "^2.9.2",
+        "node-pre-gyp": "^0.10.0"
       },
       "dependencies": {
         "abbrev": {
@@ -4531,10 +4539,10 @@
           "dev": true,
           "optional": true,
           "requires": {
-            "deep-extend": "0.5.1",
-            "ini": "1.3.5",
-            "minimist": "1.2.0",
-            "strip-json-comments": "2.0.1"
+            "deep-extend": "^0.5.1",
+            "ini": "~1.3.0",
+            "minimist": "^1.2.0",
+            "strip-json-comments": "~2.0.1"
           },
           "dependencies": {
             "minimist": {
@@ -4691,6 +4699,11 @@
       "integrity": "sha1-GwqzvVU7Kg1jmdKcDj6gslIHgyc=",
       "dev": true
     },
+    "gamma": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/gamma/-/gamma-1.0.0.tgz",
+      "integrity": "sha1-mDwck5/iPZMnAVhXEeHZpDDLdMs="
+    },
     "gaze": {
       "version": "1.1.2",
       "resolved": "https://registry.npmjs.org/gaze/-/gaze-1.1.2.tgz",

package.json

@@ -77,6 +77,7 @@
     "bootstrap-colorpicker": "^2.5.2",
     "bootstrap-switch": "^3.3.4",
     "bson": "^2.0.6",
+    "chi-squared": "^1.1.0",
     "crypto-api": "^0.8.0",
     "crypto-js": "^3.1.9-1",
     "ctph.js": "0.0.5",

src/core/ChefWorker.js

@@ -91,7 +91,7 @@ self.addEventListener("message", function(e) {
  */
 async function bake(data) {
     // Ensure the relevant modules are loaded
-    loadRequiredModules(data.recipeConfig);
+    self.loadRequiredModules(data.recipeConfig);
 
     try {
         const response = await self.chef.bake(
@@ -144,25 +144,6 @@ async function getDishAs(data) {
 }
 
 
-/**
- * Checks that all required modules are loaded and loads them if not.
- *
- * @param {Object} recipeConfig
- */
-function loadRequiredModules(recipeConfig) {
-    recipeConfig.forEach(op => {
-        const module = self.OperationConfig[op.op].module;
-
-        if (!OpModules.hasOwnProperty(module)) {
-            log.info(`Loading ${module} module`);
-            self.sendStatusMessage(`Loading ${module} module`);
-            self.importScripts(`${self.docURL}/${module}.js`);
-            self.sendStatusMessage("");
-        }
-    });
-}
-
-
 /**
  * Calculates highlight offsets if possible.
  *
@@ -182,6 +163,25 @@ function calculateHighlights(recipeConfig, direction, pos) {
 }
 
 
+/**
+ * Checks that all required modules are loaded and loads them if not.
+ *
+ * @param {Object} recipeConfig
+ */
+self.loadRequiredModules = function(recipeConfig) {
+    recipeConfig.forEach(op => {
+        const module = self.OperationConfig[op.op].module;
+
+        if (!OpModules.hasOwnProperty(module)) {
+            log.info(`Loading ${module} module`);
+            self.sendStatusMessage(`Loading ${module} module`);
+            self.importScripts(`${self.docURL}/${module}.js`);
+            self.sendStatusMessage("");
+        }
+    });
+};
+
+
 /**
  * Send status update to the app.
  *

src/core/Operation.mjs

@@ -188,7 +188,7 @@ class Operation {
     get config() {
         return {
             "op": this.name,
-            "args": this._ingList.map(ing => ing.conf)
+            "args": this._ingList.map(ing => ing.config)
         };
     }
 
@@ -274,6 +274,15 @@ class Operation {
         return this._flowControl;
     }
 
+    /**
+     * Set whether this Operation is a flowcontrol op.
+     *
+     * @param {boolean} value
+     */
+    set flowControl(value) {
+        this._flowControl = !!value;
+    }
+
 }
 
 export default Operation;

src/core/Recipe.mjs

@@ -172,9 +172,9 @@ class Recipe  {
                     numRegisters = state.numRegisters;
                 } else {
                     output = await op.run(input, op.ingValues);
-                    this.lastRunOp = op;
                     dish.set(output, op.outputType);
                 }
+                this.lastRunOp = op;
             } catch (err) {
                 // Return expected errors as output
                 if (err instanceof OperationError ||

src/core/Utils.mjs

@@ -720,7 +720,7 @@ class Utils {
      * @param {boolean} [newline=false] - whether to add a newline after each operation
      * @returns {string}
      */
-    static generatePrettyRecipe(recipeConfig, newline=false) {
+    static generatePrettyRecipe(recipeConfig, newline = false) {
         let prettyConfig = "",
             name = "",
             args = "",

src/core/config/Categories.json

@@ -338,6 +338,7 @@
     {
         "name": "Flow control",
         "ops": [
+            "Magic",
             "Fork",
             "Merge",
             "Register",

src/core/config/scripts/generateConfig.mjs

@@ -43,6 +43,10 @@ for (const opObj in Ops) {
         args: op.args
     };
 
+    if (op.hasOwnProperty("patterns")) {
+        operationConfig[op.name].patterns = op.patterns;
+    }
+
     if (!modules.hasOwnProperty(op.module))
         modules[op.module] = {};
     modules[op.module][op.name] = opObj;

src/core/config/scripts/portOperation.mjs

@@ -61,6 +61,8 @@ function main() {
     const utilsUsed = /Utils/.test(legacyFile);
     const esc = new EscapeString();
     const desc = esc.run(op.description, ["Special chars", "Double"]);
+    const patterns = op.hasOwnProperty("patterns") ? `
+        this.patterns = ${JSON.stringify(op.patterns, null, 4).split("\n").join("\n        ")};` : "";
 
     // Attempt to find the operation run function based on the JSDoc comment
     const regex = `\\* ${opName} operation[^:]+:(?: function ?\\(input, args\\))? ?{([\\s\\S]+?)\n    }`;
@@ -105,7 +107,7 @@ class ${moduleName} extends Operation {
         this.description = "${desc}";
         this.inputType = "${op.inputType}";
         this.outputType = "${op.outputType}";${op.manualBake ? "\n        this.manualBake = true;" : ""}
-        this.args = ${JSON.stringify(op.args, null, 4).split("\n").join("\n        ")};
+        this.args = ${JSON.stringify(op.args, null, 4).split("\n").join("\n        ")};${patterns}
     }
 
     /**
@@ -172,6 +174,30 @@ export default ${moduleName};
 
 
 const OP_CONFIG = {
+    "Magic": {
+        module: "Default",
+        description: "The Magic operation attempts to detect various properties of the input data and suggests which operations could help to make more sense of it.<br><br><b>Options</b><br><u>Depth:</u> If an operation appears to match the data, it will be run and the result will be analysed further. This argument controls the maximum number of levels of recursion.<br><br><u>Intensive mode:</u> When this is turned on, various operations like XOR, bit rotates, and character encodings are brute-forced to attempt to detect valid data underneath. To improve performance, only the first 100 bytes of the data is brute-forced.<br><br><u>Extensive language support:</u> At each stage, the relative byte frequencies of the data will be compared to average frequencies for a number of languages. The default set consists of ~40 of the most commonly used languages on the Internet. The extensive list consists of 284 languages and can result in many languages matching the data if their byte frequencies are similar.",
+        inputType: "ArrayBuffer",
+        outputType: "html",
+        flowControl: true,
+        args: [
+            {
+                name: "Depth",
+                type: "number",
+                value: 3
+            },
+            {
+                name: "Intensive mode",
+                type: "boolean",
+                value: false
+            },
+            {
+                name: "Extensive language support",
+                type: "boolean",
+                value: false
+            }
+        ]
+    },
     "Fork": {
         module: "Default",
         description: "Split the input data up based on the specified delimiter and run all subsequent operations on each branch separately.<br><br>For example, to decode multiple Base64 strings, enter them all on separate lines then add the 'Fork' and 'From Base64' operations to the recipe. Each string will be decoded separately.",
@@ -330,6 +356,73 @@ const OP_CONFIG = {
                 type: "boolean",
                 value: "Base64.REMOVE_NON_ALPH_CHARS"
             }
+        ],
+        patterns: [
+            {
+                match: "^(?:[A-Z\\d+/]{4})+(?:[A-Z\\d+/]{2}==|[A-Z\\d+/]{3}=)?$",
+                flags: "i",
+                args: ["A-Za-z0-9+/=", false]
+            },
+            {
+                match: "^[A-Z\\d\\-_]{20,}$",
+                flags: "i",
+                args: ["A-Za-z0-9-_", false]
+            },
+            {
+                match: "^(?:[A-Z\\d+\\-]{4}){5,}(?:[A-Z\\d+\\-]{2}==|[A-Z\\d+\\-]{3}=)?$",
+                flags: "i",
+                args: ["A-Za-z0-9+\\-=", false]
+            },
+            {
+                match: "^(?:[A-Z\\d./]{4}){5,}(?:[A-Z\\d./]{2}==|[A-Z\\d./]{3}=)?$",
+                flags: "i",
+                args: ["./0-9A-Za-z=", false]
+            },
+            {
+                match: "^[A-Z\\d_.]{20,}$",
+                flags: "i",
+                args: ["A-Za-z0-9_.", false]
+            },
+            {
+                match: "^(?:[A-Z\\d._]{4}){5,}(?:[A-Z\\d._]{2}--|[A-Z\\d._]{3}-)?$",
+                flags: "i",
+                args: ["A-Za-z0-9._-", false]
+            },
+            {
+                match: "^(?:[A-Z\\d+/]{4}){5,}(?:[A-Z\\d+/]{2}==|[A-Z\\d+/]{3}=)?$",
+                flags: "i",
+                args: ["0-9a-zA-Z+/=", false]
+            },
+            {
+                match: "^(?:[A-Z\\d+/]{4}){5,}(?:[A-Z\\d+/]{2}==|[A-Z\\d+/]{3}=)?$",
+                flags: "i",
+                args: ["0-9A-Za-z+/=", false]
+            },
+            {
+                match: "^[ !\"#$%&'()*+,\\-./\\d:;<=>?@A-Z[\\\\\\]^_]{20,}$",
+                flags: "",
+                args: [" -_", false]
+            },
+            {
+                match: "^[A-Z\\d+\\-]{20,}$",
+                flags: "i",
+                args: ["+\\-0-9A-Za-z", false]
+            },
+            {
+                match: "^[!\"#$%&'()*+,\\-0-689@A-NP-VX-Z[`a-fh-mp-r]{20,}$",
+                flags: "",
+                args: ["!-,-0-689@A-NP-VX-Z[`a-fh-mp-r", false]
+            },
+            {
+                match: "^(?:[N-ZA-M\\d+/]{4}){5,}(?:[N-ZA-M\\d+/]{2}==|[N-ZA-M\\d+/]{3}=)?$",
+                flags: "i",
+                args: ["N-ZA-Mn-za-m0-9+/=", false]
+            },
+            {
+                match: "^[A-Z\\d./]{20,}$",
+                flags: "i",
+                args: ["./0-9A-Za-z", false]
+            },
         ]
     },
     "To Base64": {
@@ -363,6 +456,18 @@ const OP_CONFIG = {
                 type: "boolean",
                 value: "Base58.REMOVE_NON_ALPH_CHARS"
             }
+        ],
+        patterns: [
+            {
+                match: "^[1-9A-HJ-NP-Za-km-z]{20,}$",
+                flags: "",
+                args: ["123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz", false]
+            },
+            {
+                match: "^[1-9A-HJ-NP-Za-km-z]{20,}$",
+                flags: "",
+                args: ["rpshnaf39wBUDNEGHJKLM4PQRST7VWXYZ2bcdeCg65jkm8oFqi1tuvAxyz", false]
+            },
         ]
     },
     "To Base58": {
@@ -394,6 +499,13 @@ const OP_CONFIG = {
                 type: "boolean",
                 value: "Base64.REMOVE_NON_ALPH_CHARS"
             }
+        ],
+        patterns: [
+            {
+                match: "^(?:[A-Z2-7]{8})+(?:[A-Z2-7]{2}={6}|[A-Z2-7]{4}={4}|[A-Z2-7]{5}={3}|[A-Z2-7]{7}={1})?$",
+                flags: "",
+                args: ["A-Z2-7=", false]
+            },
         ]
     },
     "To Base32": {
@@ -744,6 +856,53 @@ const OP_CONFIG = {
                 type: "option",
                 value: "ByteRepr.HEX_DELIM_OPTIONS"
             }
+        ],
+        patterns: [
+            {
+                match: "^(?:[\\dA-F]{2})+$",
+                flags: "i",
+                args: ["None"]
+            },
+            {
+                match: "^[\\dA-F]{2}(?: [\\dA-F]{2})*$",
+                flags: "i",
+                args: ["Space"]
+            },
+            {
+                match: "^[\\dA-F]{2}(?:,[\\dA-F]{2})*$",
+                flags: "i",
+                args: ["Comma"]
+            },
+            {
+                match: "^[\\dA-F]{2}(?:;[\\dA-F]{2})*$",
+                flags: "i",
+                args: ["Semi-colon"]
+            },
+            {
+                match: "^[\\dA-F]{2}(?::[\\dA-F]{2})*$",
+                flags: "i",
+                args: ["Colon"]
+            },
+            {
+                match: "^[\\dA-F]{2}(?:\\n[\\dA-F]{2})*$",
+                flags: "i",
+                args: ["Line feed"]
+            },
+            {
+                match: "^[\\dA-F]{2}(?:\\r\\n[\\dA-F]{2})*$",
+                flags: "i",
+                args: ["CRLF"]
+            },
+            {
+                match: "^[\\dA-F]{2}(?:0x[\\dA-F]{2})*$",
+                flags: "i",
+                args: ["0x"]
+            },
+            {
+                match: "^[\\dA-F]{2}(?:\\\\x[\\dA-F]{2})*$",
+                flags: "i",
+                args: ["\\x"]
+            }
         ]
     },
     "To Hex": {
@@ -774,6 +933,38 @@ const OP_CONFIG = {
                 type: "option",
                 value: "ByteRepr.DELIM_OPTIONS"
             }
+        ],
+        patterns: [
+            {
+                match: "^(?:[0-7]{1,2}|[123][0-7]{2})(?: (?:[0-7]{1,2}|[123][0-7]{2}))*$",
+                flags: "",
+                args: ["Space"]
+            },
+            {
+                match: "^(?:[0-7]{1,2}|[123][0-7]{2})(?:,(?:[0-7]{1,2}|[123][0-7]{2}))*$",
+                flags: "",
+                args: ["Comma"]
+            },
+            {
+                match: "^(?:[0-7]{1,2}|[123][0-7]{2})(?:;(?:[0-7]{1,2}|[123][0-7]{2}))*$",
+                flags: "",
+                args: ["Semi-colon"]
+            },
+            {
+                match: "^(?:[0-7]{1,2}|[123][0-7]{2})(?::(?:[0-7]{1,2}|[123][0-7]{2}))*$",
+                flags: "",
+                args: ["Colon"]
+            },
+            {
+                match: "^(?:[0-7]{1,2}|[123][0-7]{2})(?:\\n(?:[0-7]{1,2}|[123][0-7]{2}))*$",
+                flags: "",
+                args: ["Line feed"]
+            },
+            {
+                match: "^(?:[0-7]{1,2}|[123][0-7]{2})(?:\\r\\n(?:[0-7]{1,2}|[123][0-7]{2}))*$",
+                flags: "",
+                args: ["CRLF"]
+            },
         ]
     },
     "To Octal": {
@@ -811,7 +1002,6 @@ const OP_CONFIG = {
             }
         ]
     },
-
     "To Charcode": {
         module: "Default",
         description: "Converts text to its unicode character code equivalent.<br><br>e.g. <code>Γειά σου</code> becomes <code>0393 03b5 03b9 03ac 20 03c3 03bf 03c5</code>",
@@ -845,6 +1035,43 @@ const OP_CONFIG = {
                 type: "option",
                 value: "ByteRepr.BIN_DELIM_OPTIONS"
             }
+        ],
+        patterns: [
+            {
+                match: "^(?:[01]{8})+$",
+                flags: "",
+                args: ["None"]
+            },
+            {
+                match: "^(?:[01]{8})(?: [01]{8})*$",
+                flags: "",
+                args: ["Space"]
+            },
+            {
+                match: "^(?:[01]{8})(?:,[01]{8})*$",
+                flags: "",
+                args: ["Comma"]
+            },
+            {
+                match: "^(?:[01]{8})(?:;[01]{8})*$",
+                flags: "",
+                args: ["Semi-colon"]
+            },
+            {
+                match: "^(?:[01]{8})(?::[01]{8})*$",
+                flags: "",
+                args: ["Colon"]
+            },
+            {
+                match: "^(?:[01]{8})(?:\\n[01]{8})*$",
+                flags: "",
+                args: ["Line feed"]
+            },
+            {
+                match: "^(?:[01]{8})(?:\\r\\n[01]{8})*$",
+                flags: "",
+                args: ["CRLF"]
+            },
         ]
     },
     "To Binary": {
@@ -873,6 +1100,38 @@ const OP_CONFIG = {
                 type: "option",
                 value: "ByteRepr.DELIM_OPTIONS"
             }
+        ],
+        patterns: [
+            {
+                match: "^(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5])(?: (?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5]))*$",
+                flags: "",
+                args: ["Space"]
+            },
+            {
+                match: "^(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5])(?:,(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5]))*$",
+                flags: "",
+                args: ["Comma"]
+            },
+            {
+                match: "^(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5])(?:;(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5]))*$",
+                flags: "",
+                args: ["Semi-colon"]
+            },
+            {
+                match: "^(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5])(?::(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5]))*$",
+                flags: "",
+                args: ["Colon"]
+            },
+            {
+                match: "^(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5])(?:\\n(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5]))*$",
+                flags: "",
+                args: ["Line feed"]
+            },
+            {
+                match: "^(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5])(?:\\r\\n(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5]))*$",
+                flags: "",
+                args: ["CRLF"]
+            },
         ]
     },
     "To Decimal": {
@@ -895,7 +1154,14 @@ const OP_CONFIG = {
         highlightReverse: "func",
         inputType: "string",
         outputType: "byteArray",
-        args: []
+        args: [],
+        patterns: [
+            {
+                match: "^(?:(?:[\\dA-F]{4,16}:?)?\\s*((?:[\\dA-F]{2}\\s){1,8}(?:\\s|[\\dA-F]{2}-)(?:[\\dA-F]{2}\\s){1,8}|(?:[\\dA-F]{2}\\s|[\\dA-F]{4}\\s)+)[^\\n]*\\n?)+$",
+                flags: "i",
+                args: []
+            },
+        ]
     },
     "To Hexdump": {
         module: "Default",
@@ -953,7 +1219,14 @@ const OP_CONFIG = {
         description: "Converts HTML entities back to characters<br><br>e.g. <code>&amp;<span>amp;</span></code> becomes <code>&amp;</code>", // <span> tags required to stop the browser just printing &
         inputType: "string",
         outputType: "string",
-        args: []
+        args: [],
+        patterns: [
+            {
+                match: "&(?:#\\d{2,3}|#x[\\da-f]{2}|[a-z]{2,6});",
+                flags: "i",
+                args: []
+            },
+        ]
     },
     "To HTML Entity": {
         module: "Default",
@@ -996,7 +1269,14 @@ const OP_CONFIG = {
         description: "Converts URI/URL percent-encoded characters back to their raw values.<br><br>e.g. <code>%3d</code> becomes <code>=</code>",
         inputType: "string",
         outputType: "string",
-        args: []
+        args: [],
+        patterns: [
+            {
+                match: ".*(?:%[\\da-f]{2}.*){4}",
+                flags: "i",
+                args: []
+            },
+        ]
     },
     "URL Encode": {
         module: "URL",
@@ -1057,6 +1337,23 @@ const OP_CONFIG = {
                 type: "boolean",
                 value: true
             }
+        ],
+        patterns: [
+            {
+                match: "\\\\u(?:[\\da-f]{4,6})",
+                flags: "i",
+                args: ["\\u"]
+            },
+            {
+                match: "%u(?:[\\da-f]{4,6})",
+                flags: "i",
+                args: ["%u"]
+            },
+            {
+                match: "U\\+(?:[\\da-f]{4,6})",
+                flags: "i",
+                args: ["U+"]
+            },
         ]
     },
     "From Quoted Printable": {
@@ -1064,7 +1361,14 @@ const OP_CONFIG = {
         description: "Converts QP-encoded text back to standard text.",
         inputType: "string",
         outputType: "byteArray",
-        args: []
+        args: [],
+        patterns: [
+            {
+                match: "^[\\x21-\\x3d\\x3f-\\x7e \\t]*(?:=[\\da-f]{2}|=\\r?\\n)(?:[\\x21-\\x3d\\x3f-\\x7e \\t]|=[\\da-f]{2}|=\\r?\\n)*$",
+                flags: "i",
+                args: []
+            },
+        ]
     },
     "To Quoted Printable": {
         module: "Default",
@@ -1226,7 +1530,7 @@ const OP_CONFIG = {
                 type: "option",
                 value: "Object.keys(CharEnc.IO_FORMAT),"
             },
-        ]
+        ],
     },
     "Decode text": {
         module: "CharEnc",
@@ -2573,6 +2877,28 @@ const OP_CONFIG = {
                 type: "option",
                 value: "DateTime.UNITS"
             }
+        ],
+        patterns: [
+            {
+                match: "^1?\\d{9}$",
+                flags: "",
+                args: ["Seconds (s)"]
+            },
+            {
+                match: "^1?\\d{12}$",
+                flags: "",
+                args: ["Milliseconds (ms)"]
+            },
+            {
+                match: "^1?\\d{15}$",
+                flags: "",
+                args: ["Microseconds (μs)"]
+            },
+            {
+                match: "^1?\\d{18}$",
+                flags: "",
+                args: ["Nanoseconds (ns)"]
+            },
         ]
     },
     "To UNIX Timestamp": {
@@ -2885,6 +3211,13 @@ const OP_CONFIG = {
                 type: "boolean",
                 value: "Compress.INFLATE_VERIFY"
             }
+        ],
+        patterns: [
+            {
+                match: "^\\x78(\\x01|\\x9c|\\xda|\\x5e)",
+                flags: "",
+                args: [0, 0, "Adaptive", false, false]
+            },
         ]
     },
     "Gzip": {
@@ -2920,7 +3253,14 @@ const OP_CONFIG = {
         description: "Decompresses data which has been compressed using the deflate algorithm with gzip headers.",
         inputType: "byteArray",
         outputType: "byteArray",
-        args: []
+        args: [],
+        patterns: [
+            {
+                match: "^\\x1f\\x8b\\x08",
+                flags: "",
+                args: []
+            },
+        ]
     },
     "Zip": {
         module: "Compression",
@@ -2976,6 +3316,13 @@ const OP_CONFIG = {
                 type: "boolean",
                 value: "Compress.PKUNZIP_VERIFY"
             }
+        ],
+        patterns: [
+            {
+                match: "^\\x50\\x4b(?:\\x03|\\x05|\\x07)(?:\\x04|\\x06|\\x08)",
+                flags: "",
+                args: ["", false]
+            },
         ]
     },
     "Bzip2 Decompress": {
@@ -2983,7 +3330,14 @@ const OP_CONFIG = {
         description: "Decompresses data using the Bzip2 algorithm.",
         inputType: "byteArray",
         outputType: "string",
-        args: []
+        args: [],
+        patterns: [
+            {
+                match: "^\\x42\\x5a\\x68",
+                flags: "",
+                args: []
+            },
+        ]
     },
     "Generic Code Beautify": {
         module: "Code",
@@ -3468,7 +3822,7 @@ const OP_CONFIG = {
     },
     "Chi Square": {
         module: "Default",
-        description: "Calculates the Chi Square distribution of values.",
+        description: "Calculates the Chi-Squared distribution of values.",
         inputType: "ArrayBuffer",
         outputType: "number",
         args: []
@@ -3491,6 +3845,13 @@ const OP_CONFIG = {
                 type: "option",
                 value: "PublicKey.X509_INPUT_FORMAT"
             }
+        ],
+        patterns: [
+            {
+                match: "^-+BEGIN CERTIFICATE-+\\r?\\n[\\da-z+/\\n\\r]+-+END CERTIFICATE-+\\r?\\n?$",
+                flags: "i",
+                args: ["PEM"]
+            },
         ]
     },
     "PEM to Hex": {
@@ -3783,6 +4144,13 @@ const OP_CONFIG = {
                 type: "option",
                 value: "MorseCode.WORD_DELIM_OPTIONS"
             }
+        ],
+        patterns: [
+            {
+                match: "(?:^[-. \\n]{5,}$|^[_. \\n]{5,}$|^(?:dash|dot| |\\n){5,}$)",
+                flags: "i",
+                args: ["Space", "Line feed"]
+            },
         ]
     },
     "Tar": {
@@ -3803,7 +4171,13 @@ const OP_CONFIG = {
         description: "Unpacks a tarball and displays it per file.",
         inputType: "byteArray",
         outputType: "html",
-        args: [
+        args: [],
+        patterns: [
+            {
+                match: "^.{257}\\x75\\x73\\x74\\x61\\x72",
+                flags: "",
+                args: []
+            },
         ]
     },
     "Head": {
@@ -3945,6 +4319,14 @@ const OP_CONFIG = {
                 type: "option",
                 value: "Image.INPUT_FORMAT"
             }
+        ],
+        patterns: [
+            {
+                match: "^(?:\\xff\\xd8\\xff|\\x89\\x50\\x4e\\x47|\\x47\\x49\\x46|.{8}\\x57\\x45\\x42\\x50|\\x42\\x4d)",
+                flags: "",
+                args: ["Raw"],
+                useful: true
+            },
         ]
     },
     "Remove EXIF": {
@@ -4026,8 +4408,14 @@ const OP_CONFIG = {
                 type: "option",
                 value: "BCD.FORMAT"
             }
+        ],
+        patterns: [
+            {
+                match: "^(?:\\d{4} ){3,}\\d{4}$",
+                flags: "",
+                args: ["8 4 2 1", true, false, "Nibbles"]
+            },
         ]
-
     },
     "To BCD": {
         module: "Default",

src/core/operations/DetectFileType.mjs

@@ -0,0 +1,54 @@
+/**
+ * @author n1474335 [n1474335@gmail.com]
+ * @copyright Crown Copyright 2016
+ * @license Apache-2.0
+ */
+
+import Operation from "../Operation";
+import Magic from "../lib/Magic";
+
+/**
+ * Detect File Type operation
+ */
+class DetectFileType extends Operation {
+
+    /**
+     * DetectFileType constructor
+     */
+    constructor() {
+        super();
+
+        this.name = "Detect File Type";
+        this.module = "Default";
+        this.description = "Attempts to guess the MIME (Multipurpose Internet Mail Extensions) type of the data based on 'magic bytes'.<br><br>Currently supports the following file types: 7z, amr, avi, bmp, bz2, class, cr2, crx, dex, dmg, doc, elf, eot, epub, exe, flac, flv, gif, gz, ico, iso, jpg, jxr, m4a, m4v, mid, mkv, mov, mp3, mp4, mpg, ogg, otf, pdf, png, ppt, ps, psd, rar, rtf, sqlite, swf, tar, tar.z, tif, ttf, utf8, vmdk, wav, webm, webp, wmv, woff, woff2, xls, xz, zip.";
+        this.inputType = "ArrayBuffer";
+        this.outputType = "string";
+        this.args = [];
+    }
+
+    /**
+     * @param {ArrayBuffer} input
+     * @param {Object[]} args
+     * @returns {string}
+     */
+    run(input, args) {
+        const data = new Uint8Array(input),
+            type = Magic.magicFileType(data);
+
+        if (!type) {
+            return "Unknown file type. Have you tried checking the entropy of this data to determine whether it might be encrypted or compressed?";
+        } else {
+            let output = "File extension: " + type.ext + "\n" +
+                "MIME type:      " + type.mime;
+
+            if (type.desc && type.desc.length) {
+                output += "\nDescription:    " + type.desc;
+            }
+
+            return output;
+        }
+    }
+
+}
+
+export default DetectFileType;

src/core/operations/EscapeUnicodeCharacters.mjs

@@ -44,6 +44,23 @@ class EscapeUnicodeCharacters extends Operation {
                 "value": true
             }
         ];
+        this.patterns = [
+            {
+                match: "\\\\u(?:[\\da-f]{4,6})",
+                flags: "i",
+                args: ["\\u"]
+            },
+            {
+                match: "%u(?:[\\da-f]{4,6})",
+                flags: "i",
+                args: ["%u"]
+            },
+            {
+                match: "U\\+(?:[\\da-f]{4,6})",
+                flags: "i",
+                args: ["U+"]
+            },
+        ];
     }
 
     /**

src/core/operations/FromBCD.mjs

@@ -48,6 +48,13 @@ class FromBCD extends Operation {
                 "value": FORMAT
             }
         ];
+        this.patterns = [
+            {
+                match: "^(?:\\d{4} ){3,}\\d{4}$",
+                flags: "",
+                args: ["8 4 2 1", true, false, "Nibbles"]
+            },
+        ];
     }
 
     /**

src/core/operations/FromBase32.mjs

@@ -35,6 +35,13 @@ class FromBase32 extends Operation {
                 value: true
             }
         ];
+        this.patterns = [
+            {
+                match: "^(?:[A-Z2-7]{8})+(?:[A-Z2-7]{2}={6}|[A-Z2-7]{4}={4}|[A-Z2-7]{5}={3}|[A-Z2-7]{7}={1})?$",
+                flags: "",
+                args: ["A-Z2-7=", false]
+            },
+        ];
     }
 
     /**

src/core/operations/FromBase58.mjs

@@ -37,6 +37,18 @@ class FromBase58 extends Operation {
                 "value": true
             }
         ];
+        this.patterns = [
+            {
+                match: "^[1-9A-HJ-NP-Za-km-z]{20,}$",
+                flags: "",
+                args: ["123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz", false]
+            },
+            {
+                match: "^[1-9A-HJ-NP-Za-km-z]{20,}$",
+                flags: "",
+                args: ["rpshnaf39wBUDNEGHJKLM4PQRST7VWXYZ2bcdeCg65jkm8oFqi1tuvAxyz", false]
+            },
+        ];
     }
 
     /**

src/core/operations/FromBase64.mjs

@@ -35,6 +35,73 @@ class FromBase64 extends Operation {
                 value: true
             }
         ];
+        this.patterns = [
+            {
+                match: "^(?:[A-Z\\d+/]{4})+(?:[A-Z\\d+/]{2}==|[A-Z\\d+/]{3}=)?$",
+                flags: "i",
+                args: ["A-Za-z0-9+/=", false]
+            },
+            {
+                match: "^[A-Z\\d\\-_]{20,}$",
+                flags: "i",
+                args: ["A-Za-z0-9-_", false]
+            },
+            {
+                match: "^(?:[A-Z\\d+\\-]{4}){5,}(?:[A-Z\\d+\\-]{2}==|[A-Z\\d+\\-]{3}=)?$",
+                flags: "i",
+                args: ["A-Za-z0-9+\\-=", false]
+            },
+            {
+                match: "^(?:[A-Z\\d./]{4}){5,}(?:[A-Z\\d./]{2}==|[A-Z\\d./]{3}=)?$",
+                flags: "i",
+                args: ["./0-9A-Za-z=", false]
+            },
+            {
+                match: "^[A-Z\\d_.]{20,}$",
+                flags: "i",
+                args: ["A-Za-z0-9_.", false]
+            },
+            {
+                match: "^(?:[A-Z\\d._]{4}){5,}(?:[A-Z\\d._]{2}--|[A-Z\\d._]{3}-)?$",
+                flags: "i",
+                args: ["A-Za-z0-9._-", false]
+            },
+            {
+                match: "^(?:[A-Z\\d+/]{4}){5,}(?:[A-Z\\d+/]{2}==|[A-Z\\d+/]{3}=)?$",
+                flags: "i",
+                args: ["0-9a-zA-Z+/=", false]
+            },
+            {
+                match: "^(?:[A-Z\\d+/]{4}){5,}(?:[A-Z\\d+/]{2}==|[A-Z\\d+/]{3}=)?$",
+                flags: "i",
+                args: ["0-9A-Za-z+/=", false]
+            },
+            {
+                match: "^[ !\"#$%&'()*+,\\-./\\d:;<=>?@A-Z[\\\\\\]^_]{20,}$",
+                flags: "",
+                args: [" -_", false]
+            },
+            {
+                match: "^[A-Z\\d+\\-]{20,}$",
+                flags: "i",
+                args: ["+\\-0-9A-Za-z", false]
+            },
+            {
+                match: "^[!\"#$%&'()*+,\\-0-689@A-NP-VX-Z[`a-fh-mp-r]{20,}$",
+                flags: "",
+                args: ["!-,-0-689@A-NP-VX-Z[`a-fh-mp-r", false]
+            },
+            {
+                match: "^(?:[N-ZA-M\\d+/]{4}){5,}(?:[N-ZA-M\\d+/]{2}==|[N-ZA-M\\d+/]{3}=)?$",
+                flags: "i",
+                args: ["N-ZA-Mn-za-m0-9+/=", false]
+            },
+            {
+                match: "^[A-Z\\d./]{20,}$",
+                flags: "i",
+                args: ["./0-9A-Za-z", false]
+            },
+        ];
     }
 
     /**

src/core/operations/FromBinary.mjs

@@ -31,6 +31,43 @@ class FromBinary extends Operation {
                 "value": BIN_DELIM_OPTIONS
             }
         ];
+        this.patterns = [
+            {
+                match: "^(?:[01]{8})+$",
+                flags: "",
+                args: ["None"]
+            },
+            {
+                match: "^(?:[01]{8})(?: [01]{8})*$",
+                flags: "",
+                args: ["Space"]
+            },
+            {
+                match: "^(?:[01]{8})(?:,[01]{8})*$",
+                flags: "",
+                args: ["Comma"]
+            },
+            {
+                match: "^(?:[01]{8})(?:;[01]{8})*$",
+                flags: "",
+                args: ["Semi-colon"]
+            },
+            {
+                match: "^(?:[01]{8})(?::[01]{8})*$",
+                flags: "",
+                args: ["Colon"]
+            },
+            {
+                match: "^(?:[01]{8})(?:\\n[01]{8})*$",
+                flags: "",
+                args: ["Line feed"]
+            },
+            {
+                match: "^(?:[01]{8})(?:\\r\\n[01]{8})*$",
+                flags: "",
+                args: ["CRLF"]
+            },
+        ];
     }
 
     /**

src/core/operations/FromDecimal.mjs

@@ -31,6 +31,38 @@ class FromDecimal extends Operation {
                 "value": DELIM_OPTIONS
             }
         ];
+        this.patterns = [
+            {
+                match: "^(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5])(?: (?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5]))*$",
+                flags: "",
+                args: ["Space"]
+            },
+            {
+                match: "^(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5])(?:,(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5]))*$",
+                flags: "",
+                args: ["Comma"]
+            },
+            {
+                match: "^(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5])(?:;(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5]))*$",
+                flags: "",
+                args: ["Semi-colon"]
+            },
+            {
+                match: "^(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5])(?::(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5]))*$",
+                flags: "",
+                args: ["Colon"]
+            },
+            {
+                match: "^(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5])(?:\\n(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5]))*$",
+                flags: "",
+                args: ["Line feed"]
+            },
+            {
+                match: "^(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5])(?:\\r\\n(?:\\d{1,2}|1\\d{2}|2[0-4]\\d|25[0-5]))*$",
+                flags: "",
+                args: ["CRLF"]
+            },
+        ];
     }
 
     /**

src/core/operations/FromHTMLEntity.mjs

@@ -24,6 +24,13 @@ class FromHTMLEntity extends Operation {
         this.inputType = "string";
         this.outputType = "string";
         this.args = [];
+        this.patterns = [
+            {
+                match: "&(?:#\\d{2,3}|#x[\\da-f]{2}|[a-z]{2,6});",
+                flags: "i",
+                args: []
+            },
+        ];
     }
 
     /**

src/core/operations/FromHex.mjs

@@ -31,6 +31,53 @@ class FromHex extends Operation {
                 value: FROM_HEX_DELIM_OPTIONS
             }
         ];
+        this.patterns = [
+            {
+                match: "^(?:[\\dA-F]{2})+$",
+                flags: "i",
+                args: ["None"]
+            },
+            {
+                match: "^[\\dA-F]{2}(?: [\\dA-F]{2})*$",
+                flags: "i",
+                args: ["Space"]
+            },
+            {
+                match: "^[\\dA-F]{2}(?:,[\\dA-F]{2})*$",
+                flags: "i",
+                args: ["Comma"]
+            },
+            {
+                match: "^[\\dA-F]{2}(?:;[\\dA-F]{2})*$",
+                flags: "i",
+                args: ["Semi-colon"]
+            },
+            {
+                match: "^[\\dA-F]{2}(?::[\\dA-F]{2})*$",
+                flags: "i",
+                args: ["Colon"]
+            },
+            {
+                match: "^[\\dA-F]{2}(?:\\n[\\dA-F]{2})*$",
+                flags: "i",
+                args: ["Line feed"]
+            },
+            {
+                match: "^[\\dA-F]{2}(?:\\r\\n[\\dA-F]{2})*$",
+                flags: "i",
+                args: ["CRLF"]
+            },
+            {
+                match: "^[\\dA-F]{2}(?:0x[\\dA-F]{2})*$",
+                flags: "i",
+                args: ["0x"]
+            },
+            {
+                match: "^[\\dA-F]{2}(?:\\\\x[\\dA-F]{2})*$",
+                flags: "i",
+                args: ["\\x"]
+            }
+        ];
     }
 
     /**

src/core/operations/FromHexdump.mjs

@@ -24,6 +24,13 @@ class FromHexdump extends Operation {
         this.inputType = "string";
         this.outputType = "byteArray";
         this.args = [];
+        this.patterns = [
+            {
+                match: "^(?:(?:[\\dA-F]{4,16}:?)?\\s*((?:[\\dA-F]{2}\\s){1,8}(?:\\s|[\\dA-F]{2}-)(?:[\\dA-F]{2}\\s){1,8}|(?:[\\dA-F]{2}\\s|[\\dA-F]{4}\\s)+)[^\\n]*\\n?)+$",
+                flags: "i",
+                args: []
+            },
+        ];
     }
 
     /**

src/core/operations/FromMorseCode.mjs

@@ -36,6 +36,13 @@ class FromMorseCode extends Operation {
                 "value": WORD_DELIM_OPTIONS
             }
         ];
+        this.patterns = [
+            {
+                match: "(?:^[-. \\n]{5,}$|^[_. \\n]{5,}$|^(?:dash|dot| |\\n){5,}$)",
+                flags: "i",
+                args: ["Space", "Line feed"]
+            },
+        ];
     }
 
     /**

src/core/operations/FromOctal.mjs

@@ -31,6 +31,38 @@ class FromOctal extends Operation {
                 "value": DELIM_OPTIONS
             }
         ];
+        this.patterns = [
+            {
+                match: "^(?:[0-7]{1,2}|[123][0-7]{2})(?: (?:[0-7]{1,2}|[123][0-7]{2}))*$",
+                flags: "",
+                args: ["Space"]
+            },
+            {
+                match: "^(?:[0-7]{1,2}|[123][0-7]{2})(?:,(?:[0-7]{1,2}|[123][0-7]{2}))*$",
+                flags: "",
+                args: ["Comma"]
+            },
+            {
+                match: "^(?:[0-7]{1,2}|[123][0-7]{2})(?:;(?:[0-7]{1,2}|[123][0-7]{2}))*$",
+                flags: "",
+                args: ["Semi-colon"]
+            },
+            {
+                match: "^(?:[0-7]{1,2}|[123][0-7]{2})(?::(?:[0-7]{1,2}|[123][0-7]{2}))*$",
+                flags: "",
+                args: ["Colon"]
+            },
+            {
+                match: "^(?:[0-7]{1,2}|[123][0-7]{2})(?:\\n(?:[0-7]{1,2}|[123][0-7]{2}))*$",
+                flags: "",
+                args: ["Line feed"]
+            },
+            {
+                match: "^(?:[0-7]{1,2}|[123][0-7]{2})(?:\\r\\n(?:[0-7]{1,2}|[123][0-7]{2}))*$",
+                flags: "",
+                args: ["CRLF"]
+            },
+        ];
     }
 
     /**

src/core/operations/FromQuotedPrintable.mjs

@@ -27,6 +27,13 @@ class FromQuotedPrintable extends Operation {
         this.inputType = "string";
         this.outputType = "byteArray";
         this.args = [];
+        this.patterns = [
+            {
+                match: "^[\\x21-\\x3d\\x3f-\\x7e \\t]*(?:=[\\da-f]{2}|=\\r?\\n)(?:[\\x21-\\x3d\\x3f-\\x7e \\t]|=[\\da-f]{2}|=\\r?\\n)*$",
+                flags: "i",
+                args: []
+            },
+        ];
     }
 
     /**

src/core/operations/FromUNIXTimestamp.mjs

@@ -32,6 +32,28 @@ class FromUNIXTimestamp extends Operation {
                 "value": UNITS
             }
         ];
+        this.patterns = [
+            {
+                match: "^1?\\d{9}$",
+                flags: "",
+                args: ["Seconds (s)"]
+            },
+            {
+                match: "^1?\\d{12}$",
+                flags: "",
+                args: ["Milliseconds (ms)"]
+            },
+            {
+                match: "^1?\\d{15}$",
+                flags: "",
+                args: ["Microseconds (μs)"]
+            },
+            {
+                match: "^1?\\d{18}$",
+                flags: "",
+                args: ["Nanoseconds (ns)"]
+            },
+        ];
     }
 
     /**

src/core/operations/Gunzip.mjs

@@ -26,6 +26,13 @@ class Gunzip extends Operation {
         this.inputType = "ArrayBuffer";
         this.outputType = "ArrayBuffer";
         this.args = [];
+        this.patterns = [
+            {
+                match: "^\\x1f\\x8b\\x08",
+                flags: "",
+                args: []
+            },
+        ];
     }
 
     /**

src/core/operations/Magic.mjs

@@ -0,0 +1,140 @@
+/**
+ * @author n1474335 [n1474335@gmail.com]
+ * @copyright Crown Copyright 2016
+ * @license Apache-2.0
+ */
+
+import Operation from "../Operation";
+import Utils from "../Utils";
+import Dish from "../Dish";
+import MagicLib from "../lib/Magic";
+
+/**
+ * Magic operation
+ */
+class Magic extends Operation {
+
+    /**
+     * Magic constructor
+     */
+    constructor() {
+        super();
+
+        this.name = "Magic";
+        this.flowControl = true;
+        this.module = "Default";
+        this.description = "The Magic operation attempts to detect various properties of the input data and suggests which operations could help to make more sense of it.<br><br><b>Options</b><br><u>Depth:</u> If an operation appears to match the data, it will be run and the result will be analysed further. This argument controls the maximum number of levels of recursion.<br><br><u>Intensive mode:</u> When this is turned on, various operations like XOR, bit rotates, and character encodings are brute-forced to attempt to detect valid data underneath. To improve performance, only the first 100 bytes of the data is brute-forced.<br><br><u>Extensive language support:</u> At each stage, the relative byte frequencies of the data will be compared to average frequencies for a number of languages. The default set consists of ~40 of the most commonly used languages on the Internet. The extensive list consists of 284 languages and can result in many languages matching the data if their byte frequencies are similar.";
+        this.inputType = "ArrayBuffer";
+        this.outputType = "html";
+        this.args = [
+            {
+                "name": "Depth",
+                "type": "number",
+                "value": 3
+            },
+            {
+                "name": "Intensive mode",
+                "type": "boolean",
+                "value": false
+            },
+            {
+                "name": "Extensive language support",
+                "type": "boolean",
+                "value": false
+            }
+        ];
+    }
+
+    /**
+     * @param {Object} state - The current state of the recipe.
+     * @param {number} state.progress - The current position in the recipe.
+     * @param {Dish} state.dish - The Dish being operated on.
+     * @param {Operation[]} state.opList - The list of operations in the recipe.
+     * @returns {Object} The updated state of the recipe.
+     */
+    async run(state) {
+        const ings = state.opList[state.progress].ingValues,
+            [depth, intensive, extLang] = ings,
+            dish = state.dish,
+            currentRecipeConfig = state.opList.map(op => op.config),
+            magic = new MagicLib(await dish.get(Dish.ARRAY_BUFFER)),
+            options = await magic.speculativeExecution(depth, extLang, intensive);
+
+        let output = `<table
+                class='table table-hover table-condensed table-bordered'
+                style='table-layout: fixed;'>
+            <tr>
+                <th>Recipe (click to load)</th>
+                <th>Result snippet</th>
+                <th>Properties</th>
+            </tr>`;
+
+        /**
+         * Returns a CSS colour value based on an integer input.
+         *
+         * @param {number} val
+         * @returns {string}
+         */
+        function chooseColour(val) {
+            if (val < 3) return "green";
+            if (val < 5) return "goldenrod";
+            return "red";
+        }
+
+        options.forEach(option => {
+            // Construct recipe URL
+            // Replace this Magic op with the generated recipe
+            const recipeConfig = currentRecipeConfig.slice(0, state.progress)
+                    .concat(option.recipe)
+                    .concat(currentRecipeConfig.slice(state.progress + 1)),
+                recipeURL = "recipe=" + Utils.encodeURIFragment(Utils.generatePrettyRecipe(recipeConfig));
+
+            let language = "",
+                fileType = "",
+                matchingOps = "",
+                useful = "";
+            const entropy = `<span data-toggle="tooltip" data-container="body" title="Shannon Entropy is measured from 0 to 8. High entropy suggests encrypted or compressed data. Normal text is usually around 3.5 to 5.">Entropy: <span style="color: ${chooseColour(option.entropy)}">${option.entropy.toFixed(2)}</span></span>`,
+                validUTF8 = option.isUTF8 ? "<span data-toggle='tooltip' data-container='body' title='The data could be a valid UTF8 string based on its encoding.'>Valid UTF8</span>\n" : "";
+
+            if (option.languageScores[0].probability > 0) {
+                let likelyLangs = option.languageScores.filter(l => l.probability > 0);
+                if (likelyLangs.length < 1) likelyLangs = [option.languageScores[0]];
+                language = "<span data-toggle='tooltip' data-container='body' title='Based on a statistical comparison of the frequency of bytes in various languages. Ordered by likelihood.'>" +
+                    "Possible languages:\n    " +
+                    likelyLangs.map(lang => {
+                        return MagicLib.codeToLanguage(lang.lang);
+                    }).join("\n    ") +
+                    "</span>\n";
+            }
+
+            if (option.fileType) {
+                fileType = `<span data-toggle="tooltip" data-container="body" title="Based on the presence of magic bytes.">File type: ${option.fileType.mime} (${option.fileType.ext})</span>\n`;
+            }
+
+            if (option.matchingOps.length) {
+                matchingOps = `Matching ops: ${[...new Set(option.matchingOps.map(op => op.op))].join(", ")}\n`;
+            }
+
+            if (option.useful) {
+                useful = "<span data-toggle='tooltip' data-container='body' title='This could be an operation that displays data in a useful way, such as rendering an image.'>Useful op detected</span>\n";
+            }
+
+            output += `<tr>
+                <td><a href="#${recipeURL}">${Utils.generatePrettyRecipe(option.recipe, true)}</a></td>
+                <td>${Utils.escapeHtml(Utils.printable(Utils.truncate(option.data, 99)))}</td>
+                <td>${language}${fileType}${matchingOps}${useful}${validUTF8}${entropy}</td>
+            </tr>`;
+        });
+
+        output += "</table><script type='application/javascript'>$('[data-toggle=\"tooltip\"]').tooltip()</script>";
+
+        if (!options.length) {
+            output = "Nothing of interest could be detected about the input data.\nHave you tried modifying the operation arguments?";
+        }
+        dish.set(output, Dish.HTML);
+        return state;
+    }
+
+}
+
+export default Magic;

src/core/operations/ScanForEmbeddedFiles.mjs

@@ -0,0 +1,85 @@
+/**
+ * @author n1474335 [n1474335@gmail.com]
+ * @copyright Crown Copyright 2016
+ * @license Apache-2.0
+ */
+
+import Operation from "../Operation";
+import Utils from "../Utils";
+import Magic from "../lib/Magic";
+
+/**
+ * Scan for Embedded Files operation
+ */
+class ScanForEmbeddedFiles extends Operation {
+
+    /**
+     * ScanForEmbeddedFiles constructor
+     */
+    constructor() {
+        super();
+
+        this.name = "Scan for Embedded Files";
+        this.module = "Default";
+        this.description = "Scans the data for potential embedded files by looking for magic bytes at all offsets. This operation is prone to false positives.<br><br>WARNING: Files over about 100KB in size will take a VERY long time to process.";
+        this.inputType = "ArrayBuffer";
+        this.outputType = "string";
+        this.args = [
+            {
+                "name": "Ignore common byte sequences",
+                "type": "boolean",
+                "value": true
+            }
+        ];
+    }
+
+    /**
+     * @param {ArrayBuffer} input
+     * @param {Object[]} args
+     * @returns {string}
+     */
+    run(input, args) {
+        let output = "Scanning data for 'magic bytes' which may indicate embedded files. The following results may be false positives and should not be treat as reliable. Any suffiently long file is likely to contain these magic bytes coincidentally.\n",
+            type,
+            numFound = 0,
+            numCommonFound = 0;
+        const ignoreCommon = args[0],
+            commonExts = ["ico", "ttf", ""],
+            data = new Uint8Array(input);
+
+        for (let i = 0; i < data.length; i++) {
+            type = Magic.magicFileType(data.slice(i));
+            if (type) {
+                if (ignoreCommon && commonExts.indexOf(type.ext) > -1) {
+                    numCommonFound++;
+                    continue;
+                }
+                numFound++;
+                output += "\nOffset " + i + " (0x" + Utils.hex(i) + "):\n" +
+                    "  File extension: " + type.ext + "\n" +
+                    "  MIME type:      " + type.mime + "\n";
+
+                if (type.desc && type.desc.length) {
+                    output += "  Description:    " + type.desc + "\n";
+                }
+            }
+        }
+
+        if (numFound === 0) {
+            output += "\nNo embedded files were found.";
+        }
+
+        if (numCommonFound > 0) {
+            output += "\n\n" + numCommonFound;
+            output += numCommonFound === 1 ?
+                " file type was detected that has a common byte sequence. This is likely to be a false positive." :
+                " file types were detected that have common byte sequences. These are likely to be false positives.";
+            output += " Run this operation with the 'Ignore common byte sequences' option unchecked to see details.";
+        }
+
+        return output;
+    }
+
+}
+
+export default ScanForEmbeddedFiles;

src/core/operations/URLDecode.mjs

@@ -23,6 +23,13 @@ class URLDecode extends Operation {
         this.inputType = "string";
         this.outputType = "string";
         this.args = [];
+        this.patterns = [
+            {
+                match: ".*(?:%[\\da-f]{2}.*){4}",
+                flags: "i",
+                args: []
+            },
+        ];
     }
 
     /**

src/core/operations/Unzip.mjs

@@ -39,6 +39,13 @@ class Unzip extends Operation {
                 value: false
             }
         ];
+        this.patterns = [
+            {
+                match: "^\\x50\\x4b(?:\\x03|\\x05|\\x07)(?:\\x04|\\x06|\\x08)",
+                flags: "",
+                args: ["", false]
+            },
+        ];
     }
 
     /**

src/core/operations/ZlibInflate.mjs

@@ -58,6 +58,13 @@ class ZlibInflate extends Operation {
                 value: false
             }
         ];
+        this.patterns = [
+            {
+                match: "^\\x78(\\x01|\\x9c|\\xda|\\x5e)",
+                flags: "",
+                args: [0, 0, "Adaptive", false, false]
+            },
+        ];
     }
 
     /**

src/core/operations/legacy/FileType.js

@@ -225,8 +225,8 @@ const FileType = {
 
         if (buf[0] === 0x78 && buf[1] === 0x01) {
             return {
-                ext: "dmg",
-                mime: "application/x-apple-diskimage"
+                ext: "dmg, zlib",
+                mime: "application/x-apple-diskimage, application/x-deflate"
             };
         }
 
@@ -434,8 +434,11 @@ const FileType = {
             };
         }
 
-        // Added by n1474335 [n1474335@gmail.com] from here on
-        // ################################################################## //
+        /**
+         *
+         * Added by n1474335 [n1474335@gmail.com] from here on
+         *
+         */
         if ((buf[0] === 0x1F && buf[1] === 0x9D) || (buf[0] === 0x1F && buf[1] === 0xA0)) {
             return {
                 ext: "z, tar.z",
@@ -469,16 +472,16 @@ const FileType = {
         // Must be before Little-endian UTF-16 BOM
         if (buf[0] === 0xFF && buf[1] === 0xFE && buf[2] === 0x00 && buf[3] === 0x00) {
             return {
-                ext: "",
-                mime: "",
+                ext: "UTF32LE",
+                mime: "charset/utf32le",
                 desc: "Little-endian UTF-32 encoded Unicode byte order mark detected."
             };
         }
 
         if (buf[0] === 0xFF && buf[1] === 0xFE) {
             return {
-                ext: "",
-                mime: "",
+                ext: "UTF16LE",
+                mime: "charset/utf16le",
                 desc: "Little-endian UTF-16 encoded Unicode byte order mark detected."
             };
         }
@@ -524,6 +527,13 @@ const FileType = {
             };
         }
 
+        if (buf[0] === 0x78 && (buf[1] === 0x01 || buf[1] === 0x9C || buf[1] === 0xDA || buf[1] === 0x5e)) {
+            return {
+                ext: "zlib",
+                mime: "application/x-deflate"
+            };
+        }
+
         return null;
     },
 

src/core/operations/legacy/FlowControl.js

@@ -1,5 +1,7 @@
-import Recipe from "./Recipe";
-import Dish from "./Dish";
+import Recipe from "./Recipe.js";
+import Dish from "./Dish.js";
+import Magic from "./lib/Magic.js";
+import Utils from "./Utils.js";
 
 
 /**
@@ -183,7 +185,7 @@ const FlowControl = {
      * @returns {Object} The updated state of the recipe.
      */
     runJump: function(state) {
-        const ings     = state.opList[state.progress].ingValues,
+        const ings = state.opList[state.progress].ingValues,
             label = ings[0],
             maxJumps = ings[1],
             jmpIndex = FlowControl._getLabelIndex(label, state);
@@ -211,7 +213,7 @@ const FlowControl = {
      * @returns {Object} The updated state of the recipe.
      */
     runCondJump: async function(state) {
-        const ings     = state.opList[state.progress].ingValues,
+        const ings   = state.opList[state.progress].ingValues,
             dish     = state.dish,
             regexStr = ings[0],
             invert   = ings[1],
@@ -266,6 +268,99 @@ const FlowControl = {
     },
 
 
+    /**
+     * Magic operation.
+     *
+     * @param {Object} state - The current state of the recipe.
+     * @param {number} state.progress - The current position in the recipe.
+     * @param {Dish} state.dish - The Dish being operated on.
+     * @param {Operation[]} state.opList - The list of operations in the recipe.
+     * @returns {Object} The updated state of the recipe.
+     */
+    runMagic: async function(state) {
+        const ings = state.opList[state.progress].ingValues,
+            depth = ings[0],
+            intensive = ings[1],
+            extLang = ings[2],
+            dish = state.dish,
+            currentRecipeConfig = state.opList.map(op => op.getConfig()),
+            magic = new Magic(dish.get(Dish.ARRAY_BUFFER)),
+            options = await magic.speculativeExecution(depth, extLang, intensive);
+
+        let output = `<table
+                class='table table-hover table-condensed table-bordered'
+                style='table-layout: fixed;'>
+            <tr>
+                <th>Recipe (click to load)</th>
+                <th>Result snippet</th>
+                <th>Properties</th>
+            </tr>`;
+
+        /**
+         * Returns a CSS colour value based on an integer input.
+         *
+         * @param {number} val
+         * @returns {string}
+         */
+        function chooseColour(val) {
+            if (val < 3) return "green";
+            if (val < 5) return "goldenrod";
+            return "red";
+        }
+
+        options.forEach(option => {
+            // Construct recipe URL
+            // Replace this Magic op with the generated recipe
+            const recipeConfig = currentRecipeConfig.slice(0, state.progress)
+                    .concat(option.recipe)
+                    .concat(currentRecipeConfig.slice(state.progress + 1)),
+                recipeURL = "recipe=" + Utils.encodeURIFragment(Utils.generatePrettyRecipe(recipeConfig));
+
+            let language = "",
+                fileType = "",
+                matchingOps = "",
+                useful = "",
+                entropy = `<span data-toggle="tooltip" data-container="body" title="Shannon Entropy is measured from 0 to 8. High entropy suggests encrypted or compressed data. Normal text is usually around 3.5 to 5.">Entropy: <span style="color: ${chooseColour(option.entropy)}">${option.entropy.toFixed(2)}</span></span>`,
+                validUTF8 = option.isUTF8 ? "<span data-toggle='tooltip' data-container='body' title='The data could be a valid UTF8 string based on its encoding.'>Valid UTF8</span>\n" : "";
+
+            if (option.languageScores[0].probability > 0) {
+                let likelyLangs = option.languageScores.filter(l => l.probability > 0);
+                if (likelyLangs.length < 1) likelyLangs = [option.languageScores[0]];
+                language = "<span data-toggle='tooltip' data-container='body' title='Based on a statistical comparison of the frequency of bytes in various languages. Ordered by likelihood.'>" +
+                    "Possible languages:\n    " + likelyLangs.map(lang => {
+                        return Magic.codeToLanguage(lang.lang);
+                    }).join("\n    ") + "</span>\n";
+            }
+
+            if (option.fileType) {
+                fileType = `<span data-toggle="tooltip" data-container="body" title="Based on the presence of magic bytes.">File type: ${option.fileType.mime} (${option.fileType.ext})</span>\n`;
+            }
+
+            if (option.matchingOps.length) {
+                matchingOps = `Matching ops: ${[...new Set(option.matchingOps.map(op => op.op))].join(", ")}\n`;
+            }
+
+            if (option.useful) {
+                useful = "<span data-toggle='tooltip' data-container='body' title='This could be an operation that displays data in a useful way, such as rendering an image.'>Useful op detected</span>\n";
+            }
+
+            output += `<tr>
+                <td><a href="#${recipeURL}">${Utils.generatePrettyRecipe(option.recipe, true)}</a></td>
+                <td>${Utils.escapeHtml(Utils.printable(Utils.truncate(option.data, 99)))}</td>
+                <td>${language}${fileType}${matchingOps}${useful}${validUTF8}${entropy}</td>
+            </tr>`;
+        });
+
+        output += "</table><script type='application/javascript'>$('[data-toggle=\"tooltip\"]').tooltip()</script>";
+
+        if (!options.length) {
+            output = "Nothing of interest could be detected about the input data.\nHave you tried modifying the operation arguments?";
+        }
+        dish.set(output, Dish.HTML);
+        return state;
+    },
+
+
     /**
      * Returns the index of a label.
      *

src/core/operations/legacy/PublicKey.js

@@ -62,7 +62,7 @@ const PublicKey = {
             pkStr = "",
             sig = cert.getSignatureValueHex(),
             sigStr = "",
-            extensions = cert.getInfo().split("X509v3 Extensions:\n")[1].split("signature")[0];
+            extensions = "";
 
         // Public Key fields
         pkFields.push({
@@ -144,6 +144,10 @@ const PublicKey = {
             sigStr = "  Signature:      " + PublicKey._formatByteStr(sig, 16, 18);
         }
 
+        // Extensions
+        try {
+            extensions = cert.getInfo().split("X509v3 Extensions:\n")[1].split("signature")[0];
+        } catch (err) {}
 
         let issuerStr = PublicKey._formatDnStr(issuer, 2),
             nbDate = PublicKey._formatDate(cert.getNotBefore()),

src/web/index.js

@@ -35,7 +35,8 @@ function main() {
         "URL Decode",
         "Regular expression",
         "Entropy",
-        "Fork"
+        "Fork",
+        "Magic"
     ];
 
     const defaultOptions = {