浏览代码

Added 'Extract Files' operation and 'Forensics' category.

n1474335 6 年之前
父节点
当前提交
6aa9d2b492
共有 5 个文件被更改,包括 572 次插入74 次删除
  1. 73 69
      package-lock.json
  2. 13 5
      src/core/config/Categories.json
  3. 231 0
      src/core/lib/FileExtraction.mjs
  4. 164 0
      src/core/lib/Stream.mjs
  5. 91 0
      src/core/operations/ExtractFiles.mjs

+ 73 - 69
package-lock.json

@@ -1171,7 +1171,7 @@
     },
     "ansi-escapes": {
       "version": "3.1.0",
-      "resolved": "http://registry.npmjs.org/ansi-escapes/-/ansi-escapes-3.1.0.tgz",
+      "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-3.1.0.tgz",
       "integrity": "sha512-UgAb8H9D41AQnu/PbWlCofQVcnV4Gs2bBJi9eZPxfU/hgglFh3SMDMENRIqdr7H6XFnXdoknctFByVsCOotTVw==",
       "dev": true
     },
@@ -1284,7 +1284,7 @@
     },
     "array-equal": {
       "version": "1.0.0",
-      "resolved": "http://registry.npmjs.org/array-equal/-/array-equal-1.0.0.tgz",
+      "resolved": "https://registry.npmjs.org/array-equal/-/array-equal-1.0.0.tgz",
       "integrity": "sha1-jCpe8kcv2ep0KwTHenUJO6J1fJM=",
       "dev": true
     },
@@ -1369,7 +1369,7 @@
         },
         "util": {
           "version": "0.10.3",
-          "resolved": "http://registry.npmjs.org/util/-/util-0.10.3.tgz",
+          "resolved": "https://registry.npmjs.org/util/-/util-0.10.3.tgz",
           "integrity": "sha1-evsa/lCAUkZInj23/g7TeTNqwPk=",
           "dev": true,
           "requires": {
@@ -1457,7 +1457,7 @@
     },
     "axios": {
       "version": "0.18.0",
-      "resolved": "http://registry.npmjs.org/axios/-/axios-0.18.0.tgz",
+      "resolved": "https://registry.npmjs.org/axios/-/axios-0.18.0.tgz",
       "integrity": "sha1-MtU+SFHv3AoRmTts0AB4nXDAUQI=",
       "dev": true,
       "requires": {
@@ -1863,7 +1863,7 @@
     },
     "browserify-aes": {
       "version": "1.2.0",
-      "resolved": "http://registry.npmjs.org/browserify-aes/-/browserify-aes-1.2.0.tgz",
+      "resolved": "https://registry.npmjs.org/browserify-aes/-/browserify-aes-1.2.0.tgz",
       "integrity": "sha512-+7CHXqGuspUn/Sl5aO7Ea0xWGAtETPXNSAjHo48JfLdPWcMng33Xe4znFvQweqc/uzk5zSOI3H52CYnjCfb5hA==",
       "dev": true,
       "requires": {
@@ -1900,7 +1900,7 @@
     },
     "browserify-rsa": {
       "version": "4.0.1",
-      "resolved": "http://registry.npmjs.org/browserify-rsa/-/browserify-rsa-4.0.1.tgz",
+      "resolved": "https://registry.npmjs.org/browserify-rsa/-/browserify-rsa-4.0.1.tgz",
       "integrity": "sha1-IeCr+vbyApzy+vsTNWenAdQTVSQ=",
       "dev": true,
       "requires": {
@@ -1950,7 +1950,7 @@
     },
     "buffer": {
       "version": "4.9.1",
-      "resolved": "http://registry.npmjs.org/buffer/-/buffer-4.9.1.tgz",
+      "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.1.tgz",
       "integrity": "sha1-bRu2AbB6TvztlwlBMgkwJ8lbwpg=",
       "dev": true,
       "requires": {
@@ -2015,7 +2015,7 @@
     },
     "cacache": {
       "version": "10.0.4",
-      "resolved": "http://registry.npmjs.org/cacache/-/cacache-10.0.4.tgz",
+      "resolved": "https://registry.npmjs.org/cacache/-/cacache-10.0.4.tgz",
       "integrity": "sha512-Dph0MzuH+rTQzGPNT9fAnrPmMmjKfST6trxJeK7NQuHRaVw24VzPRWTmg9MpcwOVQZO0E1FBICUlFeNaKPIfHA==",
       "dev": true,
       "requires": {
@@ -2092,7 +2092,7 @@
     },
     "camelcase-keys": {
       "version": "2.1.0",
-      "resolved": "http://registry.npmjs.org/camelcase-keys/-/camelcase-keys-2.1.0.tgz",
+      "resolved": "https://registry.npmjs.org/camelcase-keys/-/camelcase-keys-2.1.0.tgz",
       "integrity": "sha1-MIvur/3ygRkFHvodkyITyRuPkuc=",
       "dev": true,
       "requires": {
@@ -2123,7 +2123,7 @@
     },
     "chalk": {
       "version": "1.1.3",
-      "resolved": "http://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz",
+      "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz",
       "integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=",
       "requires": {
         "ansi-styles": "^2.2.1",
@@ -2590,7 +2590,7 @@
     },
     "create-hash": {
       "version": "1.2.0",
-      "resolved": "http://registry.npmjs.org/create-hash/-/create-hash-1.2.0.tgz",
+      "resolved": "https://registry.npmjs.org/create-hash/-/create-hash-1.2.0.tgz",
       "integrity": "sha512-z00bCGNHDG8mHAkP7CtT1qVu+bFQUPjYq/4Iv3C3kWjTFV10zIjfSoeqXo9Asws8gwSHDGj/hl2u4OGIjapeCg==",
       "dev": true,
       "requires": {
@@ -2603,7 +2603,7 @@
     },
     "create-hmac": {
       "version": "1.1.7",
-      "resolved": "http://registry.npmjs.org/create-hmac/-/create-hmac-1.1.7.tgz",
+      "resolved": "https://registry.npmjs.org/create-hmac/-/create-hmac-1.1.7.tgz",
       "integrity": "sha512-MJG9liiZ+ogc4TzUwuvbER1JRdgvUFSB5+VR/g5h82fGaIRWMWddtKBHi7/sVhfjQZ6SehlyhvQYrcYkaUIpLg==",
       "dev": true,
       "requires": {
@@ -2721,7 +2721,7 @@
     },
     "css-select": {
       "version": "1.2.0",
-      "resolved": "http://registry.npmjs.org/css-select/-/css-select-1.2.0.tgz",
+      "resolved": "https://registry.npmjs.org/css-select/-/css-select-1.2.0.tgz",
       "integrity": "sha1-KzoRBTnFNV8c2NMUYj6HCxIeyFg=",
       "dev": true,
       "requires": {
@@ -3055,7 +3055,7 @@
     },
     "diffie-hellman": {
       "version": "5.0.3",
-      "resolved": "http://registry.npmjs.org/diffie-hellman/-/diffie-hellman-5.0.3.tgz",
+      "resolved": "https://registry.npmjs.org/diffie-hellman/-/diffie-hellman-5.0.3.tgz",
       "integrity": "sha512-kqag/Nl+f3GwyK25fhUMYj81BUOrZ9IuJsjIcDE5icNM9FJHAVm3VcUDxdLPoQtTuUylWm6ZIknYJwwaPxsUzg==",
       "dev": true,
       "requires": {
@@ -3119,7 +3119,7 @@
       "dependencies": {
         "domelementtype": {
           "version": "1.1.3",
-          "resolved": "http://registry.npmjs.org/domelementtype/-/domelementtype-1.1.3.tgz",
+          "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-1.1.3.tgz",
           "integrity": "sha1-vSh3PiZCiBrsUVRJJCmcXNgiGFs=",
           "dev": true
         },
@@ -3307,7 +3307,7 @@
     },
     "entities": {
       "version": "1.0.0",
-      "resolved": "http://registry.npmjs.org/entities/-/entities-1.0.0.tgz",
+      "resolved": "https://registry.npmjs.org/entities/-/entities-1.0.0.tgz",
       "integrity": "sha1-sph6o4ITR/zeZCsk/fyeT7cSvyY=",
       "dev": true
     },
@@ -3731,7 +3731,7 @@
     },
     "eventemitter2": {
       "version": "0.4.14",
-      "resolved": "http://registry.npmjs.org/eventemitter2/-/eventemitter2-0.4.14.tgz",
+      "resolved": "https://registry.npmjs.org/eventemitter2/-/eventemitter2-0.4.14.tgz",
       "integrity": "sha1-j2G3XN4BKy6esoTUVFWDtWQ7Yas=",
       "dev": true
     },
@@ -3743,7 +3743,7 @@
     },
     "events": {
       "version": "1.1.1",
-      "resolved": "http://registry.npmjs.org/events/-/events-1.1.1.tgz",
+      "resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz",
       "integrity": "sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ=",
       "dev": true
     },
@@ -4149,7 +4149,7 @@
     },
     "finalhandler": {
       "version": "1.1.1",
-      "resolved": "http://registry.npmjs.org/finalhandler/-/finalhandler-1.1.1.tgz",
+      "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.1.1.tgz",
       "integrity": "sha512-Y1GUDo39ez4aHAw7MysnUD5JzYX+WaIj8I57kO3aEPT1fFRL4sr7mjei97FgnwhAyyzRYmQZaTHb2+9uZ1dPtg==",
       "dev": true,
       "requires": {
@@ -4377,7 +4377,7 @@
     },
     "fs-extra": {
       "version": "1.0.0",
-      "resolved": "http://registry.npmjs.org/fs-extra/-/fs-extra-1.0.0.tgz",
+      "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-1.0.0.tgz",
       "integrity": "sha1-zTzl9+fLYUWIP8rjGR6Yd/hYeVA=",
       "dev": true,
       "requires": {
@@ -4445,12 +4445,14 @@
         "balanced-match": {
           "version": "1.0.0",
           "bundled": true,
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "brace-expansion": {
           "version": "1.1.11",
           "bundled": true,
           "dev": true,
+          "optional": true,
           "requires": {
             "balanced-match": "^1.0.0",
             "concat-map": "0.0.1"
@@ -4470,7 +4472,8 @@
         "concat-map": {
           "version": "0.0.1",
           "bundled": true,
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "console-control-strings": {
           "version": "1.1.0",
@@ -4618,6 +4621,7 @@
           "version": "3.0.4",
           "bundled": true,
           "dev": true,
+          "optional": true,
           "requires": {
             "brace-expansion": "^1.1.7"
           }
@@ -5023,7 +5027,7 @@
     },
     "get-stream": {
       "version": "3.0.0",
-      "resolved": "http://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz",
+      "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-3.0.0.tgz",
       "integrity": "sha1-jpQ9E1jcN1VQVOy+LtsFqhdO3hQ=",
       "dev": true
     },
@@ -5173,7 +5177,7 @@
         },
         "grunt-cli": {
           "version": "1.2.0",
-          "resolved": "http://registry.npmjs.org/grunt-cli/-/grunt-cli-1.2.0.tgz",
+          "resolved": "https://registry.npmjs.org/grunt-cli/-/grunt-cli-1.2.0.tgz",
           "integrity": "sha1-VisRnrsGndtGSs4oRVAb6Xs1tqg=",
           "dev": true,
           "requires": {
@@ -5221,7 +5225,7 @@
       "dependencies": {
         "shelljs": {
           "version": "0.5.3",
-          "resolved": "http://registry.npmjs.org/shelljs/-/shelljs-0.5.3.tgz",
+          "resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.5.3.tgz",
           "integrity": "sha1-xUmCuZbHbvDB5rWfvcWCX1txMRM=",
           "dev": true
         }
@@ -5241,7 +5245,7 @@
       "dependencies": {
         "async": {
           "version": "1.5.2",
-          "resolved": "http://registry.npmjs.org/async/-/async-1.5.2.tgz",
+          "resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz",
           "integrity": "sha1-7GphrlZIDAw8skHJVhjiCJL5Zyo=",
           "dev": true
         }
@@ -5269,7 +5273,7 @@
     },
     "grunt-contrib-jshint": {
       "version": "1.1.0",
-      "resolved": "http://registry.npmjs.org/grunt-contrib-jshint/-/grunt-contrib-jshint-1.1.0.tgz",
+      "resolved": "https://registry.npmjs.org/grunt-contrib-jshint/-/grunt-contrib-jshint-1.1.0.tgz",
       "integrity": "sha1-Np2QmyWTxA6L55lAshNAhQx5Oaw=",
       "dev": true,
       "requires": {
@@ -5368,7 +5372,7 @@
       "dependencies": {
         "colors": {
           "version": "1.1.2",
-          "resolved": "http://registry.npmjs.org/colors/-/colors-1.1.2.tgz",
+          "resolved": "https://registry.npmjs.org/colors/-/colors-1.1.2.tgz",
           "integrity": "sha1-FopHAXVran9RoSzgyXv6KMCE7WM=",
           "dev": true
         }
@@ -5432,7 +5436,7 @@
       "dependencies": {
         "async": {
           "version": "1.5.2",
-          "resolved": "http://registry.npmjs.org/async/-/async-1.5.2.tgz",
+          "resolved": "https://registry.npmjs.org/async/-/async-1.5.2.tgz",
           "integrity": "sha1-7GphrlZIDAw8skHJVhjiCJL5Zyo=",
           "dev": true
         }
@@ -5450,7 +5454,7 @@
     },
     "handle-thing": {
       "version": "1.2.5",
-      "resolved": "http://registry.npmjs.org/handle-thing/-/handle-thing-1.2.5.tgz",
+      "resolved": "https://registry.npmjs.org/handle-thing/-/handle-thing-1.2.5.tgz",
       "integrity": "sha1-/Xqtcmvxpf0W38KbL3pmAdJxOcQ=",
       "dev": true
     },
@@ -5677,7 +5681,7 @@
     },
     "html-webpack-plugin": {
       "version": "3.2.0",
-      "resolved": "http://registry.npmjs.org/html-webpack-plugin/-/html-webpack-plugin-3.2.0.tgz",
+      "resolved": "https://registry.npmjs.org/html-webpack-plugin/-/html-webpack-plugin-3.2.0.tgz",
       "integrity": "sha1-sBq71yOsqqeze2r0SS69oD2d03s=",
       "dev": true,
       "requires": {
@@ -5725,7 +5729,7 @@
     },
     "htmlparser2": {
       "version": "3.8.3",
-      "resolved": "http://registry.npmjs.org/htmlparser2/-/htmlparser2-3.8.3.tgz",
+      "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-3.8.3.tgz",
       "integrity": "sha1-mWwosZFRaovoZQGn15dX5ccMEGg=",
       "dev": true,
       "requires": {
@@ -5744,7 +5748,7 @@
     },
     "http-errors": {
       "version": "1.6.3",
-      "resolved": "http://registry.npmjs.org/http-errors/-/http-errors-1.6.3.tgz",
+      "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.6.3.tgz",
       "integrity": "sha1-i1VoC7S+KDoLW/TqLjhYC+HZMg0=",
       "dev": true,
       "requires": {
@@ -5773,7 +5777,7 @@
     },
     "http-proxy-middleware": {
       "version": "0.18.0",
-      "resolved": "http://registry.npmjs.org/http-proxy-middleware/-/http-proxy-middleware-0.18.0.tgz",
+      "resolved": "https://registry.npmjs.org/http-proxy-middleware/-/http-proxy-middleware-0.18.0.tgz",
       "integrity": "sha512-Fs25KVMPAIIcgjMZkVHJoKg9VcXcC1C8yb9JUgeDvVXY0S/zgVIhMb+qVswDIgtJe2DfckMSY2d6TuTEutlk6Q==",
       "dev": true,
       "requires": {
@@ -6225,7 +6229,7 @@
     },
     "is-builtin-module": {
       "version": "1.0.0",
-      "resolved": "http://registry.npmjs.org/is-builtin-module/-/is-builtin-module-1.0.0.tgz",
+      "resolved": "https://registry.npmjs.org/is-builtin-module/-/is-builtin-module-1.0.0.tgz",
       "integrity": "sha1-VAVy0096wxGfj3bDDLwbHgN6/74=",
       "dev": true,
       "requires": {
@@ -6750,7 +6754,7 @@
     },
     "jsonfile": {
       "version": "2.4.0",
-      "resolved": "http://registry.npmjs.org/jsonfile/-/jsonfile-2.4.0.tgz",
+      "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-2.4.0.tgz",
       "integrity": "sha1-NzaitCi4e72gzIO1P6PWM6NcKug=",
       "dev": true,
       "requires": {
@@ -6856,7 +6860,7 @@
     },
     "kew": {
       "version": "0.7.0",
-      "resolved": "http://registry.npmjs.org/kew/-/kew-0.7.0.tgz",
+      "resolved": "https://registry.npmjs.org/kew/-/kew-0.7.0.tgz",
       "integrity": "sha1-edk9LTM2PW/dKXCzNdkUGtWR15s=",
       "dev": true
     },
@@ -6948,7 +6952,7 @@
     },
     "load-json-file": {
       "version": "1.1.0",
-      "resolved": "http://registry.npmjs.org/load-json-file/-/load-json-file-1.1.0.tgz",
+      "resolved": "https://registry.npmjs.org/load-json-file/-/load-json-file-1.1.0.tgz",
       "integrity": "sha1-lWkFcI1YtLq0wiYbBPWfMcmTdMA=",
       "dev": true,
       "requires": {
@@ -6961,7 +6965,7 @@
       "dependencies": {
         "pify": {
           "version": "2.3.0",
-          "resolved": "http://registry.npmjs.org/pify/-/pify-2.3.0.tgz",
+          "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz",
           "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=",
           "dev": true
         }
@@ -7196,7 +7200,7 @@
     },
     "media-typer": {
       "version": "0.3.0",
-      "resolved": "http://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz",
+      "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz",
       "integrity": "sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g=",
       "dev": true
     },
@@ -7255,7 +7259,7 @@
     },
     "meow": {
       "version": "3.7.0",
-      "resolved": "http://registry.npmjs.org/meow/-/meow-3.7.0.tgz",
+      "resolved": "https://registry.npmjs.org/meow/-/meow-3.7.0.tgz",
       "integrity": "sha1-cstmi0JSKCkKu/qFaJJYcwioAfs=",
       "dev": true,
       "requires": {
@@ -7432,7 +7436,7 @@
     },
     "mkdirp": {
       "version": "0.5.1",
-      "resolved": "http://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz",
+      "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz",
       "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=",
       "dev": true,
       "requires": {
@@ -7554,7 +7558,7 @@
     },
     "ncp": {
       "version": "1.0.1",
-      "resolved": "http://registry.npmjs.org/ncp/-/ncp-1.0.1.tgz",
+      "resolved": "https://registry.npmjs.org/ncp/-/ncp-1.0.1.tgz",
       "integrity": "sha1-0VNn5cuHQyuhF9K/gP30Wuz7QkY=",
       "dev": true
     },
@@ -7617,7 +7621,7 @@
       "dependencies": {
         "semver": {
           "version": "5.3.0",
-          "resolved": "http://registry.npmjs.org/semver/-/semver-5.3.0.tgz",
+          "resolved": "https://registry.npmjs.org/semver/-/semver-5.3.0.tgz",
           "integrity": "sha1-myzl094C0XxgEq0yaqa00M9U+U8=",
           "dev": true
         }
@@ -7756,7 +7760,7 @@
       "dependencies": {
         "colors": {
           "version": "0.5.1",
-          "resolved": "http://registry.npmjs.org/colors/-/colors-0.5.1.tgz",
+          "resolved": "https://registry.npmjs.org/colors/-/colors-0.5.1.tgz",
           "integrity": "sha1-fQAj6usVTo7p/Oddy5I9DtFmd3Q="
         },
         "underscore": {
@@ -8015,13 +8019,13 @@
     },
     "os-homedir": {
       "version": "1.0.2",
-      "resolved": "http://registry.npmjs.org/os-homedir/-/os-homedir-1.0.2.tgz",
+      "resolved": "https://registry.npmjs.org/os-homedir/-/os-homedir-1.0.2.tgz",
       "integrity": "sha1-/7xJiDNuDoM94MFox+8VISGqf7M=",
       "dev": true
     },
     "os-locale": {
       "version": "1.4.0",
-      "resolved": "http://registry.npmjs.org/os-locale/-/os-locale-1.4.0.tgz",
+      "resolved": "https://registry.npmjs.org/os-locale/-/os-locale-1.4.0.tgz",
       "integrity": "sha1-IPnxeuKe00XoveWDsT0gCYA8FNk=",
       "dev": true,
       "requires": {
@@ -8030,7 +8034,7 @@
     },
     "os-tmpdir": {
       "version": "1.0.2",
-      "resolved": "http://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz",
+      "resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz",
       "integrity": "sha1-u+Z0BseaqFxc/sdm/lc0VV36EnQ=",
       "dev": true
     },
@@ -8173,7 +8177,7 @@
     },
     "parse-asn1": {
       "version": "5.1.1",
-      "resolved": "http://registry.npmjs.org/parse-asn1/-/parse-asn1-5.1.1.tgz",
+      "resolved": "https://registry.npmjs.org/parse-asn1/-/parse-asn1-5.1.1.tgz",
       "integrity": "sha512-KPx7flKXg775zZpnp9SxJlz00gTd4BmJ2yJufSc44gMCRrRQ7NSzAcSJQfifuOLgW6bEi+ftrALtsgALeB2Adw==",
       "dev": true,
       "requires": {
@@ -8231,7 +8235,7 @@
     },
     "path-is-absolute": {
       "version": "1.0.1",
-      "resolved": "http://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz",
+      "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz",
       "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=",
       "dev": true
     },
@@ -8272,7 +8276,7 @@
       "dependencies": {
         "pify": {
           "version": "2.3.0",
-          "resolved": "http://registry.npmjs.org/pify/-/pify-2.3.0.tgz",
+          "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz",
           "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=",
           "dev": true
         }
@@ -8839,7 +8843,7 @@
     },
     "progress": {
       "version": "1.1.8",
-      "resolved": "http://registry.npmjs.org/progress/-/progress-1.1.8.tgz",
+      "resolved": "https://registry.npmjs.org/progress/-/progress-1.1.8.tgz",
       "integrity": "sha1-4mDHj2Fhzdmw5WzD4Khd4Xx6V74="
     },
     "promise-inflight": {
@@ -8864,13 +8868,13 @@
       "dependencies": {
         "async": {
           "version": "1.0.0",
-          "resolved": "http://registry.npmjs.org/async/-/async-1.0.0.tgz",
+          "resolved": "https://registry.npmjs.org/async/-/async-1.0.0.tgz",
           "integrity": "sha1-+PwEyjoTeErenhZBr5hXjPvWR6k=",
           "dev": true
         },
         "winston": {
           "version": "2.1.1",
-          "resolved": "http://registry.npmjs.org/winston/-/winston-2.1.1.tgz",
+          "resolved": "https://registry.npmjs.org/winston/-/winston-2.1.1.tgz",
           "integrity": "sha1-PJNJ0ZYgf9G9/51LxD73JRDjoS4=",
           "dev": true,
           "requires": {
@@ -8885,7 +8889,7 @@
           "dependencies": {
             "colors": {
               "version": "1.0.3",
-              "resolved": "http://registry.npmjs.org/colors/-/colors-1.0.3.tgz",
+              "resolved": "https://registry.npmjs.org/colors/-/colors-1.0.3.tgz",
               "integrity": "sha1-BDP0TYCWgP3rYO0mDxsMJi6CpAs=",
               "dev": true
             },
@@ -9064,7 +9068,7 @@
       "dependencies": {
         "pify": {
           "version": "2.3.0",
-          "resolved": "http://registry.npmjs.org/pify/-/pify-2.3.0.tgz",
+          "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz",
           "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=",
           "dev": true
         }
@@ -9426,7 +9430,7 @@
     },
     "require-uncached": {
       "version": "1.0.3",
-      "resolved": "http://registry.npmjs.org/require-uncached/-/require-uncached-1.0.3.tgz",
+      "resolved": "https://registry.npmjs.org/require-uncached/-/require-uncached-1.0.3.tgz",
       "integrity": "sha1-Tg1W1slmL9MeQwEcS5WqSZVUIdM=",
       "dev": true,
       "requires": {
@@ -9593,7 +9597,7 @@
     },
     "safe-regex": {
       "version": "1.1.0",
-      "resolved": "http://registry.npmjs.org/safe-regex/-/safe-regex-1.1.0.tgz",
+      "resolved": "https://registry.npmjs.org/safe-regex/-/safe-regex-1.1.0.tgz",
       "integrity": "sha1-QKNmnzsHfR6UPURinhV91IAjvy4=",
       "dev": true,
       "requires": {
@@ -9914,7 +9918,7 @@
     },
     "sha.js": {
       "version": "2.4.11",
-      "resolved": "http://registry.npmjs.org/sha.js/-/sha.js-2.4.11.tgz",
+      "resolved": "https://registry.npmjs.org/sha.js/-/sha.js-2.4.11.tgz",
       "integrity": "sha512-QMEp5B7cftE7APOjk5Y6xgrbWu+WkLVQwk8JNjZ8nKRciZaByEW6MubieAiToS7+dwvrjGhH8jRXz3MVd0AYqQ==",
       "dev": true,
       "requires": {
@@ -9958,7 +9962,7 @@
     },
     "shelljs": {
       "version": "0.3.0",
-      "resolved": "http://registry.npmjs.org/shelljs/-/shelljs-0.3.0.tgz",
+      "resolved": "https://registry.npmjs.org/shelljs/-/shelljs-0.3.0.tgz",
       "integrity": "sha1-NZbmMHp4FUT1kfN9phg2DzHbV7E=",
       "dev": true
     },
@@ -10610,7 +10614,7 @@
     },
     "strip-ansi": {
       "version": "3.0.1",
-      "resolved": "http://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz",
+      "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz",
       "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=",
       "requires": {
         "ansi-regex": "^2.0.0"
@@ -10627,7 +10631,7 @@
     },
     "strip-eof": {
       "version": "1.0.0",
-      "resolved": "http://registry.npmjs.org/strip-eof/-/strip-eof-1.0.0.tgz",
+      "resolved": "https://registry.npmjs.org/strip-eof/-/strip-eof-1.0.0.tgz",
       "integrity": "sha1-u0P/VZim6wXYm1n80SnJgzE2Br8=",
       "dev": true
     },
@@ -10706,7 +10710,7 @@
     },
     "tar": {
       "version": "2.2.1",
-      "resolved": "http://registry.npmjs.org/tar/-/tar-2.2.1.tgz",
+      "resolved": "https://registry.npmjs.org/tar/-/tar-2.2.1.tgz",
       "integrity": "sha1-jk0qJWwOIYXGsYrWlK7JaLg8sdE=",
       "dev": true,
       "requires": {
@@ -10734,7 +10738,7 @@
     },
     "through": {
       "version": "2.3.8",
-      "resolved": "http://registry.npmjs.org/through/-/through-2.3.8.tgz",
+      "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz",
       "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=",
       "dev": true
     },
@@ -11381,7 +11385,7 @@
       "dependencies": {
         "async": {
           "version": "0.9.2",
-          "resolved": "http://registry.npmjs.org/async/-/async-0.9.2.tgz",
+          "resolved": "https://registry.npmjs.org/async/-/async-0.9.2.tgz",
           "integrity": "sha1-rqdNXmHB+JlhO/ZL2mbUx48v0X0=",
           "dev": true
         },
@@ -11407,7 +11411,7 @@
     },
     "valid-data-url": {
       "version": "0.1.6",
-      "resolved": "http://registry.npmjs.org/valid-data-url/-/valid-data-url-0.1.6.tgz",
+      "resolved": "https://registry.npmjs.org/valid-data-url/-/valid-data-url-0.1.6.tgz",
       "integrity": "sha512-FXg2qXMzfAhZc0y2HzELNfUeiOjPr+52hU1DNBWiJJ2luXD+dD1R9NA48Ug5aj0ibbxroeGDc/RJv6ThiGgkDw==",
       "dev": true
     },
@@ -11423,7 +11427,7 @@
     },
     "validator": {
       "version": "9.4.1",
-      "resolved": "http://registry.npmjs.org/validator/-/validator-9.4.1.tgz",
+      "resolved": "https://registry.npmjs.org/validator/-/validator-9.4.1.tgz",
       "integrity": "sha512-YV5KjzvRmSyJ1ee/Dm5UED0G+1L4GZnLN3w6/T+zZm8scVua4sOhYKWTUrKa0H/tMiJyO9QLHMPN+9mB/aMunA==",
       "dev": true
     },
@@ -11847,7 +11851,7 @@
     },
     "webpack-node-externals": {
       "version": "1.7.2",
-      "resolved": "http://registry.npmjs.org/webpack-node-externals/-/webpack-node-externals-1.7.2.tgz",
+      "resolved": "https://registry.npmjs.org/webpack-node-externals/-/webpack-node-externals-1.7.2.tgz",
       "integrity": "sha512-ajerHZ+BJKeCLviLUUmnyd5B4RavLF76uv3cs6KNuO8W+HuQaEs0y0L7o40NQxdPy5w0pcv8Ew7yPUAQG0UdCg==",
       "dev": true
     },
@@ -11984,7 +11988,7 @@
     },
     "wrap-ansi": {
       "version": "2.1.0",
-      "resolved": "http://registry.npmjs.org/wrap-ansi/-/wrap-ansi-2.1.0.tgz",
+      "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-2.1.0.tgz",
       "integrity": "sha1-2Pw9KE3QV5T+hJc8rs3Rz4JP3YU=",
       "dev": true,
       "requires": {

+ 13 - 5
src/core/config/Categories.json

@@ -245,7 +245,8 @@
             "XPath expression",
             "JPath expression",
             "CSS selector",
-            "Extract EXIF"
+            "Extract EXIF",
+            "Extract Files"
         ]
     },
     {
@@ -336,14 +337,23 @@
             "From MessagePack"
         ]
     },
+    {
+        "name": "Forensics",
+        "ops": [
+            "Detect File Type",
+            "Scan for Embedded Files",
+            "Extract Files",
+            "Remove EXIF",
+            "Extract EXIF",
+            "Render Image"
+        ]
+    },
     {
         "name": "Other",
         "ops": [
             "Entropy",
             "Frequency distribution",
             "Chi Square",
-            "Detect File Type",
-            "Scan for Embedded Files",
             "Disassemble x86",
             "Pseudo-Random Number Generator",
             "Generate UUID",
@@ -351,8 +361,6 @@
             "Generate HOTP",
             "Haversine distance",
             "Render Image",
-            "Remove EXIF",
-            "Extract EXIF",
             "Numberwang",
             "XKCD Random Number"
         ]

+ 231 - 0
src/core/lib/FileExtraction.mjs

@@ -0,0 +1,231 @@
+/**
+ * File extraction functions
+ *
+ * @author n1474335 [n1474335@gmail.com]
+ * @copyright Crown Copyright 2018
+ * @license Apache-2.0
+ *
+ */
+import Stream from "./Stream";
+
+/**
+ * Attempts to extract a file from a data stream given its mime type and offset.
+ *
+ * @param {Uint8Array} bytes
+ * @param {Object} fileDetail
+ * @param {string} fileDetail.mime
+ * @param {string} fileDetail.ext
+ * @param {number} fileDetail.offset
+ * @returns {File}
+ */
+export function extractFile(bytes, fileDetail) {
+    let fileData;
+    switch (fileDetail.mime) {
+        case "image/jpeg":
+            fileData = extractJPEG(bytes, fileDetail.offset);
+            break;
+        case "application/x-msdownload":
+            fileData = extractMZPE(bytes, fileDetail.offset);
+            break;
+        case "application/pdf":
+            fileData = extractPDF(bytes, fileDetail.offset);
+            break;
+        case "application/zip":
+            fileData = extractZIP(bytes, fileDetail.offset);
+            break;
+        default:
+            throw new Error(`No extraction algorithm available for "${fileDetail.mime}" files`);
+    }
+
+    return new File([fileData], `extracted_at_0x${fileDetail.offset.toString(16)}.${fileDetail.ext}`);
+}
+
+
+/**
+ * JPEG extractor.
+ *
+ * @param {Uint8Array} bytes
+ * @param {number} offset
+ * @returns {Uint8Array}
+ */
+export function extractJPEG(bytes, offset) {
+    const stream = new Stream(bytes.slice(offset));
+
+    while (stream.hasMore()) {
+        const marker = stream.getBytes(2);
+        if (marker[0] !== 0xff) throw new Error("Invalid JPEG marker: " + marker);
+
+        let segmentSize = 0;
+        switch (marker[1]) {
+            // No length
+            case 0xd8: // Start of Image
+            case 0x01: // For temporary use in arithmetic coding
+                break;
+            case 0xd9: // End found
+                return stream.carve();
+
+            // Variable size segment
+            case 0xc0: // Start of frame (Baseline DCT)
+            case 0xc1: // Start of frame (Extended sequential DCT)
+            case 0xc2: // Start of frame (Progressive DCT)
+            case 0xc3: // Start of frame (Lossless sequential)
+            case 0xc4: // Define Huffman Table
+            case 0xc5: // Start of frame (Differential sequential DCT)
+            case 0xc6: // Start of frame (Differential progressive DCT)
+            case 0xc7: // Start of frame (Differential lossless)
+            case 0xc8: // Reserved for JPEG extensions
+            case 0xc9: // Start of frame (Extended sequential DCT)
+            case 0xca: // Start of frame (Progressive DCT)
+            case 0xcb: // Start of frame (Lossless sequential)
+            case 0xcc: // Define arithmetic conditioning table
+            case 0xcd: // Start of frame (Differential sequential DCT)
+            case 0xce: // Start of frame (Differential progressive DCT)
+            case 0xcf: // Start of frame (Differential lossless)
+            case 0xdb: // Define Quantization Table
+            case 0xde: // Define hierarchical progression
+            case 0xe0: // Application-specific
+            case 0xe1: // Application-specific
+            case 0xe2: // Application-specific
+            case 0xe3: // Application-specific
+            case 0xe4: // Application-specific
+            case 0xe5: // Application-specific
+            case 0xe6: // Application-specific
+            case 0xe7: // Application-specific
+            case 0xe8: // Application-specific
+            case 0xe9: // Application-specific
+            case 0xea: // Application-specific
+            case 0xeb: // Application-specific
+            case 0xec: // Application-specific
+            case 0xed: // Application-specific
+            case 0xee: // Application-specific
+            case 0xef: // Application-specific
+            case 0xfe: // Comment
+                segmentSize = stream.readInt(2, "be");
+                stream.position += segmentSize - 2;
+                break;
+
+            // 1 byte
+            case 0xdf: // Expand reference image
+                stream.position++;
+                break;
+
+            // 2 bytes
+            case 0xdc: // Define number of lines
+            case 0xdd: // Define restart interval
+                stream.position += 2;
+                break;
+
+            // Start scan
+            case 0xda: // Start of scan
+                segmentSize = stream.readInt(2, "be");
+                stream.position += segmentSize - 2;
+                stream.continueUntil(0xff);
+                break;
+
+            // Continue through encoded data
+            case 0x00: // Byte stuffing
+            case 0xd0: // Restart
+            case 0xd1: // Restart
+            case 0xd2: // Restart
+            case 0xd3: // Restart
+            case 0xd4: // Restart
+            case 0xd5: // Restart
+            case 0xd6: // Restart
+            case 0xd7: // Restart
+                stream.continueUntil(0xff);
+                break;
+
+            default:
+                stream.continueUntil(0xff);
+                break;
+        }
+    }
+
+    throw new Error("Unable to parse JPEG successfully");
+}
+
+
+/**
+ * Portable executable extractor.
+ * Assumes that the offset refers to an MZ header.
+ *
+ * @param {Uint8Array} bytes
+ * @param {number} offset
+ * @returns {Uint8Array}
+ */
+export function extractMZPE(bytes, offset) {
+    const stream = new Stream(bytes.slice(offset));
+
+    // Move to PE header pointer
+    stream.moveTo(0x3c);
+    const peAddress = stream.readInt(4, "le");
+
+    // Move to PE header
+    stream.moveTo(peAddress);
+
+    // Get number of sections
+    stream.moveForwardsBy(6);
+    const numSections = stream.readInt(2, "le");
+
+    // Get optional header size
+    stream.moveForwardsBy(12);
+    const optionalHeaderSize = stream.readInt(2, "le");
+
+    // Move past optional header to section header
+    stream.moveForwardsBy(2 + optionalHeaderSize);
+
+    // Move to final section header
+    stream.moveForwardsBy((numSections - 1) * 0x28);
+
+    // Get raw data info
+    stream.moveForwardsBy(16);
+    const rawDataSize = stream.readInt(4, "le");
+    const rawDataAddress = stream.readInt(4, "le");
+
+    // Move to end of final section
+    stream.moveTo(rawDataAddress + rawDataSize);
+
+    return stream.carve();
+}
+
+
+/**
+ * PDF extractor.
+ *
+ * @param {Uint8Array} bytes
+ * @param {number} offset
+ * @returns {Uint8Array}
+ */
+export function extractPDF(bytes, offset) {
+    const stream = new Stream(bytes.slice(offset));
+
+    // Find end-of-file marker (%%EOF)
+    stream.continueUntil([0x25, 0x25, 0x45, 0x4f, 0x46]);
+    stream.moveForwardsBy(5);
+    stream.consumeIf(0x0d);
+    stream.consumeIf(0x0a);
+
+    return stream.carve();
+}
+
+
+/**
+ * ZIP extractor.
+ *
+ * @param {Uint8Array} bytes
+ * @param {number} offset
+ * @returns {Uint8Array}
+ */
+export function extractZIP(bytes, offset) {
+    const stream = new Stream(bytes.slice(offset));
+
+    // Find End of central directory record
+    stream.continueUntil([0x50, 0x4b, 0x05, 0x06]);
+
+    // Get comment length and consume
+    stream.moveForwardsBy(20);
+    const commentLength = stream.readInt(2, "le");
+    stream.moveForwardsBy(commentLength);
+
+    return stream.carve();
+}

+ 164 - 0
src/core/lib/Stream.mjs

@@ -0,0 +1,164 @@
+/**
+ * Stream class for parsing binary protocols.
+ *
+ * @author n1474335 [n1474335@gmail.com]
+ * @author tlwr [toby@toby.codes]
+ * @copyright Crown Copyright 2018
+ * @license Apache-2.0
+ *
+ */
+
+/**
+ * A Stream can be used to traverse a binary blob, interpreting sections of it
+ * as various data types.
+ *
+ * @param {Uint8Array} bytes
+ * @param {Object} fileDetail
+ * @param {string} fileDetail.mime
+ * @param {string} fileDetail.ext
+ * @param {number} fileDetail.offset
+ * @returns {File}
+ */
+export default class Stream {
+
+    /**
+     * Stream constructor.
+     *
+     * @param {Uint8Array} input
+     */
+    constructor(input) {
+        this.bytes = input;
+        this.position = 0;
+    }
+
+    /**
+     * Get a number of bytes from the current position.
+     *
+     * @param {number} numBytes
+     * @returns {Uint8Array}
+     */
+    getBytes(numBytes) {
+        const newPosition = this.position + numBytes;
+        const bytes = this.bytes.slice(this.position, newPosition);
+        this.position = newPosition;
+        return bytes;
+    }
+
+    /**
+     * Interpret the following bytes as a string, stopping at the next null byte or
+     * the supplied limit.
+     *
+     * @param {number} numBytes
+     * @returns {string}
+     */
+    readString(numBytes) {
+        let result = "";
+        for (let i = this.position; i < this.position + numBytes; i++) {
+            const currentByte = this.bytes[i];
+            if (currentByte === 0) break;
+            result += String.fromCharCode(currentByte);
+        }
+        this.position += numBytes;
+        return result;
+    }
+
+    /**
+     * Interpret the following bytes as an integer in big or little endian.
+     *
+     * @param {number} numBytes
+     * @param {string} [endianness="be"]
+     * @returns {number}
+     */
+    readInt(numBytes, endianness="be") {
+        let val = 0;
+        if (endianness === "be") {
+            for (let i = this.position; i < this.position + numBytes; i++) {
+                val = val << 8;
+                val |= this.bytes[i];
+            }
+        } else {
+            for (let i = this.position + numBytes - 1; i >= this.position; i--) {
+                val = val << 8;
+                val |= this.bytes[i];
+            }
+        }
+        this.position += numBytes;
+        return val;
+    }
+
+    /**
+     * Consume the stream until we reach the specified byte or sequence of bytes.
+     *
+     * @param {number|List<number>} val
+     */
+    continueUntil(val) {
+        if (typeof val === "number") {
+            while (++this.position < this.bytes.length && this.bytes[this.position] !== val) {
+                continue;
+            }
+            return;
+        }
+
+        // val is an array
+        let found = false;
+        while (!found && this.position < this.bytes.length) {
+            while (++this.position < this.bytes.length && this.bytes[this.position] !== val[0]) {
+                continue;
+            }
+            found = true;
+            for (let i = 1; i < val.length; i++) {
+                if (this.position + i > this.bytes.length || this.bytes[this.position + i] !== val[i])
+                    found = false;
+            }
+        }
+    }
+
+    /**
+     * Consume the next byte if it matches the supplied value.
+     *
+     * @param {number} val
+     */
+    consumeIf(val) {
+        if (this.bytes[this.position] === val)
+            this.position++;
+    }
+
+    /**
+     * Move forwards through the stream by the specified number of bytes.
+     *
+     * @param {number} numBytes
+     */
+    moveForwardsBy(numBytes) {
+        this.position += numBytes;
+    }
+
+    /**
+     * Move to a specified position in the stream.
+     *
+     * @param {number} pos
+     */
+    moveTo(pos) {
+        if (pos < 0 || pos > this.bytes.length - 1)
+            throw new Error("Cannot move to position " + pos + " in stream. Out of bounds.");
+        this.position = pos;
+    }
+
+    /**
+     * Returns true if there are more bytes left in the stream.
+     *
+     * @returns {boolean}
+     */
+    hasMore() {
+        return this.position < this.bytes.length;
+    }
+
+    /**
+     * Returns a slice of the stream up to the current position.
+     *
+     * @returns {Uint8Array}
+     */
+    carve() {
+        return this.bytes.slice(0, this.position);
+    }
+
+}

+ 91 - 0
src/core/operations/ExtractFiles.mjs

@@ -0,0 +1,91 @@
+/**
+ * @author n1474335 [n1474335@gmail.com]
+ * @copyright Crown Copyright 2018
+ * @license Apache-2.0
+ */
+
+import Operation from "../Operation";
+// import OperationError from "../errors/OperationError";
+import Magic from "../lib/Magic";
+import Utils from "../Utils";
+import {extractFile} from "../lib/FileExtraction";
+
+/**
+ * Extract Files operation
+ */
+class ExtractFiles extends Operation {
+
+    /**
+     * ExtractFiles constructor
+     */
+    constructor() {
+        super();
+
+        this.name = "Extract Files";
+        this.module = "Default";
+        this.description = "TODO";
+        this.infoURL = "https://forensicswiki.org/wiki/File_Carving";
+        this.inputType = "ArrayBuffer";
+        this.outputType = "List<File>";
+        this.presentType = "html";
+        this.args = [];
+    }
+
+    /**
+     * @param {ArrayBuffer} input
+     * @param {Object[]} args
+     * @returns {List<File>}
+     */
+    run(input, args) {
+        const bytes = new Uint8Array(input);
+
+        // Scan for embedded files
+        const fileDetails = scanForEmbeddedFiles(bytes);
+
+        // Extract each file that we support
+        const files = [];
+        fileDetails.forEach(fileDetail => {
+            try {
+                files.push(extractFile(bytes, fileDetail));
+            } catch (err) {}
+        });
+
+        return files;
+    }
+
+    /**
+     * Displays the files in HTML for web apps.
+     *
+     * @param {File[]} files
+     * @returns {html}
+     */
+    async present(files) {
+        return await Utils.displayFilesAsHTML(files);
+    }
+
+}
+
+/**
+ * TODO refactor
+ * @param data
+ */
+function scanForEmbeddedFiles(data) {
+    let type;
+    const types = [];
+
+    for (let i = 0; i < data.length; i++) {
+        type = Magic.magicFileType(data.slice(i));
+        if (type) {
+            types.push({
+                offset: i,
+                ext: type.ext,
+                mime: type.mime,
+                desc: type.desc
+            });
+        }
+    }
+
+    return types;
+}
+
+export default ExtractFiles;