Jelajahi Sumber

Merge branch 'artemisbot-features/bz2-comp'

n1474335 6 tahun lalu
induk
melakukan
12898a1a8e

+ 35 - 11
package-lock.json

@@ -5792,7 +5792,8 @@
           "version": "2.1.1",
           "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz",
           "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=",
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "aproba": {
           "version": "1.2.0",
@@ -5816,13 +5817,15 @@
           "version": "1.0.0",
           "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz",
           "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=",
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "brace-expansion": {
           "version": "1.1.11",
           "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
           "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
           "dev": true,
+          "optional": true,
           "requires": {
             "balanced-match": "^1.0.0",
             "concat-map": "0.0.1"
@@ -5839,19 +5842,22 @@
           "version": "1.1.0",
           "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz",
           "integrity": "sha1-DQcLTQQ6W+ozovGkDi7bPZpMz3c=",
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "concat-map": {
           "version": "0.0.1",
           "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
           "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=",
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "console-control-strings": {
           "version": "1.1.0",
           "resolved": "https://registry.npmjs.org/console-control-strings/-/console-control-strings-1.1.0.tgz",
           "integrity": "sha1-PXz0Rk22RG6mRL9LOVB/mFEAjo4=",
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "core-util-is": {
           "version": "1.0.2",
@@ -5982,7 +5988,8 @@
           "version": "2.0.3",
           "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz",
           "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=",
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "ini": {
           "version": "1.3.5",
@@ -5996,6 +6003,7 @@
           "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-1.0.0.tgz",
           "integrity": "sha1-754xOG8DGn8NZDr4L95QxFfvAMs=",
           "dev": true,
+          "optional": true,
           "requires": {
             "number-is-nan": "^1.0.0"
           }
@@ -6012,6 +6020,7 @@
           "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz",
           "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==",
           "dev": true,
+          "optional": true,
           "requires": {
             "brace-expansion": "^1.1.7"
           }
@@ -6020,13 +6029,15 @@
           "version": "0.0.8",
           "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz",
           "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=",
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "minipass": {
           "version": "2.3.5",
           "resolved": "https://registry.npmjs.org/minipass/-/minipass-2.3.5.tgz",
           "integrity": "sha512-Gi1W4k059gyRbyVUZQ4mEqLm0YIUiGYfvxhF6SIlk3ui1WVxMTGfGdQ2SInh3PDrRTVvPKgULkpJtT4RH10+VA==",
           "dev": true,
+          "optional": true,
           "requires": {
             "safe-buffer": "^5.1.2",
             "yallist": "^3.0.0"
@@ -6047,6 +6058,7 @@
           "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz",
           "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=",
           "dev": true,
+          "optional": true,
           "requires": {
             "minimist": "0.0.8"
           }
@@ -6135,7 +6147,8 @@
           "version": "1.0.1",
           "resolved": "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz",
           "integrity": "sha1-CXtgK1NCKlIsGvuHkDGDNpQaAR0=",
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "object-assign": {
           "version": "4.1.1",
@@ -6149,6 +6162,7 @@
           "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
           "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=",
           "dev": true,
+          "optional": true,
           "requires": {
             "wrappy": "1"
           }
@@ -6244,7 +6258,8 @@
           "version": "5.1.2",
           "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
           "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==",
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "safer-buffer": {
           "version": "2.1.2",
@@ -6286,6 +6301,7 @@
           "resolved": "https://registry.npmjs.org/string-width/-/string-width-1.0.2.tgz",
           "integrity": "sha1-EYvfW4zcUaKn5w0hHgfisLmxB9M=",
           "dev": true,
+          "optional": true,
           "requires": {
             "code-point-at": "^1.0.0",
             "is-fullwidth-code-point": "^1.0.0",
@@ -6307,6 +6323,7 @@
           "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz",
           "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=",
           "dev": true,
+          "optional": true,
           "requires": {
             "ansi-regex": "^2.0.0"
           }
@@ -6355,13 +6372,15 @@
           "version": "1.0.2",
           "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
           "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=",
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "yallist": {
           "version": "3.0.3",
           "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.0.3.tgz",
           "integrity": "sha512-S+Zk8DEWE6oKpV+vI3qWkaK+jSbIK86pCwe2IF/xwIpQ8jEuxpw9NyaGjmp9+BoJv5FV2piqCDcoCtStppiq2A==",
-          "dev": true
+          "dev": true,
+          "optional": true
         }
       }
     },
@@ -8524,6 +8543,11 @@
       "resolved": "https://registry.npmjs.org/lex-parser/-/lex-parser-0.1.4.tgz",
       "integrity": "sha1-ZMTwJfF/1Tv7RXY/rrFvAVp0dVA="
     },
+    "libbzip2-wasm": {
+      "version": "0.0.4",
+      "resolved": "https://registry.npmjs.org/libbzip2-wasm/-/libbzip2-wasm-0.0.4.tgz",
+      "integrity": "sha512-RqscTx95+RTKhFAyjedsboR0Lmo3zd8//EuRwQXkdWmsCwYlzarVRaiYg6kS1O8m10MCQkGdrnlK9L4eAmZUwA=="
+    },
     "libyara-wasm": {
       "version": "0.0.12",
       "resolved": "https://registry.npmjs.org/libyara-wasm/-/libyara-wasm-0.0.12.tgz",

+ 1 - 0
package.json

@@ -117,6 +117,7 @@
     "jsqr": "^1.2.0",
     "jsrsasign": "8.0.12",
     "kbpgp": "2.1.0",
+    "libbzip2-wasm": "0.0.4",
     "libyara-wasm": "0.0.12",
     "lodash": "^4.17.11",
     "loglevel": "^1.6.1",

+ 72 - 0
src/core/operations/Bzip2Compress.mjs

@@ -0,0 +1,72 @@
+/**
+ * @author Matt C [me@mitt.dev]
+ * @copyright Crown Copyright 2019
+ * @license Apache-2.0
+ */
+
+import Operation from "../Operation";
+import OperationError from "../errors/OperationError";
+import Bzip2 from "libbzip2-wasm";
+
+/**
+ * Bzip2 Compress operation
+ */
+class Bzip2Compress extends Operation {
+
+    /**
+     * Bzip2Compress constructor
+     */
+    constructor() {
+        super();
+
+        this.name = "Bzip2 Compress";
+        this.module = "Compression";
+        this.description = "Bzip2 is a compression library developed by Julian Seward (of GHC fame) that uses the Burrows-Wheeler algorithm. It only supports compressing single files and its compression is slow, however is more effective than Deflate (.gz & .zip).";
+        this.infoURL = "https://wikipedia.org/wiki/Bzip2";
+        this.inputType = "ArrayBuffer";
+        this.outputType = "ArrayBuffer";
+        this.args = [
+            {
+                name: "Block size (100s of kb)",
+                type: "number",
+                value: 9,
+                min: 1,
+                max: 9
+            },
+            {
+                name: "Work factor",
+                type: "number",
+                value: 30
+            }
+        ];
+    }
+
+    /**
+     * @param {ArrayBuffer} input
+     * @param {Object[]} args
+     * @returns {File}
+     */
+    run(input, args) {
+        const [blockSize, workFactor] = args;
+        if (input.byteLength <= 0) {
+            throw new OperationError("Please provide an input.");
+        }
+        if (ENVIRONMENT_IS_WORKER()) self.sendStatusMessage("Loading Bzip2...");
+        return new Promise((resolve, reject) => {
+            Bzip2().then(bzip2 => {
+                if (ENVIRONMENT_IS_WORKER()) self.sendStatusMessage("Compressing data...");
+                const inpArray = new Uint8Array(input);
+                const bzip2cc = bzip2.compressBZ2(inpArray, blockSize, workFactor);
+                if (bzip2cc.error !== 0) {
+                    reject(new OperationError(bzip2cc.error_msg));
+                } else {
+                    const output = bzip2cc.output;
+                    resolve(output.buffer.slice(output.byteOffset, output.byteLength + output.byteOffset));
+                }
+            });
+        });
+    }
+
+}
+
+export default Bzip2Compress;

+ 29 - 13
src/core/operations/Bzip2Decompress.mjs

@@ -1,12 +1,12 @@
 /**
- * @author n1474335 [n1474335@gmail.com]
- * @copyright Crown Copyright 2016
+ * @author Matt C [me@mitt.dev]
+ * @copyright Crown Copyright 2019
  * @license Apache-2.0
  */
 
 import Operation from "../Operation";
-import bzip2 from "../vendor/bzip2";
 import OperationError from "../errors/OperationError";
+import Bzip2 from "libbzip2-wasm";
 
 /**
  * Bzip2 Decompress operation
@@ -23,9 +23,15 @@ class Bzip2Decompress extends Operation {
         this.module = "Compression";
         this.description = "Decompresses data using the Bzip2 algorithm.";
         this.infoURL = "https://wikipedia.org/wiki/Bzip2";
-        this.inputType = "byteArray";
-        this.outputType = "string";
-        this.args = [];
+        this.inputType = "ArrayBuffer";
+        this.outputType = "ArrayBuffer";
+        this.args = [
+            {
+                name: "Use low-memory, slower decompression algorithm",
+                type: "boolean",
+                value: false
+            }
+        ];
         this.patterns = [
             {
                 "match": "^\\x42\\x5a\\x68",
@@ -41,14 +47,24 @@ class Bzip2Decompress extends Operation {
      * @returns {string}
      */
     run(input, args) {
-        const compressed = new Uint8Array(input);
-
-        try {
-            const bzip2Reader = bzip2.array(compressed);
-            return bzip2.simple(bzip2Reader);
-        } catch (err) {
-            throw new OperationError(err);
+        const [small] = args;
+        if (input.byteLength <= 0) {
+            throw new OperationError("Please provide an input.");
         }
+        if (ENVIRONMENT_IS_WORKER()) self.sendStatusMessage("Loading Bzip2...");
+        return new Promise((resolve, reject) => {
+            Bzip2().then(bzip2 => {
+                if (ENVIRONMENT_IS_WORKER()) self.sendStatusMessage("Decompressing data...");
+                const inpArray = new Uint8Array(input);
+                const bzip2cc = bzip2.decompressBZ2(inpArray, small ? 1 : 0);
+                if (bzip2cc.error !== 0) {
+                    reject(new OperationError(bzip2cc.error_msg));
+                } else {
+                    const output = bzip2cc.output;
+                    resolve(output.buffer.slice(output.byteOffset, output.byteLength + output.byteOffset));
+                }
+            });
+        });
     }
 
 }

+ 0 - 265
src/core/vendor/bzip2.mjs

@@ -1,265 +0,0 @@
-/** @license
-========================================================================
-  bzip2.js - a small bzip2 decompression implementation
-  
-  Copyright 2011 by antimatter15 (antimatter15@gmail.com)
-  
-  Based on micro-bunzip by Rob Landley (rob@landley.net).
-
-  Copyright (c) 2011 by antimatter15 (antimatter15@gmail.com).
-
-  Permission is hereby granted, free of charge, to any person obtaining a
-  copy of this software and associated documentation files (the "Software"),
-  to deal in the Software without restriction, including without limitation
-  the rights to use, copy, modify, merge, publish, distribute, sublicense,
-  and/or sell copies of the Software, and to permit persons to whom the
-  Software is furnished to do so, subject to the following conditions:
-  
-  The above copyright notice and this permission notice shall be included
-  in all copies or substantial portions of the Software.
-  
-  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
-  DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
-  TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH
-  THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-*/
-"use strict";
-
-var bzip2 = {};
-
-bzip2.array = function(bytes){
-  var bit = 0, byte = 0;
-  var BITMASK = [0, 0x01, 0x03, 0x07, 0x0F, 0x1F, 0x3F, 0x7F, 0xFF ];
-  return function(n){
-    var result = 0;
-    while(n > 0){
-      var left = 8 - bit;
-      if(n >= left){
-        result <<= left;
-        result |= (BITMASK[left] & bytes[byte++]);
-        bit = 0;
-        n -= left;
-      }else{
-        result <<= n;
-        result |= ((bytes[byte] & (BITMASK[n] << (8 - n - bit))) >> (8 - n - bit));
-        bit += n;
-        n = 0;
-      }
-    }
-    return result
-  }
-}
-
-bzip2.simple = function(bits){
-  var size = bzip2.header(bits);
-  var all = '', chunk = '';
-  do{
-    all += chunk;
-    chunk = bzip2.decompress(bits, size);
-  }while(chunk != -1);
-  return all;
-}
-
-bzip2.header = function(bits){
-  if(bits(8*3) != 4348520) throw "No magic number found";
-  var i = bits(8) - 48;
-  if(i < 1 || i > 9) throw "Not a BZIP archive";
-  return i;
-};
-
-
-//takes a function for reading the block data (starting with 0x314159265359)
-//a block size (0-9) (optional, defaults to 9)
-//a length at which to stop decompressing and return the output
-bzip2.decompress = function(bits, size, len){
-  var MAX_HUFCODE_BITS = 20;
-  var MAX_SYMBOLS = 258;
-  var SYMBOL_RUNA = 0;
-  var SYMBOL_RUNB = 1;
-  var GROUP_SIZE = 50;
-  
-  var bufsize = 100000 * size;
-  for(var h = '', i = 0; i < 6; i++) h += bits(8).toString(16);
-  if(h == "177245385090") return -1; //last block
-  if(h != "314159265359") throw "Not valid bzip data";
-  bits(32); //ignore CRC codes
-  if(bits(1)) throw "Unsupported obsolete version";
-  var origPtr = bits(24);
-  if(origPtr > bufsize) throw "Initial position larger than buffer size";
-  var t = bits(16);
-  var symToByte = new Uint8Array(256), 
-      symTotal = 0;
-  for (i = 0; i < 16; i++) {
-    if(t & (1 << (15 - i))) {
-      var k = bits(16);
-      for(j = 0; j < 16; j++){
-        if(k & (1 << (15 - j))){
-          symToByte[symTotal++] = (16 * i) + j;
-        }
-      }
-    }
-  }
-
-  var groupCount = bits(3);
-  if(groupCount < 2 || groupCount > 6) throw "Error 1";
-  var nSelectors = bits(15);
-  if(nSelectors == 0) throw "Error";
-  var mtfSymbol = []; //TODO: possibly replace JS array with typed arrays
-  for(var i = 0; i < groupCount; i++) mtfSymbol[i] = i;
-  var selectors = new Uint8Array(32768);
-  
-  for(var i = 0; i < nSelectors; i++){
-    for(var j = 0; bits(1); j++) if(j >= groupCount) throw "Error 2"; 
-    var uc = mtfSymbol[j];
-    mtfSymbol.splice(j, 1); //this is a probably inefficient MTF transform
-    mtfSymbol.splice(0, 0, uc);
-    selectors[i] = uc;
-  }
-  
-  var symCount = symTotal + 2;
-  var groups = [];
-  for(var j = 0; j < groupCount; j++){
-    var length = new Uint8Array(MAX_SYMBOLS), 
-        temp = new Uint8Array(MAX_HUFCODE_BITS+1);
-    t = bits(5); //lengths
-    for(var i = 0; i < symCount; i++){
-      while(true){
-        if (t < 1 || t > MAX_HUFCODE_BITS) throw "Error 3";
-        if(!bits(1)) break;
-        if(!bits(1)) t++;
-        else t--;
-      }
-      length[i] = t;
-    }
-    var  minLen,  maxLen;
-    minLen = maxLen = length[0];
-    for(var i = 1; i < symCount; i++){
-      if(length[i] > maxLen) maxLen = length[i];
-      else if(length[i] < minLen) minLen = length[i];
-    }
-    var hufGroup;
-    hufGroup = groups[j] = {};
-    hufGroup.permute = new Uint32Array(MAX_SYMBOLS);
-    hufGroup.limit = new Uint32Array(MAX_HUFCODE_BITS + 1);
-    hufGroup.base = new Uint32Array(MAX_HUFCODE_BITS + 1);
-    hufGroup.minLen = minLen;
-    hufGroup.maxLen = maxLen;
-    var base = hufGroup.base.subarray(1);
-    var limit = hufGroup.limit.subarray(1);
-    var pp = 0;
-    for(var i = minLen; i <= maxLen; i++)
-      for(var t = 0; t < symCount; t++) 
-      if(length[t] == i) hufGroup.permute[pp++] = t;
-      for(i = minLen; i <= maxLen; i++) temp[i] = limit[i] = 0;
-      for(i = 0; i < symCount; i++) temp[length[i]]++;
-      pp = t = 0;
-      for(i = minLen; i < maxLen; i++) {
-        pp += temp[i];
-        limit[i] = pp - 1;
-        pp <<= 1;
-        base[i+1] = pp - (t += temp[i]);
-      }
-      limit[maxLen]=pp+temp[maxLen]-1;
-      base[minLen]=0;
-  }
-  var byteCount = new Uint32Array(256);
-  for(var i = 0; i < 256; i++) mtfSymbol[i] = i;
-  var runPos, count, symCount, selector;
-  runPos = count = symCount = selector = 0;
-  var buf = new Uint32Array(bufsize);
-  while(true){
-    if(!(symCount--)){
-      symCount = GROUP_SIZE - 1;
-      if(selector >= nSelectors) throw "Error 4";
-      hufGroup = groups[selectors[selector++]];
-      base = hufGroup.base.subarray(1);
-      limit = hufGroup.limit.subarray(1);
-    }
-    i = hufGroup.minLen;
-    j = bits(i);
-    while(true){
-      if(i > hufGroup.maxLen) throw "Error 5";
-      if(j <= limit[i]) break;
-      i++;
-      j = (j << 1) | bits(1);
-    }
-    j -= base[i];
-    if(j < 0 || j >= MAX_SYMBOLS) throw "Error 6";
-    var nextSym = hufGroup.permute[j];
-    if (nextSym == SYMBOL_RUNA || nextSym == SYMBOL_RUNB) {
-      if(!runPos){
-        runPos = 1;
-        t = 0;
-      }
-      if(nextSym == SYMBOL_RUNA) t += runPos;
-      else t += 2 * runPos;
-      runPos <<= 1;
-      continue;
-    }
-    if(runPos){
-      runPos = 0;
-      if(count + t >= bufsize) throw "Error 7";
-      uc = symToByte[mtfSymbol[0]];
-      byteCount[uc] += t;
-      while(t--) buf[count++] = uc;
-    }
-    if(nextSym > symTotal) break;
-    if(count >= bufsize) throw "Error 8";
-    i = nextSym -1;
-    uc = mtfSymbol[i];
-    mtfSymbol.splice(i, 1);
-    mtfSymbol.splice(0, 0, uc);
-    uc = symToByte[uc];
-    byteCount[uc]++;
-    buf[count++] = uc;
-  }
-  if(origPtr < 0 || origPtr >= count) throw "Error 9";
-  var j = 0;
-  for(var i = 0; i < 256; i++){
-    k = j + byteCount[i];
-    byteCount[i] = j;
-    j = k;
-  }
-  for(var i = 0; i < count; i++){
-    uc = buf[i] & 0xff;
-    buf[byteCount[uc]] |= (i << 8);
-    byteCount[uc]++;
-  }
-  var pos = 0, current = 0, run = 0;
-  if(count) {
-    pos = buf[origPtr];
-    current = (pos & 0xff);
-    pos >>= 8;
-    run = -1;
-  }
-  count = count;
-  var output = '';
-  var copies, previous, outbyte;
-  if(!len) len = Infinity;
-  while(count){
-    count--;
-    previous = current;
-    pos = buf[pos];
-    current = pos & 0xff;
-    pos >>= 8;
-    if(run++ == 3){
-      copies = current;
-      outbyte = previous;
-      current = -1;
-    }else{
-      copies = 1;
-      outbyte = current;
-    }
-    while(copies--){
-      output += (String.fromCharCode(outbyte));
-      if(!--len) return output;
-    }
-    if(current != previous) run = 0;
-  }
-  return output;
-}
-
-export default bzip2;