Merge branch 'master' into fernet

.devcontainer/devcontainer.json

@@ -0,0 +1,41 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the
+// README at: https://github.com/devcontainers/templates/tree/main/src/javascript-node
+{
+	"name": "CyberChef",
+	// Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile
+	"image": "mcr.microsoft.com/devcontainers/javascript-node:1-18-bookworm",
+
+	// Features to add to the dev container. More info: https://containers.dev/features.
+	"features": {
+		"ghcr.io/devcontainers/features/github-cli": "latest"
+	},
+
+	// Use 'forwardPorts' to make a list of ports inside the container available locally.
+	"forwardPorts": [8080],
+
+	// Use 'postCreateCommand' to run commands after the container is created.
+	"postCreateCommand": {
+		"npm": "bash -c \"sudo chown node node_modules && npm install\""
+	},
+
+	"containerEnv": {
+		"DISPLAY": ":99"
+	},
+
+	"mounts": [
+		"source=${localWorkspaceFolderBasename}-node_modules,target=${containerWorkspaceFolder}/node_modules,type=volume"
+	],
+
+	// Configure tool-specific properties.
+	"customizations": {
+		"vscode": {
+			"extensions": [
+				"dbaeumer.vscode-eslint",
+				"GitHub.vscode-github-actions"
+			]
+		}
+	}
+
+	// Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root.
+	// "remoteUser": "root"
+}

.dockerignore

@@ -0,0 +1,2 @@
+node_modules
+build

.eslintignore

@@ -1 +1 @@
-src/core/vendor/**
+src/core/vendor/**

.eslintrc.json

@@ -1,7 +1,7 @@
 {
-    "parser": "babel-eslint",
+    "parser": "@babel/eslint-parser",
     "parserOptions": {
-        "ecmaVersion": 9,
+        "ecmaVersion": 2022,
         "ecmaFeatures": {
             "impliedStrict": true
         },
@@ -38,12 +38,16 @@
 
         // disable rules from base configurations
         "no-control-regex": "off",
+        "require-atomic-updates": "off",
+        "no-async-promise-executor": "off",
 
         // stylistic conventions
         "brace-style": ["error", "1tbs"],
+        "space-before-blocks": ["error", "always"],
         "block-spacing": "error",
         "array-bracket-spacing": "error",
         "comma-spacing": "error",
+        "spaced-comment": ["error", "always", { "exceptions": ["/"] } ],
         "comma-style": "error",
         "computed-property-spacing": "error",
         "no-trailing-spaces": "warn",
@@ -59,7 +63,8 @@
         }],
         "linebreak-style": ["error", "unix"],
         "quotes": ["error", "double", {
-            "avoidEscape": true
+            "avoidEscape": true,
+            "allowTemplateLiterals": true
         }],
         "camelcase": ["error", {
             "properties": "always"
@@ -102,12 +107,10 @@
         "$": false,
         "jQuery": false,
         "log": false,
+        "app": false,
 
         "COMPILE_TIME": false,
         "COMPILE_MSG": false,
-        "PKG_VERSION": false,
-        "ENVIRONMENT_IS_WORKER": false,
-        "ENVIRONMENT_IS_NODE": false,
-        "ENVIRONMENT_IS_WEB": false
+        "PKG_VERSION": false
     }
 }

.gitattributes

@@ -0,0 +1 @@
+* text=auto eol=lf

.github/CONTRIBUTING.md

@@ -6,7 +6,7 @@ There are lots of opportunities to contribute to CyberChef. If you want ideas, t
 
 Before your contributions can be accepted, you must:
 
- - Sign the [GCHQ Contributor Licence Agreement](https://github.com/gchq/Gaffer/wiki/GCHQ-OSS-Contributor-License-Agreement-V1.0)
+ - Sign the [GCHQ Contributor Licence Agreement](https://cla-assistant.io/gchq/CyberChef)
  - Push your changes to your fork.
  - Submit a pull request.
 
@@ -22,15 +22,15 @@ Before your contributions can be accepted, you must:
 * Line endings: UNIX style (\n)
 
 
-## Design Principals
+## Design Principles
 
 1. If at all possible, all operations and features should be client-side and not rely on connections to an external server. This increases the utility of CyberChef on closed networks and in virtual machines that are not connected to the Internet. Calls to external APIs may be accepted if there is no other option, but not for critical components.
-2. Latency should be kept to a minimum to enhance the user experience. This means that all operation code should sit on the client, rather than being loaded dynamically from a server.
-3. Use Vanilla JS if at all possible to reduce the number of libraries required and relied upon. Frameworks like jQuery, although included, should not be used unless absolutely necessary.
-4. Minimise the use of large libraries, especially for niche operations that won't be used very often - these will be downloaded by everyone using the app, whether they use that operation or not (due to principal 2).
+2. Latency should be kept to a minimum to enhance the user experience. This means that operation code should sit on the client and be executed there. However, as a trade-off between latency and bandwidth, operation code with large dependencies can be loaded in discrete modules in order to reduce the size of the initial download. The downloading of additional modules must remain entirely transparent so that the user is not inconvenienced.
+3. Large libraries should be kept in separate modules so that they are not downloaded by everyone who uses the app, just those who specifically require the relevant operations.
+4. Use Vanilla JS if at all possible to reduce the number of libraries required and relied upon. Frameworks like jQuery, although included, should not be used unless absolutely necessary.
 
 
-With these principals in mind, any changes or additions to CyberChef should keep it:
+With these principles in mind, any changes or additions to CyberChef should keep it:
 
  - Standalone
  - Efficient

.github/ISSUE_TEMPLATE.md

@@ -1,14 +1 @@
-<!-- Prefix the title above with one of the following: -->
-<!-- Bug report: -->
-<!-- Operation request: -->
-<!-- Feature request: -->
-<!-- Misc: -->
-
-### Summary
-
-
-### Example
-<!-- If describing a bug, tell us what happens instead of the expected behavior -->
-<!-- Include a link that triggers the bug if possible -->
-<!-- If you are requesting a new operation, include example input and output -->
-
+<!-- Prefix the title above with 'Misc:' -->

.github/ISSUE_TEMPLATE/bug-report.md

@@ -0,0 +1,33 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: 'Bug report: <Insert title here>'
+labels: bug
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behaviour or a link to the recipe / input used to cause the bug:
+
+1. Go to '...'
+2. Click on '....'
+3. Scroll down to '....'
+4. See error
+
+**Expected behaviour**
+A clear and concise description of what you expected to happen.
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
+
+**Desktop (if relevant, please complete the following information):**
+ - OS: [e.g. Windows] 
+ - Browser: [e.g. chrome 72, firefox 60]
+ - CyberChef version: [e.g. 9.7.14]
+
+**Additional context**
+Add any other context about the problem here.

.github/ISSUE_TEMPLATE/feature-request.md

@@ -0,0 +1,20 @@
+---
+name: Feature request
+about: Suggest an idea for the project
+title: 'Feature request: <Insert title here>'
+labels: feature
+assignees: ''
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. E.g. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

.github/ISSUE_TEMPLATE/operation-request.md

@@ -0,0 +1,14 @@
+---
+name: Operation request
+about: Suggest a new operation
+title: 'Operation request: <Insert title here>'
+labels: operation
+assignees: ''
+
+---
+
+## Summary
+
+### Example Input
+
+### Example Output

.github/workflows/codeql.yml

@@ -0,0 +1,40 @@
+name: "CodeQL Analysis"
+
+on:
+  workflow_dispatch:
+  push:
+    branches: [ master ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ master ]
+    types: [synchronize, opened, reopened]
+  schedule:
+    - cron: '22 17 * * 5'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'javascript' ]
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v3
+
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v2
+      with:
+        languages: ${{ matrix.language }}
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v2
+      with:
+        category: "/language:${{matrix.language}}"

.github/workflows/master.yml

@@ -0,0 +1,57 @@
+name: "Master Build, Test & Deploy"
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+    - master
+
+jobs:
+  main:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Set node version
+      uses: actions/setup-node@v3
+      with:
+        node-version: '18.x'
+
+    - name: Install
+      run: |
+        npm install
+        npm run setheapsize
+
+    - name: Lint
+      run: npx grunt lint
+
+    - name: Unit Tests
+      run: |
+        npm test
+        npm run testnodeconsumer
+
+    - name: Production Build
+      if: success()
+      run: npx grunt prod --msg="Version 10 is here! Read about the new features <a href='https://github.com/gchq/CyberChef/wiki/Character-encoding,-EOL-separators,-and-editor-features'>here</a>"
+
+    - name: Generate sitemap
+      run: npx grunt exec:sitemap
+
+    - name: UI Tests
+      if: success()
+      run: |
+        sudo apt-get install xvfb
+        xvfb-run --server-args="-screen 0 1200x800x24" npx grunt testui
+
+    - name: Prepare for GitHub Pages
+      if: success()
+      run: npx grunt copy:ghPages
+
+    - name: Deploy to GitHub Pages
+      if: success() && github.ref == 'refs/heads/master'
+      uses: crazy-max/ghaction-github-pages@v3
+      with:
+        target_branch: gh-pages
+        build_dir: ./build/prod
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/pull_requests.yml

@@ -0,0 +1,54 @@
+name: "Pull Requests"
+
+on:
+  workflow_dispatch:
+  pull_request:
+    types: [synchronize, opened, reopened]
+
+jobs:
+  main:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Set node version
+      uses: actions/setup-node@v3
+      with:
+        node-version: '18.x'
+
+    - name: Install
+      run: |
+        npm install
+        npm run setheapsize
+
+    - name: Lint
+      run: npx grunt lint
+
+    - name: Unit Tests
+      run: |
+        npm test
+        npm run testnodeconsumer
+
+    - name: Production Build
+      if: success()
+      run: npx grunt prod
+
+    - name: Production Image Build
+      if: success()
+      id: build-image
+      uses: redhat-actions/buildah-build@v2
+      with:
+        # Not being uploaded to any registry, use a simple name to allow Buildah to build correctly.
+        image: cyberchef
+        containerfiles: ./Dockerfile
+        platforms: linux/amd64
+        oci: true
+        # Webpack seems to use a lot of open files, increase the max open file limit to accomodate.
+        extra-args: |
+          --ulimit nofile=10000
+
+    - name: UI Tests
+      if: success()
+      run: |
+        sudo apt-get install xvfb
+        xvfb-run --server-args="-screen 0 1200x800x24" npx grunt testui

.github/workflows/releases.yml

@@ -0,0 +1,93 @@
+name: "Releases"
+
+on:
+  workflow_dispatch:
+  push:
+    tags:
+    - 'v*'
+
+env:
+  REGISTRY: ghcr.io
+  REGISTRY_USER: ${{ github.actor }}
+  REGISTRY_PASSWORD: ${{ github.token }}
+  IMAGE_NAME: ${{ github.repository }}
+
+jobs:
+  main:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Set node version
+      uses: actions/setup-node@v3
+      with:
+        node-version: '18.x'
+
+    - name: Install
+      run: |
+        npm ci
+        npm run setheapsize
+
+    - name: Lint
+      run: npx grunt lint
+
+    - name: Unit Tests
+      run: |
+        npm test
+        npm run testnodeconsumer
+
+    - name: Production Build
+      run: npx grunt prod
+
+    - name: UI Tests
+      run: |
+        sudo apt-get install xvfb
+        xvfb-run --server-args="-screen 0 1200x800x24" npx grunt testui
+
+    - name: Image Metadata
+      id: image-metadata
+      uses: docker/metadata-action@v4
+      with:
+        images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+        tags: |
+          type=semver,pattern={{major}}
+          type=semver,pattern={{major}}.{{minor}}
+          type=semver,pattern={{version}}
+
+    - name: Production Image Build
+      id: build-image
+      uses: redhat-actions/buildah-build@v2
+      with:
+        tags: ${{ steps.image-metadata.outputs.tags }}
+        labels: ${{ steps.image-metadata.outputs.labels }}
+        containerfiles: ./Dockerfile
+        platforms: linux/amd64
+        oci: true
+        # Webpack seems to use a lot of open files, increase the max open file limit to accomodate.
+        extra-args: |
+          --ulimit nofile=10000
+
+
+    - name: Upload Release Assets
+      id: upload-release-assets
+      uses: svenstaro/upload-release-action@v2
+      with:
+        repo_token: ${{ secrets.GITHUB_TOKEN }}
+        file: build/prod/*.zip
+        tag: ${{ github.ref }}
+        overwrite: true
+        file_glob: true
+        body: "See the [CHANGELOG](https://github.com/gchq/CyberChef/blob/master/CHANGELOG.md) and [commit messages](https://github.com/gchq/CyberChef/commits/master) for details."
+
+    - name: Publish to NPM
+      uses: JS-DevTools/npm-publish@v1
+      with:
+        token: ${{ secrets.NPM_TOKEN }}
+
+    - name: Publish to GHCR
+      uses: redhat-actions/push-to-registry@v2
+      with:
+        tags: ${{ steps.build-image.outputs.tags }}
+        registry: ${{ env.REGISTRY }}
+        username: ${{ env.REGISTRY_USER }}
+        password: ${{ env.REGISTRY_PASSWORD }}

.gitignore

@@ -2,12 +2,14 @@ node_modules
 npm-debug.log
 travis.log
 build
-docs/*
-!docs/*.conf.json
-!docs/*.ico
 .vscode
+.idea
+.*.swp
 src/core/config/modules/*
 src/core/config/OperationConfig.json
 src/core/operations/index.mjs
+src/node/config/OperationConfig.json
+src/node/index.mjs
+**/*.DS_Store
 tests/browser/output/*
-
+.node-version

.npmignore

@@ -3,6 +3,5 @@ npm-debug.log
 travis.log
 build/*
 !build/node
-docs
 .vscode
 .github

.nvmrc

@@ -0,0 +1 @@
+18

.travis.yml

@@ -1,53 +0,0 @@
-language: node_js
-node_js:
-  - node
-addons:
-  chrome: stable
-install: npm install
-before_script:
-  - npm install -g grunt
-  - export NODE_OPTIONS=--max_old_space_size=2048
-script:
-  - grunt lint
-  - grunt test
-  - grunt docs
-  - grunt node
-  - grunt prod --msg="$COMPILE_MSG"
-  - xvfb-run --server-args="-screen 0 1200x800x24" grunt testui
-before_deploy:
-  - grunt exec:sitemap
-  - grunt copy:ghPages
-deploy:
-  - provider: pages
-    skip_cleanup: true
-    github_token: $GITHUB_TOKEN
-    local_dir: build/prod/
-    target_branch: gh-pages
-    on:
-      repo: gchq/CyberChef
-      branch: master
-  - provider: releases
-    skip_cleanup: true
-    api_key:
-      secure: "HV1WSKv4l/0Y2bKKs1iBJocBcmLj08PCRUeEM/jTwA4jqJ8EiLHWiXtER/D5sEg2iibRVKd2OQjfrmS6bo4AiwdeVgAKmv0FtS2Jw+391N8Nd5AkEANHa5Om/IpHLTL2YRAjpJTsDpY72bMUTJIwjQA3TFJkgrpOw6KYfohOcgbxLpZ4XuNJRU3VL4Hsxdv5V9aOVmfFOmMOVPQlakXy7NgtW5POp1f2WJwgcZxylkR1CjwaqMyXmSoVl46pyH3tr5+dptsQoKSGdi6sIHGA60oDotFPcm+0ifa47wZw+vapuuDi4tdNxhrHGaDMG8xiE0WFDHwQUDlk2/+W7j9SEX0H3Em7us371JXRp56EDwEcDa34VpVkC6i8HGcHK55hnxVbMZXGf3qhOFD8wY7qMbjMRvIpucrMHBi86OfkDfv0vDj2LyvIl5APj/AX50BrE0tfH1MZbH26Jkx4NdlkcxQ14GumarmUqfmVvbX/fsoA6oUuAAE9ZgRRi3KHO4wci6KUcRfdm+XOeUkaBFsL86G3EEYIvrtBTuaypdz+Cx7nd1iPZyWMx5Y1gXnVzha4nBdV4+7l9JIsFggD8QVpw2uHXQiS1KXFjOeqA3DBD8tjMB7q26Fl2fD3jkOo4BTbQ2NrRIZUu/iL+fOmMPsyMt2qulB0yaSBCfkbEq8xrUA="
-    file:
-      - build/prod/cyberchef.htm
-      - build/node/CyberChef.js
-    on:
-      repo: gchq/CyberChef
-      tags: true
-  - provider: npm
-    skip_cleanup: true
-    email: "n1474335@gmail.com"
-    api_key:
-      secure: "UnDQL3Kh+GK2toL0TK3FObO0ujVssU3Eg4BBuYdjwLB81GhiGE5/DTh7THdZPOpbLo6wQeOwfZDuMeKC1OU+0Uf4NsdYFu1aq6xMO20qBQ4qUfgsyiK4Qgywj9gk0p1+OFZdGAZ/j1CNRAaF71XQIY6iV84c+SO4WoizXYrNT0Jh4sr2DA4/97G2xmJtPi0qOzYrJ09R56ZUozmqeik5G0pMRIuJRbpjS/7bZXV+N7WV0ombZc9RkUaetbabEVOLQ+Xx5YAIVq+VuEeMe9VBSnxY/FfCLmy1wJsjGzpLCyBI9nbrG4nw8Wgc2m8NfK9rcpIvBTGner9r2j60NVDkZ8kLZPrqXhq6AZMwa+oz6K5UQCqRo2RRQzSGwXxg67HY5Tcq+oNmjd+DqpPg4LZ3eGlluyP5XfG+hpSr9Ya4d8q8SrUWLxkoLHI6ZKMtoKFbTCSSQPiluW5hsZxjz3yDkkjsJw64M/EM8UyJrgaXqDklQu+7rBGKLfsK6os7RDiqjBWpQ7gwpo8HvY0O8yqEAabPz+QGkanpjcCOZCXFbSkzWxYy37RMAPu88iINVZVlZE4l+WJenCpZY95ueyy0mG9cyMSzVRPyX6A+/n4H6VMFPFjpGDLTD588ACEjY1lmHfS/eXwXJcgqPPD2gW0XdRdUheU/ssqlfCfGWQMTDXs="
-    on:
-      tags: true
-      branch: master
-notifications:
-    webhooks:
-        urls:
-            - https://webhooks.gitter.im/e/83c143a6822e218d5b34
-        on_success: change
-        on_failure: always
-        on_start: never

CHANGELOG.md

@@ -1,7 +1,287 @@
 # Changelog
+
+## Versioning
+
+CyberChef uses the [semver](https://semver.org/) system to manage versioning: `<MAJOR>.<MINOR>.<PATCH>`.
+
+- MAJOR version changes represent a significant change to the fundamental architecture of CyberChef and may (but don't always) make breaking changes that are not backwards compatible.
+- MINOR version changes usually mean the addition of new operations or reasonably significant new features.
+- PATCH versions are used for bug fixes and any other small tweaks that modify or improve existing capabilities.
+
 All major and minor version changes will be documented in this file. Details of patch-level version changes can be found in [commit messages](https://github.com/gchq/CyberChef/commits/master).
 
 
+## Details
+
+### [10.8.0] - 2024-02-13
+- Add official Docker images [@AshCorr] | [#1699]
+
+### [10.7.0] - 2024-02-09
+- Added 'File Tree' operation [@sw5678] | [#1667]
+- Added 'RISON' operation [@sg5506844] | [#1555]
+- Added 'MurmurHash3' operation [@AliceGrey] | [#1694]
+
+### [10.6.0] -  2024-02-03
+- Updated 'Forensics Wiki' URLs to new domain [@a3957273] | [#1703]
+- Added 'LZNT1 Decompress' operation [@0xThiebaut] | [#1675]
+- Updated 'Regex Expression' UUID matcher [@cnotin] | [#1678]
+- Removed duplicate 'hover' message within baking info [@KevinSJ] | [#1541]
+
+### [10.5.0] - 2023-07-14
+- Added GOST Encrypt, Decrypt, Sign, Verify, Key Wrap, and Key Unwrap operations [@n1474335] | [#592]
+
+### [10.4.0] - 2023-03-24
+- Added 'Generate De Bruijn Sequence' operation [@gchq77703] | [#493]
+
+### [10.3.0] - 2023-03-24
+- Added 'Argon2' and 'Argon2 compare' operations [@Xenonym] | [#661]
+
+### [10.2.0] - 2023-03-23
+- Added 'Derive HKDF key' operation [@mikecat] | [#1528]
+
+### [10.1.0] - 2023-03-23
+- Added 'Levenshtein Distance' operation [@mikecat] | [#1498]
+- Added 'Swap case' operation [@mikecat] | [#1499]
+
+## [10.0.0] - 2023-03-22
+- [Full details explained here](https://github.com/gchq/CyberChef/wiki/Character-encoding,-EOL-separators,-and-editor-features)
+- Status bars added to the Input and Output [@n1474335] | [#1405]
+- Character encoding selection added to the Input and Output [@n1474335] | [#1405]
+- End of line separator selection added to the Input and Output [@n1474335] | [#1405]
+- Non-printable characters are rendered as control character pictures [@n1474335] | [#1405]
+- Loaded files can now be edited in the Input [@n1474335] | [#1405]
+- Various editor features added such as multiple selections and bracket matching [@n1474335] | [#1405]
+- Contextual help added, activated by pressing F1 while hovering over features [@n1474335] | [#1405]
+- Many, many UI tests added for I/O features and operations [@n1474335] | [#1405]
+
+<details>
+    <summary>Click to expand v9 minor versions</summary>
+
+### [9.55.0] - 2022-12-09
+- Added 'AMF Encode' and 'AMF Decode' operations [@n1474335] | [760eff4]
+
+### [9.54.0] - 2022-11-25
+- Added 'Rabbit' operation [@mikecat] | [#1450]
+
+### [9.53.0] - 2022-11-25
+- Added 'AES Key Wrap' and 'AES Key Unwrap' operations [@mikecat] | [#1456]
+
+### [9.52.0] - 2022-11-25
+- Added 'ChaCha' operation [@joostrijneveld] | [#1466]
+
+### [9.51.0] - 2022-11-25
+- Added 'CMAC' operation [@mikecat] | [#1457]
+
+### [9.50.0] - 2022-11-25
+- Added 'Shuffle' operation [@mikecat] | [#1472]
+
+### [9.49.0] - 2022-11-11
+- Added 'LZ4 Compress' and 'LZ4 Decompress' operations [@n1474335] | [31a7f83]
+
+### [9.48.0] - 2022-10-14
+- Added 'LM Hash' and 'NT Hash' operations [@n1474335] [@brun0ne] | [#1427]
+
+### [9.47.0] - 2022-10-14
+- Added 'LZMA Decompress' and 'LZMA Compress' operations [@mattnotmitt] | [#1421]
+
+### [9.46.0] - 2022-07-08
+- Added 'Cetacean Cipher Encode' and 'Cetacean Cipher Decode' operations [@valdelaseras] | [#1308]
+
+### [9.45.0] - 2022-07-08
+- Added 'ROT8000' operation [@thomasleplus] | [#1250]
+
+### [9.44.0] - 2022-07-08
+- Added 'LZString Compress' and 'LZString Decompress' operations [@crespyl] | [#1266]
+
+### [9.43.0] - 2022-07-08
+- Added 'ROT13 Brute Force' and 'ROT47 Brute Force' operations [@mikecat] | [#1264]
+
+### [9.42.0] - 2022-07-08
+- Added 'LS47 Encrypt' and 'LS47 Decrypt' operations [@n1073645] | [#951]
+
+### [9.41.0] - 2022-07-08
+- Added 'Caesar Box Cipher' operation [@n1073645] | [#1066]
+
+### [9.40.0] - 2022-07-08
+- Added 'P-list Viewer' operation [@n1073645] | [#906]
+
+### [9.39.0] - 2022-06-09
+- Added 'ELF Info' operation [@n1073645] | [#1364]
+
+### [9.38.0] - 2022-05-30
+- Added 'Parse TCP' operation [@n1474335] | [a895d1d]
+
+### [9.37.0] - 2022-03-29
+- 'SM4 Encrypt' and 'SM4 Decrypt' operations added [@swesven] | [#1189]
+- NoPadding options added for CBC and ECB modes in AES, DES and Triple DES Decrypt operations [@swesven] | [#1189]
+
+### [9.36.0] - 2022-03-29
+- 'SIGABA' operation added [@hettysymes] | [#934]
+
+### [9.35.0] - 2022-03-28
+- 'To Base45' and 'From Base45' operations added [@t-8ch] | [#1242]
+
+### [9.34.0] - 2022-03-28
+- 'Get All Casings' operation added [@n1073645] | [#1065]
+
+### [9.33.0] - 2022-03-25
+- Updated to support Node 17 [@n1474335] [@john19696] [@t-8ch] | [[#1326] [#1313] [#1244]
+- Improved CJS and ESM module support [@d98762625] | [#1037]
+
+### [9.32.0] - 2021-08-18
+- 'Protobuf Encode' operation added and decode operation modified to allow decoding with full and partial schemas [@n1474335] | [dd18e52]
+
+### [9.31.0] - 2021-08-10
+- 'HASSH Client Fingerprint' and 'HASSH Server Fingerprint' operations added [@n1474335] | [e9ca4dc]
+
+### [9.30.0] - 2021-08-10
+- 'JA3S Fingerprint' operation added [@n1474335] | [289a417]
+
+### [9.29.0] - 2021-07-28
+- 'JA3 Fingerprint' operation added [@n1474335] | [9a33498]
+
+### [9.28.0] - 2021-03-26
+- 'CBOR Encode' and 'CBOR Decode' operations added [@Danh4] | [#999]
+
+### [9.27.0] - 2021-02-12
+- 'Fuzzy Match' operation added [@n1474335] | [8ad18b]
+
+### [9.26.0] - 2021-02-11
+- 'Get Time' operation added [@n1073645] [@n1474335] | [#1045]
+
+### [9.25.0] - 2021-02-11
+- 'Extract ID3' operation added [@n1073645] [@n1474335] | [#1006]
+
+### [9.24.0] - 2021-02-02
+- 'SM3' hashing function added along with more configuration options for other hashing operations [@n1073645] [@n1474335] | [#1022]
+
+### [9.23.0] - 2021-02-01
+- Various RSA operations added to encrypt, decrypt, sign, verify and generate keys [@mattnotmitt] [@GCHQ77703] | [#652]
+
+### [9.22.0] - 2021-02-01
+- 'Unicode Text Format' operation added [@mattnotmitt] | [#1083]
+
+### [9.21.0] - 2020-06-12
+- Node API now exports `magic` operation [@d98762625] | [#1049]
+
+### [9.20.0] - 2020-03-27
+- 'Parse ObjectID Timestamp' operation added [@dmfj] | [#987]
+
+### [9.19.0] - 2020-03-24
+- Improvements to the 'Magic' operation, allowing it to recognise more data formats and provide more accurate results [@n1073645] [@n1474335] | [#966] [b765534b](https://github.com/gchq/CyberChef/commit/b765534b8b2a0454a5132a0a52d1d8844bcbdaaa)
+
+### [9.18.0] - 2020-03-13
+- 'Convert to NATO alphabet' operation added [@MarvinJWendt] | [#674]
+
+### [9.17.0] - 2020-03-13
+- 'Generate Image' operation added [@pointhi] | [#683]
+
+### [9.16.0] - 2020-03-06
+- 'Colossus' operation added [@VirtualColossus] | [#917]
+
+### [9.15.0] - 2020-03-05
+- 'CipherSaber2 Encrypt' and 'CipherSaber2 Decrypt' operations added [@n1073645] | [#952]
+
+### [9.14.0] - 2020-03-05
+- 'Luhn Checksum' operation added [@n1073645] | [#965]
+
+### [9.13.0] - 2020-02-13
+- 'Rail Fence Cipher Encode' and 'Rail Fence Cipher Decode' operations added [@Flavsditz] | [#948]
+
+### [9.12.0] - 2019-12-20
+- 'Normalise Unicode' operation added [@matthieuxyz] | [#912]
+
+### [9.11.0] - 2019-11-06
+- Implemented CFB, OFB, and CTR modes for Blowfish operations [@cbeuw] | [#653]
+
+### [9.10.0] - 2019-11-06
+- 'Lorenz' operation added [@VirtualColossus] | [#528]
+
+### [9.9.0] - 2019-11-01
+- Added support for 109 more character encodings [@n1474335]
+
+### [9.8.0] - 2019-10-31
+- 'Avro to JSON' operation added [@jarrodconnolly] | [#865]
+
+### [9.7.0] - 2019-09-13
+- 'Optical Character Recognition' operation added [@MShwed] [@n1474335] | [#632]
+
+### [9.6.0] - 2019-09-04
+- 'Bacon Cipher Encode' and 'Bacon Cipher Decode' operations added [@kassi] | [#500]
+
+### [9.5.0] - 2019-09-04
+- Various Steganography operations added: 'Extract LSB', 'Extract RGBA', 'Randomize Colour Palette', and 'View Bit Plane' [@Ge0rg3] | [#625]
+
+### [9.4.0] - 2019-08-30
+- 'Render Markdown' operation added [@j433866] | [#627]
+
+### [9.3.0] - 2019-08-30
+- 'Show on map' operation added [@j433866] | [#477]
+
+### [9.2.0] - 2019-08-23
+- 'Parse UDP' operation added [@h345983745] | [#614]
+
+### [9.1.0] - 2019-08-22
+- 'Parse SSH Host Key' operation added [@j433866] | [#595]
+- 'Defang IP Addresses' operation added [@h345983745] | [#556]
+
+</details>
+
+## [9.0.0] - 2019-07-09
+- [Multiple inputs](https://github.com/gchq/CyberChef/wiki/Multiple-Inputs) are now supported in the main web UI, allowing you to upload and process multiple files at once [@j433866] | [#566]
+- A [Node.js API](https://github.com/gchq/CyberChef/wiki/Node-API) has been implemented, meaning that CyberChef can now be used as a library, either to provide specific operations, or an entire baking environment [@d98762625] | [#291]
+- A [read-eval-print loop (REPL)](https://github.com/gchq/CyberChef/wiki/Node-API#repl) is also included to enable prototyping and experimentation with the API [@d98762625] | [#291]
+- Light and dark Solarized themes added [@j433866] | [#566]
+
+<details>
+    <summary>Click to expand v8 minor versions</summary>
+
+### [8.38.0] - 2019-07-03
+- 'Streebog' and 'GOST hash' operations added [@MShwed] [@n1474335] | [#530]
+
+### [8.37.0] - 2019-07-03
+- 'CRC-8 Checksum' operation added [@MShwed] | [#591]
+
+### [8.36.0] - 2019-07-03
+- 'PGP Verify' operation added [@artemisbot] | [#585]
+
+### [8.35.0] - 2019-07-03
+- 'Sharpen Image', 'Convert Image Format' and 'Add Text To Image' operations added [@j433866] | [#515]
+
+### [8.34.0] - 2019-06-28
+- Various new visualisations added to the 'Entropy' operation [@MShwed] | [#535]
+- Efficiency improvements made to the 'Entropy' operation for large file support [@n1474335]
+
+### [8.33.0] - 2019-06-27
+- 'Bzip2 Compress' operation added and 'Bzip2 Decompress' operation greatly improved [@artemisbot] | [#531]
+
+### [8.32.0] - 2019-06-27
+- 'Index of Coincidence' operation added [@Ge0rg3] | [#571]
+
+### [8.31.0] - 2019-04-12
+- The downloadable version of CyberChef is now a .zip file containing separate modules rather than a single .htm file. It is still completely standalone and will not make any external network requests. This change reduces the complexity of the build process significantly. [@n1474335]
+
+### [8.30.0] - 2019-04-12
+- 'Decode Protobuf' operation added [@n1474335] | [#533]
+
+### [8.29.0] - 2019-03-31
+- 'BLAKE2s' and 'BLAKE2b' hashing operations added [@h345983745] | [#525]
+
+### [8.28.0] - 2019-03-31
+- 'Heatmap Chart', 'Hex Density Chart', 'Scatter Chart' and 'Series Chart' operation added [@artemisbot] [@tlwr] | [#496] [#143]
+
+### [8.27.0] - 2019-03-14
+- 'Enigma', 'Typex', 'Bombe' and 'Multiple Bombe' operations added [@s2224834] | [#516]
+- See [this wiki article](https://github.com/gchq/CyberChef/wiki/Enigma,-the-Bombe,-and-Typex) for a full explanation of these operations.
+- New Bombe-style loading animation added for long-running operations [@n1474335]
+- New operation argument types added: `populateMultiOption` and `argSelector` [@n1474335]
+
+### [8.26.0] - 2019-03-09
+- Various image manipulation operations added [@j433866] | [#506]
+
+### [8.25.0] - 2019-03-09
+- 'Extract Files' operation added and more file formats supported [@n1474335] | [#440]
+
 ### [8.24.0] - 2019-02-08
 - 'DNS over HTTPS' operation added [@h345983745] | [#489]
 
@@ -77,6 +357,8 @@ All major and minor version changes will be documented in this file. Details of
 ### [8.1.0] - 2018-08-19
 - 'Dechunk HTTP response' operation added [@sevzero] | [#311]
 
+</details>
+
 ## [8.0.0] - 2018-08-05
 - Codebase rewritten using [ES modules](https://hacks.mozilla.org/2018/03/es-modules-a-cartoon-deep-dive/) and [classes](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Classes) [@n1474335] [@d98762625] [@artemisbot] [@picapi] | [#284]
 - Operation architecture restructured to make adding new operations a lot simpler [@n1474335] | [#284]
@@ -104,8 +386,85 @@ All major and minor version changes will be documented in this file. Details of
 ## [4.0.0] - 2016-11-28
 -  Initial open source commit [@n1474335] | [b1d73a72](https://github.com/gchq/CyberChef/commit/b1d73a725dc7ab9fb7eb789296efd2b7e4b08306)
 
-
-
+[10.8.0]: https://github.com/gchq/CyberChef/releases/tag/v10.7.0
+[10.7.0]: https://github.com/gchq/CyberChef/releases/tag/v10.7.0
+[10.6.0]: https://github.com/gchq/CyberChef/releases/tag/v10.6.0
+[10.5.0]: https://github.com/gchq/CyberChef/releases/tag/v10.5.0
+[10.4.0]: https://github.com/gchq/CyberChef/releases/tag/v10.4.0
+[10.3.0]: https://github.com/gchq/CyberChef/releases/tag/v10.3.0
+[10.2.0]: https://github.com/gchq/CyberChef/releases/tag/v10.2.0
+[10.1.0]: https://github.com/gchq/CyberChef/releases/tag/v10.1.0
+[10.0.0]: https://github.com/gchq/CyberChef/releases/tag/v10.0.0
+[9.55.0]: https://github.com/gchq/CyberChef/releases/tag/v9.55.0
+[9.54.0]: https://github.com/gchq/CyberChef/releases/tag/v9.54.0
+[9.53.0]: https://github.com/gchq/CyberChef/releases/tag/v9.53.0
+[9.52.0]: https://github.com/gchq/CyberChef/releases/tag/v9.52.0
+[9.51.0]: https://github.com/gchq/CyberChef/releases/tag/v9.51.0
+[9.50.0]: https://github.com/gchq/CyberChef/releases/tag/v9.50.0
+[9.49.0]: https://github.com/gchq/CyberChef/releases/tag/v9.49.0
+[9.48.0]: https://github.com/gchq/CyberChef/releases/tag/v9.48.0
+[9.47.0]: https://github.com/gchq/CyberChef/releases/tag/v9.47.0
+[9.46.0]: https://github.com/gchq/CyberChef/releases/tag/v9.46.0
+[9.45.0]: https://github.com/gchq/CyberChef/releases/tag/v9.45.0
+[9.44.0]: https://github.com/gchq/CyberChef/releases/tag/v9.44.0
+[9.43.0]: https://github.com/gchq/CyberChef/releases/tag/v9.43.0
+[9.42.0]: https://github.com/gchq/CyberChef/releases/tag/v9.42.0
+[9.41.0]: https://github.com/gchq/CyberChef/releases/tag/v9.41.0
+[9.40.0]: https://github.com/gchq/CyberChef/releases/tag/v9.40.0
+[9.39.0]: https://github.com/gchq/CyberChef/releases/tag/v9.39.0
+[9.38.0]: https://github.com/gchq/CyberChef/releases/tag/v9.38.0
+[9.37.0]: https://github.com/gchq/CyberChef/releases/tag/v9.37.0
+[9.36.0]: https://github.com/gchq/CyberChef/releases/tag/v9.36.0
+[9.35.0]: https://github.com/gchq/CyberChef/releases/tag/v9.35.0
+[9.34.0]: https://github.com/gchq/CyberChef/releases/tag/v9.34.0
+[9.33.0]: https://github.com/gchq/CyberChef/releases/tag/v9.33.0
+[9.32.0]: https://github.com/gchq/CyberChef/releases/tag/v9.32.0
+[9.31.0]: https://github.com/gchq/CyberChef/releases/tag/v9.31.0
+[9.30.0]: https://github.com/gchq/CyberChef/releases/tag/v9.30.0
+[9.29.0]: https://github.com/gchq/CyberChef/releases/tag/v9.29.0
+[9.28.0]: https://github.com/gchq/CyberChef/releases/tag/v9.28.0
+[9.27.0]: https://github.com/gchq/CyberChef/releases/tag/v9.27.0
+[9.26.0]: https://github.com/gchq/CyberChef/releases/tag/v9.26.0
+[9.25.0]: https://github.com/gchq/CyberChef/releases/tag/v9.25.0
+[9.24.0]: https://github.com/gchq/CyberChef/releases/tag/v9.24.0
+[9.23.0]: https://github.com/gchq/CyberChef/releases/tag/v9.23.0
+[9.22.0]: https://github.com/gchq/CyberChef/releases/tag/v9.22.0
+[9.21.0]: https://github.com/gchq/CyberChef/releases/tag/v9.21.0
+[9.20.0]: https://github.com/gchq/CyberChef/releases/tag/v9.20.0
+[9.19.0]: https://github.com/gchq/CyberChef/releases/tag/v9.19.0
+[9.18.0]: https://github.com/gchq/CyberChef/releases/tag/v9.18.0
+[9.17.0]: https://github.com/gchq/CyberChef/releases/tag/v9.17.0
+[9.16.0]: https://github.com/gchq/CyberChef/releases/tag/v9.16.0
+[9.15.0]: https://github.com/gchq/CyberChef/releases/tag/v9.15.0
+[9.14.0]: https://github.com/gchq/CyberChef/releases/tag/v9.14.0
+[9.13.0]: https://github.com/gchq/CyberChef/releases/tag/v9.13.0
+[9.12.0]: https://github.com/gchq/CyberChef/releases/tag/v9.12.0
+[9.11.0]: https://github.com/gchq/CyberChef/releases/tag/v9.11.0
+[9.10.0]: https://github.com/gchq/CyberChef/releases/tag/v9.10.0
+[9.9.0]: https://github.com/gchq/CyberChef/releases/tag/v9.9.0
+[9.8.0]: https://github.com/gchq/CyberChef/releases/tag/v9.8.0
+[9.7.0]: https://github.com/gchq/CyberChef/releases/tag/v9.7.0
+[9.6.0]: https://github.com/gchq/CyberChef/releases/tag/v9.6.0
+[9.5.0]: https://github.com/gchq/CyberChef/releases/tag/v9.5.0
+[9.4.0]: https://github.com/gchq/CyberChef/releases/tag/v9.4.0
+[9.3.0]: https://github.com/gchq/CyberChef/releases/tag/v9.3.0
+[9.2.0]: https://github.com/gchq/CyberChef/releases/tag/v9.2.0
+[9.1.0]: https://github.com/gchq/CyberChef/releases/tag/v9.1.0
+[9.0.0]: https://github.com/gchq/CyberChef/releases/tag/v9.0.0
+[8.38.0]: https://github.com/gchq/CyberChef/releases/tag/v8.38.0
+[8.37.0]: https://github.com/gchq/CyberChef/releases/tag/v8.37.0
+[8.36.0]: https://github.com/gchq/CyberChef/releases/tag/v8.36.0
+[8.35.0]: https://github.com/gchq/CyberChef/releases/tag/v8.35.0
+[8.34.0]: https://github.com/gchq/CyberChef/releases/tag/v8.34.0
+[8.33.0]: https://github.com/gchq/CyberChef/releases/tag/v8.33.0
+[8.32.0]: https://github.com/gchq/CyberChef/releases/tag/v8.32.0
+[8.31.0]: https://github.com/gchq/CyberChef/releases/tag/v8.31.0
+[8.30.0]: https://github.com/gchq/CyberChef/releases/tag/v8.30.0
+[8.29.0]: https://github.com/gchq/CyberChef/releases/tag/v8.29.0
+[8.28.0]: https://github.com/gchq/CyberChef/releases/tag/v8.28.0
+[8.27.0]: https://github.com/gchq/CyberChef/releases/tag/v8.27.0
+[8.26.0]: https://github.com/gchq/CyberChef/releases/tag/v8.26.0
+[8.25.0]: https://github.com/gchq/CyberChef/releases/tag/v8.25.0
 [8.24.0]: https://github.com/gchq/CyberChef/releases/tag/v8.24.0
 [8.23.1]: https://github.com/gchq/CyberChef/releases/tag/v8.23.1
 [8.23.0]: https://github.com/gchq/CyberChef/releases/tag/v8.23.0
@@ -140,9 +499,12 @@ All major and minor version changes will be documented in this file. Details of
 [@n1474335]: https://github.com/n1474335
 [@d98762625]: https://github.com/d98762625
 [@j433866]: https://github.com/j433866
+[@n1073645]: https://github.com/n1073645
 [@GCHQ77703]: https://github.com/GCHQ77703
 [@h345983745]: https://github.com/h345983745
+[@s2224834]: https://github.com/s2224834
 [@artemisbot]: https://github.com/artemisbot
+[@tlwr]: https://github.com/tlwr
 [@picapi]: https://github.com/picapi
 [@Dachande663]: https://github.com/Dachande663
 [@JustAnotherMark]: https://github.com/JustAnotherMark
@@ -156,9 +518,53 @@ All major and minor version changes will be documented in this file. Details of
 [@Cynser]: https://github.com/Cynser
 [@anthony-arnold]: https://github.com/anthony-arnold
 [@masq]: https://github.com/masq
+[@Ge0rg3]: https://github.com/Ge0rg3
+[@MShwed]: https://github.com/MShwed
+[@kassi]: https://github.com/kassi
+[@jarrodconnolly]: https://github.com/jarrodconnolly
+[@VirtualColossus]: https://github.com/VirtualColossus
+[@cbeuw]: https://github.com/cbeuw
+[@matthieuxyz]: https://github.com/matthieuxyz
+[@Flavsditz]: https://github.com/Flavsditz
+[@pointhi]: https://github.com/pointhi
+[@MarvinJWendt]: https://github.com/MarvinJWendt
+[@dmfj]: https://github.com/dmfj
+[@mattnotmitt]: https://github.com/mattnotmitt
+[@Danh4]: https://github.com/Danh4
+[@john19696]: https://github.com/john19696
+[@t-8ch]: https://github.com/t-8ch
+[@hettysymes]: https://github.com/hettysymes
+[@swesven]: https://github.com/swesven
+[@mikecat]: https://github.com/mikecat
+[@crespyl]: https://github.com/crespyl
+[@thomasleplus]: https://github.com/thomasleplus
+[@valdelaseras]: https://github.com/valdelaseras
+[@brun0ne]: https://github.com/brun0ne
+[@joostrijneveld]: https://github.com/joostrijneveld
+[@Xenonym]: https://github.com/Xenonym
+[@gchq77703]: https://github.com/gchq77703
+[@a3957273]: https://github.com/a3957273
+[@0xThiebaut]: https://github.com/0xThiebaut
+[@cnotin]: https://github.com/cnotin
+[@KevinSJ]: https://github.com/KevinSJ
+[@sw5678]: https://github.com/sw5678
+[@sg5506844]: https://github.com/sg5506844
+[@AliceGrey]: https://github.com/AliceGrey
+[@AshCorr]: https://github.com/AshCorr
+
+
+[8ad18b]: https://github.com/gchq/CyberChef/commit/8ad18bc7db6d9ff184ba3518686293a7685bf7b7
+[9a33498]: https://github.com/gchq/CyberChef/commit/9a33498fed26a8df9c9f35f39a78a174bf50a513
+[289a417]: https://github.com/gchq/CyberChef/commit/289a417dfb5923de5e1694354ec42a08d9395bfe
+[e9ca4dc]: https://github.com/gchq/CyberChef/commit/e9ca4dc9caf98f33fd986431cd400c88082a42b8
+[dd18e52]: https://github.com/gchq/CyberChef/commit/dd18e529939078b89867297b181a584e8b2cc7da
+[a895d1d]: https://github.com/gchq/CyberChef/commit/a895d1d82a2f92d440a0c5eca2bc7c898107b737
+[31a7f83]: https://github.com/gchq/CyberChef/commit/31a7f83b82e78927f89689f323fcb9185144d6ff
+[760eff4]: https://github.com/gchq/CyberChef/commit/760eff49b5307aaa3104c5e5b437ffe62299acd1
 
 [#95]: https://github.com/gchq/CyberChef/pull/299
 [#173]: https://github.com/gchq/CyberChef/pull/173
+[#143]: https://github.com/gchq/CyberChef/pull/143
 [#224]: https://github.com/gchq/CyberChef/pull/224
 [#239]: https://github.com/gchq/CyberChef/pull/239
 [#248]: https://github.com/gchq/CyberChef/pull/248
@@ -166,6 +572,7 @@ All major and minor version changes will be documented in this file. Details of
 [#277]: https://github.com/gchq/CyberChef/issues/277
 [#281]: https://github.com/gchq/CyberChef/pull/281
 [#284]: https://github.com/gchq/CyberChef/pull/284
+[#291]: https://github.com/gchq/CyberChef/pull/291
 [#294]: https://github.com/gchq/CyberChef/pull/294
 [#296]: https://github.com/gchq/CyberChef/pull/296
 [#298]: https://github.com/gchq/CyberChef/pull/298
@@ -180,6 +587,7 @@ All major and minor version changes will be documented in this file. Details of
 [#394]: https://github.com/gchq/CyberChef/pull/394
 [#428]: https://github.com/gchq/CyberChef/pull/428
 [#439]: https://github.com/gchq/CyberChef/pull/439
+[#440]: https://github.com/gchq/CyberChef/pull/440
 [#441]: https://github.com/gchq/CyberChef/pull/441
 [#443]: https://github.com/gchq/CyberChef/pull/443
 [#446]: https://github.com/gchq/CyberChef/pull/446
@@ -191,4 +599,82 @@ All major and minor version changes will be documented in this file. Details of
 [#467]: https://github.com/gchq/CyberChef/pull/467
 [#468]: https://github.com/gchq/CyberChef/pull/468
 [#476]: https://github.com/gchq/CyberChef/pull/476
+[#477]: https://github.com/gchq/CyberChef/pull/477
 [#489]: https://github.com/gchq/CyberChef/pull/489
+[#496]: https://github.com/gchq/CyberChef/pull/496
+[#500]: https://github.com/gchq/CyberChef/pull/500
+[#506]: https://github.com/gchq/CyberChef/pull/506
+[#515]: https://github.com/gchq/CyberChef/pull/515
+[#516]: https://github.com/gchq/CyberChef/pull/516
+[#525]: https://github.com/gchq/CyberChef/pull/525
+[#528]: https://github.com/gchq/CyberChef/pull/528
+[#530]: https://github.com/gchq/CyberChef/pull/530
+[#531]: https://github.com/gchq/CyberChef/pull/531
+[#533]: https://github.com/gchq/CyberChef/pull/533
+[#535]: https://github.com/gchq/CyberChef/pull/535
+[#556]: https://github.com/gchq/CyberChef/pull/556
+[#566]: https://github.com/gchq/CyberChef/pull/566
+[#571]: https://github.com/gchq/CyberChef/pull/571
+[#585]: https://github.com/gchq/CyberChef/pull/585
+[#591]: https://github.com/gchq/CyberChef/pull/591
+[#595]: https://github.com/gchq/CyberChef/pull/595
+[#614]: https://github.com/gchq/CyberChef/pull/614
+[#625]: https://github.com/gchq/CyberChef/pull/625
+[#627]: https://github.com/gchq/CyberChef/pull/627
+[#632]: https://github.com/gchq/CyberChef/pull/632
+[#652]: https://github.com/gchq/CyberChef/pull/652
+[#653]: https://github.com/gchq/CyberChef/pull/653
+[#674]: https://github.com/gchq/CyberChef/pull/674
+[#683]: https://github.com/gchq/CyberChef/pull/683
+[#865]: https://github.com/gchq/CyberChef/pull/865
+[#906]: https://github.com/gchq/CyberChef/pull/906
+[#912]: https://github.com/gchq/CyberChef/pull/912
+[#917]: https://github.com/gchq/CyberChef/pull/917
+[#934]: https://github.com/gchq/CyberChef/pull/934
+[#948]: https://github.com/gchq/CyberChef/pull/948
+[#951]: https://github.com/gchq/CyberChef/pull/951
+[#952]: https://github.com/gchq/CyberChef/pull/952
+[#965]: https://github.com/gchq/CyberChef/pull/965
+[#966]: https://github.com/gchq/CyberChef/pull/966
+[#987]: https://github.com/gchq/CyberChef/pull/987
+[#999]: https://github.com/gchq/CyberChef/pull/999
+[#1006]: https://github.com/gchq/CyberChef/pull/1006
+[#1022]: https://github.com/gchq/CyberChef/pull/1022
+[#1037]: https://github.com/gchq/CyberChef/pull/1037
+[#1045]: https://github.com/gchq/CyberChef/pull/1045
+[#1049]: https://github.com/gchq/CyberChef/pull/1049
+[#1065]: https://github.com/gchq/CyberChef/pull/1065
+[#1066]: https://github.com/gchq/CyberChef/pull/1066
+[#1083]: https://github.com/gchq/CyberChef/pull/1083
+[#1189]: https://github.com/gchq/CyberChef/pull/1189
+[#1242]: https://github.com/gchq/CyberChef/pull/1242
+[#1244]: https://github.com/gchq/CyberChef/pull/1244
+[#1313]: https://github.com/gchq/CyberChef/pull/1313
+[#1326]: https://github.com/gchq/CyberChef/pull/1326
+[#1364]: https://github.com/gchq/CyberChef/pull/1364
+[#1264]: https://github.com/gchq/CyberChef/pull/1264
+[#1266]: https://github.com/gchq/CyberChef/pull/1266
+[#1250]: https://github.com/gchq/CyberChef/pull/1250
+[#1308]: https://github.com/gchq/CyberChef/pull/1308
+[#1405]: https://github.com/gchq/CyberChef/pull/1405
+[#1421]: https://github.com/gchq/CyberChef/pull/1421
+[#1427]: https://github.com/gchq/CyberChef/pull/1427
+[#1472]: https://github.com/gchq/CyberChef/pull/1472
+[#1457]: https://github.com/gchq/CyberChef/pull/1457
+[#1466]: https://github.com/gchq/CyberChef/pull/1466
+[#1456]: https://github.com/gchq/CyberChef/pull/1456
+[#1450]: https://github.com/gchq/CyberChef/pull/1450
+[#1498]: https://github.com/gchq/CyberChef/pull/1498
+[#1499]: https://github.com/gchq/CyberChef/pull/1499
+[#1528]: https://github.com/gchq/CyberChef/pull/1528
+[#661]: https://github.com/gchq/CyberChef/pull/661
+[#493]: https://github.com/gchq/CyberChef/pull/493
+[#592]: https://github.com/gchq/CyberChef/issues/592
+[#1703]: https://github.com/gchq/CyberChef/issues/1703
+[#1675]: https://github.com/gchq/CyberChef/issues/1675
+[#1678]: https://github.com/gchq/CyberChef/issues/1678
+[#1541]: https://github.com/gchq/CyberChef/issues/1541
+[#1667]: https://github.com/gchq/CyberChef/issues/1667
+[#1555]: https://github.com/gchq/CyberChef/issues/1555
+[#1694]: https://github.com/gchq/CyberChef/issues/1694
+[#1699]: https://github.com/gchq/CyberChef/issues/1694

Dockerfile

@@ -0,0 +1,9 @@
+FROM node:18-alpine AS build
+
+COPY . .
+RUN npm ci
+RUN npm run build
+
+FROM nginx:1.25-alpine3.18 AS cyberchef
+
+COPY --from=build ./build/prod /usr/share/nginx/html/

Gruntfile.js

@@ -3,11 +3,11 @@
 const webpack = require("webpack");
 const HtmlWebpackPlugin = require("html-webpack-plugin");
 const BundleAnalyzerPlugin = require("webpack-bundle-analyzer").BundleAnalyzerPlugin;
-const NodeExternals = require("webpack-node-externals");
-const Inliner = require("web-resource-inliner");
 const glob = require("glob");
 const path = require("path");
 
+const nodeFlags = "--experimental-modules --experimental-json-modules --experimental-specifier-resolution=node --no-warnings --no-deprecation";
+
 /**
  * Grunt configuration for building the app in various formats.
  *
@@ -25,53 +25,64 @@ module.exports = function (grunt) {
         "A persistent task which creates a development build whenever source files are modified.",
         ["clean:dev", "clean:config", "exec:generateConfig", "concurrent:dev"]);
 
+    grunt.registerTask("prod",
+        "Creates a production-ready build. Use the --msg flag to add a compile message.",
+        [
+            "eslint", "clean:prod", "clean:config", "exec:generateConfig", "findModules", "webpack:web",
+            "copy:standalone", "zip:standalone", "clean:standalone", "exec:calcDownloadHash", "chmod"
+        ]);
+
     grunt.registerTask("node",
         "Compiles CyberChef into a single NodeJS module.",
-        ["clean:node", "clean:config", "exec:generateConfig", "webpack:node", "chmod:build"]);
+        [
+            "clean:node", "clean:config", "clean:nodeConfig", "exec:generateConfig", "exec:generateNodeIndex"
+        ]);
 
-    grunt.registerTask("test",
-        "A task which runs all the operation tests in the tests directory.",
-        ["exec:generateConfig", "exec:opTests"]);
+    grunt.registerTask("configTests",
+        "A task which configures config files in preparation for tests to be run. Use `npm test` to run tests.",
+        [
+            "clean:config", "clean:nodeConfig", "exec:generateConfig", "exec:generateNodeIndex"
+        ]);
 
     grunt.registerTask("testui",
         "A task which runs all the UI tests in the tests directory. The prod task must already have been run.",
         ["connect:prod", "exec:browserTests"]);
 
-    grunt.registerTask("docs",
-        "Compiles documentation in the /docs directory.",
-        ["clean:docs", "jsdoc", "chmod:docs"]);
-
-    grunt.registerTask("prod",
-        "Creates a production-ready build. Use the --msg flag to add a compile message.",
-        ["eslint", "clean:prod", "clean:config", "exec:generateConfig", "webpack:web", "inline", "chmod"]);
+    grunt.registerTask("testnodeconsumer",
+        "A task which checks whether consuming CJS and ESM apps work with the CyberChef build",
+        ["exec:setupNodeConsumers", "exec:testCJSNodeConsumer", "exec:testESMNodeConsumer", "exec:teardownNodeConsumers"]);
 
     grunt.registerTask("default",
         "Lints the code base",
         ["eslint", "exec:repoSize"]);
 
-    grunt.registerTask("inline",
-        "Compiles a production build of CyberChef into a single, portable web page.",
-        ["exec:generateConfig", "webpack:webInline", "runInliner", "clean:inlineScripts"]);
+    grunt.registerTask("lint", "eslint");
+
+    grunt.registerTask("findModules",
+        "Finds all generated modules and updates the entry point list for Webpack",
+        function(arg1, arg2) {
+            const moduleEntryPoints = listEntryModules();
 
+            grunt.log.writeln(`Found ${Object.keys(moduleEntryPoints).length} modules.`);
 
-    grunt.registerTask("runInliner", runInliner);
-    grunt.registerTask("doc", "docs");
-    grunt.registerTask("tests", "test");
-    grunt.registerTask("lint", "eslint");
+            grunt.config.set("webpack.web.entry",
+                Object.assign({
+                    main: "./src/web/index.js"
+                }, moduleEntryPoints));
+        });
 
 
     // Load tasks provided by each plugin
     grunt.loadNpmTasks("grunt-eslint");
     grunt.loadNpmTasks("grunt-webpack");
-    grunt.loadNpmTasks("grunt-jsdoc");
     grunt.loadNpmTasks("grunt-contrib-clean");
     grunt.loadNpmTasks("grunt-contrib-copy");
     grunt.loadNpmTasks("grunt-contrib-watch");
     grunt.loadNpmTasks("grunt-chmod");
     grunt.loadNpmTasks("grunt-exec");
-    grunt.loadNpmTasks("grunt-accessibility");
     grunt.loadNpmTasks("grunt-concurrent");
     grunt.loadNpmTasks("grunt-contrib-connect");
+    grunt.loadNpmTasks("grunt-zip");
 
 
     // Project configuration
@@ -82,44 +93,56 @@ module.exports = function (grunt) {
             COMPILE_TIME: JSON.stringify(compileTime),
             COMPILE_MSG: JSON.stringify(grunt.option("compile-msg") || grunt.option("msg") || ""),
             PKG_VERSION: JSON.stringify(pkg.version),
-            ENVIRONMENT_IS_WORKER: function() {
-                return typeof importScripts === "function";
-            },
-            ENVIRONMENT_IS_NODE: function() {
-                return typeof process === "object" && typeof require === "function";
-            },
-            ENVIRONMENT_IS_WEB: function() {
-                return typeof window === "object";
-            }
         },
-        moduleEntryPoints = listEntryModules();
+        moduleEntryPoints = listEntryModules(),
+        nodeConsumerTestPath = "~/tmp-cyberchef",
+        /**
+         * Configuration for Webpack production build. Defined as a function so that it
+         * can be recalculated when new modules are generated.
+         */
+        webpackProdConf = () => {
+            return {
+                mode: "production",
+                target: "web",
+                entry: Object.assign({
+                    main: "./src/web/index.js"
+                }, moduleEntryPoints),
+                output: {
+                    path: __dirname + "/build/prod",
+                    filename: chunkData => {
+                        return chunkData.chunk.name === "main" ? "assets/[name].js": "[name].js";
+                    },
+                    globalObject: "this"
+                },
+                resolve: {
+                    alias: {
+                        "./config/modules/OpModules.mjs": "./config/modules/Default.mjs"
+                    }
+                },
+                plugins: [
+                    new webpack.DefinePlugin(BUILD_CONSTANTS),
+                    new HtmlWebpackPlugin({
+                        filename: "index.html",
+                        template: "./src/web/html/index.html",
+                        chunks: ["main"],
+                        compileTime: compileTime,
+                        version: pkg.version,
+                        minify: {
+                            removeComments: true,
+                            collapseWhitespace: true,
+                            minifyJS: true,
+                            minifyCSS: true
+                        }
+                    }),
+                    new BundleAnalyzerPlugin({
+                        analyzerMode: "static",
+                        reportFilename: "BundleAnalyzerReport.html",
+                        openAnalyzer: false
+                    }),
+                ]
+            };
+        };
 
-    /**
-     * Compiles a production build of CyberChef into a single, portable web page.
-     */
-    function runInliner() {
-        const done = this.async();
-        Inliner.html({
-            relativeTo: "build/prod/",
-            fileContent: grunt.file.read("build/prod/cyberchef.htm"),
-            images: true,
-            svgs: true,
-            scripts: true,
-            links: true,
-            strict: true
-        }, function(error, result) {
-            if (error) {
-                if (error instanceof Error) {
-                    done(error);
-                } else {
-                    done(new Error(error));
-                }
-            } else {
-                grunt.file.write("build/prod/cyberchef.htm", result);
-                done(true);
-            }
-        });
-    }
 
     /**
      * Generates an entry list for all the modules.
@@ -130,196 +153,101 @@ module.exports = function (grunt) {
         glob.sync("./src/core/config/modules/*.mjs").forEach(file => {
             const basename = path.basename(file);
             if (basename !== "Default.mjs" && basename !== "OpModules.mjs")
-                entryModules[basename.split(".mjs")[0]] = path.resolve(file);
+                entryModules["modules/" + basename.split(".mjs")[0]] = path.resolve(file);
         });
 
         return entryModules;
     }
 
+    /**
+     * Detects the correct delimiter to use to chain shell commands together
+     * based on the current OS.
+     *
+     * @param {string[]} cmds
+     * @returns {string}
+     */
+    function chainCommands(cmds) {
+        const win = process.platform === "win32";
+        if (!win) {
+            return cmds.join(";");
+        }
+        return cmds
+            // && means that subsequent commands will not be executed if the
+            // previous one fails. & would coninue on a fail
+            .join("&&")
+            // Windows does not support \n properly
+            .replace(/\n/g, "\\n");
+    }
+
     grunt.initConfig({
         clean: {
             dev: ["build/dev/*"],
             prod: ["build/prod/*"],
             node: ["build/node/*"],
             config: ["src/core/config/OperationConfig.json", "src/core/config/modules/*", "src/code/operations/index.mjs"],
-            docs: ["docs/*", "!docs/*.conf.json", "!docs/*.ico", "!docs/*.png"],
-            inlineScripts: ["build/prod/scripts.js"],
+            nodeConfig: ["src/node/index.mjs", "src/node/config/OperationConfig.json"],
+            standalone: ["build/prod/CyberChef*.html"]
         },
         eslint: {
-            options: {
-                configFile: "./.eslintrc.json"
-            },
             configs: ["*.{js,mjs}"],
             core: ["src/core/**/*.{js,mjs}", "!src/core/vendor/**/*", "!src/core/operations/legacy/**/*"],
-            web: ["src/web/**/*.{js,mjs}"],
+            web: ["src/web/**/*.{js,mjs}", "!src/web/static/**/*"],
             node: ["src/node/**/*.{js,mjs}"],
             tests: ["tests/**/*.{js,mjs}"],
         },
-        jsdoc: {
-            options: {
-                destination: "docs",
-                template: "node_modules/ink-docstrap/template",
-                recurse: true,
-                readme: "./README.md",
-                configure: "docs/jsdoc.conf.json"
-            },
-            all: {
-                src: [
-                    "src/**/*.js",
-                    "src/**/*.mjs",
-                    "!src/core/vendor/**/*"
-                ],
-            }
-        },
-        accessibility: {
-            options: {
-                accessibilityLevel: "WCAG2A",
-                verbose: false,
-                ignore: [
-                    "WCAG2A.Principle1.Guideline1_3.1_3_1.H42.2"
-                ]
-            },
-            test: {
-                src: ["build/**/*.html"]
-            }
-        },
         webpack: {
             options: webpackConfig,
-            web: () => {
-                return {
-                    mode: "production",
-                    target: "web",
-                    entry: Object.assign({
-                        main: "./src/web/index.js",
-                        sitemap: "./src/web/static/sitemap.js"
-                    }, moduleEntryPoints),
-                    output: {
-                        path: __dirname + "/build/prod",
-                        globalObject: "this"
-                    },
-                    resolve: {
-                        alias: {
-                            "./config/modules/OpModules": "./config/modules/Default"
-                        }
-                    },
-                    plugins: [
-                        new webpack.DefinePlugin(BUILD_CONSTANTS),
-                        new HtmlWebpackPlugin({
-                            filename: "index.html",
-                            template: "./src/web/html/index.html",
-                            chunks: ["main"],
-                            compileTime: compileTime,
-                            version: pkg.version,
-                            minify: {
-                                removeComments: true,
-                                collapseWhitespace: true,
-                                minifyJS: true,
-                                minifyCSS: true
-                            }
-                        }),
-                        new BundleAnalyzerPlugin({
-                            analyzerMode: "static",
-                            reportFilename: "BundleAnalyzerReport.html",
-                            openAnalyzer: false
-                        }),
-                    ]
-                };
-            },
-            webInline: {
-                mode: "production",
+            myConfig: webpackConfig,
+            web: webpackProdConf(),
+        },
+        "webpack-dev-server": {
+            options: webpackConfig,
+            start: {
+                mode: "development",
                 target: "web",
-                entry: "./src/web/index.js",
-                output: {
-                    filename: "scripts.js",
-                    path: __dirname + "/build/prod"
+                entry: Object.assign({
+                    main: "./src/web/index.js"
+                }, moduleEntryPoints),
+                resolve: {
+                    alias: {
+                        "./config/modules/OpModules.mjs": "./config/modules/Default.mjs"
+                    }
+                },
+                devServer: {
+                    port: grunt.option("port") || 8080,
+                    client: {
+                        logging: "error",
+                        overlay: true
+                    },
+                    hot: "only"
                 },
                 plugins: [
-                    new webpack.DefinePlugin(Object.assign({}, BUILD_CONSTANTS, {
-                        INLINE: "true"
-                    })),
+                    new webpack.DefinePlugin(BUILD_CONSTANTS),
                     new HtmlWebpackPlugin({
-                        filename: "cyberchef.htm",
+                        filename: "index.html",
                         template: "./src/web/html/index.html",
+                        chunks: ["main"],
                         compileTime: compileTime,
-                        version: pkg.version + "s",
-                        inline: true,
-                        minify: {
-                            removeComments: true,
-                            collapseWhitespace: true,
-                            minifyJS: true,
-                            minifyCSS: true
-                        }
-                    }),
-                ]
-            },
-            node: {
-                mode: "production",
-                target: "node",
-                entry: "./src/node/index.mjs",
-                externals: [NodeExternals()],
-                output: {
-                    filename: "CyberChef.js",
-                    path: __dirname + "/build/node",
-                    library: "CyberChef",
-                    libraryTarget: "commonjs2"
-                },
-                plugins: [
-                    new webpack.DefinePlugin(BUILD_CONSTANTS)
+                        version: pkg.version,
+                    })
                 ]
             }
         },
-        "webpack-dev-server": {
-            options: {
-                webpack: webpackConfig,
-                host: "0.0.0.0",
-                disableHostCheck: true,
-                overlay: true,
-                inline: false,
-                clientLogLevel: "error",
-                stats: {
-                    children: false,
-                    chunks: false,
-                    modules: false,
-                    entrypoints: false,
-                    warningsFilter: [
-                        /source-map/,
-                        /dependency is an expression/,
-                        /export 'default'/
-                    ],
-                }
-            },
-            start: {
-                webpack: {
-                    mode: "development",
-                    target: "web",
-                    entry: Object.assign({
-                        main: "./src/web/index.js"
-                    }, moduleEntryPoints),
-                    resolve: {
-                        alias: {
-                            "./config/modules/OpModules": "./config/modules/Default"
-                        }
-                    },
-                    output: {
-                        globalObject: "this",
-                    },
-                    plugins: [
-                        new webpack.DefinePlugin(BUILD_CONSTANTS),
-                        new HtmlWebpackPlugin({
-                            filename: "index.html",
-                            template: "./src/web/html/index.html",
-                            chunks: ["main"],
-                            compileTime: compileTime,
-                            version: pkg.version,
-                        })
-                    ]
-                }
+        zip: {
+            standalone: {
+                cwd: "build/prod/",
+                src: [
+                    "build/prod/**/*",
+                    "!build/prod/index.html",
+                    "!build/prod/BundleAnalyzerReport.html",
+                ],
+                dest: `build/prod/CyberChef_v${pkg.version}.zip`
             }
         },
         connect: {
             prod: {
                 options: {
-                    port: 8000,
+                    port: grunt.option("port") || 8000,
                     base: "build/prod/"
                 }
             }
@@ -328,10 +256,16 @@ module.exports = function (grunt) {
             ghPages: {
                 options: {
                     process: function (content, srcpath) {
-                        // Add Google Analytics code to index.html
                         if (srcpath.indexOf("index.html") >= 0) {
+                            // Add Google Analytics code to index.html
                             content = content.replace("</body></html>",
                                 grunt.file.read("src/web/static/ga.html") + "</body></html>");
+
+                            // Add Structured Data for SEO
+                            content = content.replace("</head>",
+                                "<script type='application/ld+json'>" +
+                                JSON.stringify(JSON.parse(grunt.file.read("src/web/static/structuredData.json"))) +
+                                "</script></head>");
                             return grunt.template.process(content, srcpath);
                         } else {
                             return content;
@@ -341,14 +275,31 @@ module.exports = function (grunt) {
                 },
                 files: [
                     {
-                        src: "build/prod/index.html",
+                        src: ["build/prod/index.html"],
                         dest: "build/prod/index.html"
+                    }
+                ]
+            },
+            standalone: {
+                options: {
+                    process: function (content, srcpath) {
+                        if (srcpath.indexOf("index.html") >= 0) {
+                            // Replace download link with version number
+                            content = content.replace(/<a [^>]+>Download CyberChef.+?<\/a>/,
+                                `<span>Version ${pkg.version}</span>`);
+
+                            return grunt.template.process(content, srcpath);
+                        } else {
+                            return content;
+                        }
                     },
+                    noProcess: ["**", "!**/*.html"]
+                },
+                files: [
                     {
-                        expand: true,
-                        src: "docs/**",
-                        dest: "build/prod/"
-                    },
+                        src: ["build/prod/index.html"],
+                        dest: `build/prod/CyberChef_v${pkg.version}.html`
+                    }
                 ]
             }
         },
@@ -358,18 +309,12 @@ module.exports = function (grunt) {
                     mode: "755",
                 },
                 src: ["build/**/*", "build/"]
-            },
-            docs: {
-                options: {
-                    mode: "755",
-                },
-                src: ["docs/**/*", "docs/"]
             }
         },
         watch: {
             config: {
                 files: ["src/core/operations/**/*", "!src/core/operations/index.mjs"],
-                tasks: ["exec:generateConfig"]
+                tasks: ["exec:generateNodeIndex", "exec:generateConfig"]
             }
         },
         concurrent: {
@@ -379,33 +324,109 @@ module.exports = function (grunt) {
             }
         },
         exec: {
+            calcDownloadHash: {
+                command: function () {
+                    switch (process.platform) {
+                        case "darwin":
+                            return chainCommands([
+                                `shasum -a 256 build/prod/CyberChef_v${pkg.version}.zip | awk '{print $1;}' > build/prod/sha256digest.txt`,
+                                `sed -i '' -e "s/DOWNLOAD_HASH_PLACEHOLDER/$(cat build/prod/sha256digest.txt)/" build/prod/index.html`
+                            ]);
+                        default:
+                            return chainCommands([
+                                `sha256sum build/prod/CyberChef_v${pkg.version}.zip | awk '{print $1;}' > build/prod/sha256digest.txt`,
+                                `sed -i -e "s/DOWNLOAD_HASH_PLACEHOLDER/$(cat build/prod/sha256digest.txt)/" build/prod/index.html`
+                            ]);
+                    }
+                },
+            },
             repoSize: {
-                command: [
+                command: chainCommands([
                     "git ls-files | wc -l | xargs printf '\n%b\ttracked files\n'",
                     "du -hs | egrep -o '^[^\t]*' | xargs printf '%b\trepository size\n'"
-                ].join(";"),
+                ]),
                 stderr: false
             },
             cleanGit: {
                 command: "git gc --prune=now --aggressive"
             },
             sitemap: {
-                command: "node build/prod/sitemap.js > build/prod/sitemap.xml"
+                command: `node ${nodeFlags} src/web/static/sitemap.mjs > build/prod/sitemap.xml`,
+                sync: true
             },
             generateConfig: {
-                command: [
+                command: chainCommands([
                     "echo '\n--- Regenerating config files. ---'",
                     "echo [] > src/core/config/OperationConfig.json",
-                    "node --experimental-modules --no-warnings --no-deprecation src/core/config/scripts/generateOpsIndex.mjs",
-                    "node --experimental-modules --no-warnings --no-deprecation src/core/config/scripts/generateConfig.mjs",
+                    `node ${nodeFlags} src/core/config/scripts/generateOpsIndex.mjs`,
+                    `node ${nodeFlags} src/core/config/scripts/generateConfig.mjs`,
                     "echo '--- Config scripts finished. ---\n'"
-                ].join(";")
+                ]),
+                sync: true
             },
-            opTests: {
-                command: "node --experimental-modules --no-warnings --no-deprecation tests/operations/index.mjs"
+            generateNodeIndex: {
+                command: chainCommands([
+                    "echo '\n--- Regenerating node index ---'",
+                    `node ${nodeFlags} src/node/config/scripts/generateNodeIndex.mjs`,
+                    "echo '--- Node index generated. ---\n'"
+                ]),
+                sync: true
             },
             browserTests: {
-                command: "./node_modules/.bin/nightwatch --env prod,inline"
+                command: "./node_modules/.bin/nightwatch --env prod"
+            },
+            setupNodeConsumers: {
+                command: chainCommands([
+                    "echo '\n--- Testing node consumers ---'",
+                    "npm link",
+                    `mkdir ${nodeConsumerTestPath}`,
+                    `cp tests/node/consumers/* ${nodeConsumerTestPath}`,
+                    `cd ${nodeConsumerTestPath}`,
+                    "npm link cyberchef"
+                ]),
+                sync: true
+            },
+            teardownNodeConsumers: {
+                command: chainCommands([
+                    `rm -rf ${nodeConsumerTestPath}`,
+                    "echo '\n--- Node consumer tests complete ---'"
+                ]),
+            },
+            testCJSNodeConsumer: {
+                command: chainCommands([
+                    `cd ${nodeConsumerTestPath}`,
+                    `node ${nodeFlags} cjs-consumer.js`,
+                ]),
+                stdout: false,
+            },
+            testESMNodeConsumer: {
+                command: chainCommands([
+                    `cd ${nodeConsumerTestPath}`,
+                    `node ${nodeFlags} esm-consumer.mjs`,
+                ]),
+                stdout: false,
+            },
+            fixCryptoApiImports: {
+                command: function () {
+                    switch (process.platform) {
+                        case "darwin":
+                            return `find ./node_modules/crypto-api/src/ \\( -type d -name .git -prune \\) -o -type f -print0 | xargs -0 sed -i '' -e '/\\.mjs/!s/\\(from "\\.[^"]*\\)";/\\1.mjs";/g'`;
+                        default:
+                            return `find ./node_modules/crypto-api/src/ \\( -type d -name .git -prune \\) -o -type f -print0 | xargs -0 sed -i -e '/\\.mjs/!s/\\(from "\\.[^"]*\\)";/\\1.mjs";/g'`;
+                    }
+                },
+                stdout: false
+            },
+            fixSnackbarMarkup: {
+                command: function () {
+                    switch (process.platform) {
+                        case "darwin":
+                            return `sed -i '' 's/<div id=snackbar-container\\/>/<div id=snackbar-container>/g' ./node_modules/snackbarjs/src/snackbar.js`;
+                        default:
+                            return `sed -i 's/<div id=snackbar-container\\/>/<div id=snackbar-container>/g' ./node_modules/snackbarjs/src/snackbar.js`;
+                    }
+                },
+                stdout: false
             }
         },
     });

README.md

@@ -1,16 +1,14 @@
 # CyberChef
 
-[![Build Status](https://travis-ci.org/gchq/CyberChef.svg?branch=master)](https://travis-ci.org/gchq/CyberChef)
-[![dependencies Status](https://david-dm.org/gchq/CyberChef/status.svg)](https://david-dm.org/gchq/CyberChef)
+[![](https://github.com/gchq/CyberChef/workflows/Master%20Build,%20Test%20&%20Deploy/badge.svg)](https://github.com/gchq/CyberChef/actions?query=workflow%3A%22Master+Build%2C+Test+%26+Deploy%22)
 [![npm](https://img.shields.io/npm/v/cyberchef.svg)](https://www.npmjs.com/package/cyberchef)
-![](https://reposs.herokuapp.com/?path=gchq/CyberChef&color=blue)
 [![](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/gchq/CyberChef/blob/master/LICENSE)
 [![Gitter](https://badges.gitter.im/gchq/CyberChef.svg)](https://gitter.im/gchq/CyberChef?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge)
 
 
 #### *The Cyber Swiss Army Knife*
 
-CyberChef is a simple, intuitive web app for carrying out all manner of "cyber" operations within a web browser. These operations include simple encoding like XOR or Base64, more complex encryption like AES, DES and Blowfish, creating binary and hexdumps, compression and decompression of data, calculating hashes and checksums, IPv6 and X.509 parsing, changing character encodings, and much more.
+CyberChef is a simple, intuitive web app for carrying out all manner of "cyber" operations within a web browser. These operations include simple encoding like XOR and Base64, more complex encryption like AES, DES and Blowfish, creating binary and hexdumps, compression and decompression of data, calculating hashes and checksums, IPv6 and X.509 parsing, changing character encodings, and much more.
 
 The tool is designed to enable both technical and non-technical analysts to manipulate data in complex ways without having to deal with complex tools or algorithms. It was conceived, designed, built and incrementally improved by an analyst in their 10% innovation time over several years.
 
@@ -22,6 +20,22 @@ Cryptographic operations in CyberChef should not be relied upon to provide secur
 
 [A live demo can be found here][1] - have fun!
 
+## Containers
+
+If you would like to try out CyberChef locally you can either build it yourself:
+
+```bash
+docker build --tag cyberchef --ulimit nofile=10000 .
+docker run -it -p 8080:80 cyberchef
+```
+
+Or you can use our image directly:
+
+```bash
+docker run -it -p 8080:80 ghcr.io/gchq/cyberchef:latest
+```
+
+This image is built and published through our [GitHub Workflows](.github/workflows/releases.yml)
 
 ## How it works
 
@@ -50,12 +64,12 @@ You can use as many operations as you like in simple or complex ways. Some examp
 
  - Drag and drop
      - Operations can be dragged in and out of the recipe list, or reorganised.
-     - Files up to 500MB can be dragged over the input box to load them directly into the browser.
+     - Files up to 2GB can be dragged over the input box to load them directly into the browser.
  - Auto Bake
      - Whenever you modify the input or the recipe, CyberChef will automatically "bake" for you and produce the output immediately.
      - This can be turned off and operated manually if it is affecting performance (if the input is very large, for instance).
  - Automated encoding detection
-     - CyberChef uses [a number of techniques](https://github.com/gchq/CyberChef/wiki/Automatic-detection-of-encoded-data-using-CyberChef-Magic) to attempt to automatically detect which encodings your data is under. If it finds a suitable operation which can make sense of your data, it displays the 'magic' icon in the Output field which you can click to decode your data.
+     - CyberChef uses [a number of techniques](https://github.com/gchq/CyberChef/wiki/Automatic-detection-of-encoded-data-using-CyberChef-Magic) to attempt to automatically detect which encodings your data is under. If it finds a suitable operation that make sense of your data, it displays the 'magic' icon in the Output field which you can click to decode your data.
  - Breakpoints
      - You can set breakpoints on any operation in your recipe to pause execution before running it.
      - You can also step through the recipe one operation at a time to see what the data looks like at each stage.
@@ -67,26 +81,38 @@ You can use as many operations as you like in simple or complex ways. Some examp
  - Highlighting
      - When you highlight text in the input or output, the offset and length values will be displayed and, if possible, the corresponding data will be highlighted in the output or input respectively (example: [highlight the word 'question' in the input to see where it appears in the output][11]).
  - Save to file and load from file
-     - You can save the output to a file at any time or load a file by dragging and dropping it into the input field. Files up to around 500MB are supported (depending on your browser), however some operations may take a very long time to run over this much data.
+     - You can save the output to a file at any time or load a file by dragging and dropping it into the input field. Files up to around 2GB are supported (depending on your browser), however, some operations may take a very long time to run over this much data.
  - CyberChef is entirely client-side
      - It should be noted that none of your recipe configuration or input (either text or files) is ever sent to the CyberChef web server - all processing is carried out within your browser, on your own computer.
-     - Due to this feature, CyberChef can be compiled into a single HTML file. You can download this file and drop it into a virtual machine, share it with other people, or use it independently on your local machine.
+     - Due to this feature, CyberChef can be downloaded and run locally. You can use the link in the top left corner of the app to download a full copy of CyberChef and drop it into a virtual machine, share it with other people, or host it in a closed network.
+
+
+## Deep linking
+
+By manipulating CyberChef's URL hash, you can change the initial settings with which the page opens.
+The format is `https://gchq.github.io/CyberChef/#recipe=Operation()&input=...`
+
+Supported arguments are `recipe`, `input` (encoded in Base64), and `theme`.
 
 
 ## Browser support
 
 CyberChef is built to support
 
- - Google Chrome 40+
- - Mozilla Firefox 35+
- - Microsoft Edge 14+
+ - Google Chrome 50+
+ - Mozilla Firefox 38+
+
+
+## Node.js support
+
+CyberChef is built to fully support Node.js `v16`. For more information, see the ["Node API" wiki page](https://github.com/gchq/CyberChef/wiki/Node-API)
 
 
 ## Contributing
 
-Contributing a new operation to CyberChef is super easy! There is a quickstart script which will walk you through the process. If you can write basic JavaScript, you can write a CyberChef operation.
+Contributing a new operation to CyberChef is super easy! The quickstart script will walk you through the process. If you can write basic JavaScript, you can write a CyberChef operation.
 
-An installation walkthrough, how-to guides for adding new operations and themes, descriptions of the repository structure, available data types and coding conventions can all be found in the project [wiki pages](https://github.com/gchq/CyberChef/wiki).
+An installation walkthrough, how-to guides for adding new operations and themes, descriptions of the repository structure, available data types and coding conventions can all be found in the ["Contributing" wiki page](https://github.com/gchq/CyberChef/wiki/Contributing).
 
  - Push your changes to your fork.
  - Submit a pull request. If you are doing this for the first time, you will be prompted to sign the [GCHQ Contributor Licence Agreement](https://cla-assistant.io/gchq/CyberChef) via the CLA assistant on the pull request. This will also ask whether you are happy for GCHQ to contact you about a token of thanks for your contribution, or about job opportunities at GCHQ.
@@ -94,7 +120,7 @@ An installation walkthrough, how-to guides for adding new operations and themes,
 
 ## Licencing
 
-CyberChef is released under the [Apache 2.0 Licence](https://www.apache.org/licenses/LICENSE-2.0) and is covered by [Crown Copyright](https://www.nationalarchives.gov.uk/information-management/re-using-public-sector-information/copyright-and-re-use/crown-copyright/).
+CyberChef is released under the [Apache 2.0 Licence](https://www.apache.org/licenses/LICENSE-2.0) and is covered by [Crown Copyright](https://www.nationalarchives.gov.uk/information-management/re-using-public-sector-information/uk-government-licensing-framework/crown-copyright/).
 
 
   [1]: https://gchq.github.io/CyberChef

SECURITY.md

@@ -0,0 +1,26 @@
+# Security Policy
+
+## Supported Versions
+
+CyberChef is supported on a best endeavours basis. Patches will be applied to
+the latest version rather than retroactively to older versions. To ensure you
+are using the most secure version of CyberChef, please make sure you have the
+[latest release](https://github.com/gchq/CyberChef/releases/latest). The
+official [live demo](https://gchq.github.io/CyberChef/) is always up to date.
+
+## Reporting a Vulnerability
+
+In most scenarios, the most appropriate way to report a vulnerability is to
+[raise a new issue](https://github.com/gchq/CyberChef/issues/new/choose)
+describing the problem in as much detail as possible, ideally with examples.
+This will obviously be public. If you feel that the vulnerability is
+significant enough to warrant a private disclosure, please email
+[oss@gchq.gov.uk](mailto:oss@gchq.gov.uk) and
+[n1474335@gmail.com](mailto:n1474335@gmail.com).
+
+Disclosures of vulnerabilities in CyberChef are always welcomed. Whilst we aim
+to write clean and secure code free from bugs, we recognise that this is an open
+source project written by analysts in their spare time, relying on dozens of
+open source libraries that are modified and updated on a regular basis. We hope
+that the community will continue to support us as we endeavour to maintain and
+develop this tool together.

babel.config.js

@@ -4,21 +4,24 @@ module.exports = function(api) {
     return {
         "presets": [
             ["@babel/preset-env", {
-                "targets": {
-                    "chrome": 40,
-                    "firefox": 35,
-                    "edge": 14,
-                    "node": "6.5"
-                },
                 "modules": false,
-                "useBuiltIns": "entry"
+                "useBuiltIns": "entry",
+                "corejs": 3
             }]
         ],
         "plugins": [
-            "babel-plugin-syntax-dynamic-import",
-            ["babel-plugin-transform-builtin-extend", {
-                "globals": ["Error"]
-            }]
+            "dynamic-import-node",
+            "@babel/plugin-syntax-import-assertions",
+            [
+                "babel-plugin-transform-builtin-extend", {
+                    "globals": ["Error"]
+                }
+            ],
+            [
+                "@babel/plugin-transform-runtime", {
+                    "regenerator": true
+                }
+            ]
         ]
     };
 };

docs/jsdoc.conf.json

@@ -1,32 +0,0 @@
-{
-    "tags": {
-        "allowUnknownTags": true
-    },
-    "plugins": [
-        "plugins/markdown",
-        "node_modules/jsdoc-babel"
-    ],
-    "templates": {
-        "systemName": "CyberChef",
-        "footer": "",
-        "copyright": "© Crown Copyright 2017",
-        "navType": "inline",
-        "theme": "cerulean",
-        "linenums": true,
-        "collapseSymbols": false,
-        "inverseNav": true,
-        "outputSourceFiles": true,
-        "outputSourcePath": true,
-        "dateFormat": "ddd MMM Do YYYY",
-        "sort": false,
-        "logoFile": "cyberchef-32x32.png",
-        "cleverLinks": false,
-        "monospaceLinks": false,
-        "protocol": "html://",
-        "methodHeadingReturns": false
-    },
-    "markdown": {
-        "parser": "gfm",
-        "hardwrap": true
-    }
-}

nightwatch.json

@@ -1,5 +1,6 @@
 {
   "src_folders": ["tests/browser"],
+  "exclude": ["tests/browser/browserUtils.js"],
   "output_folder": "tests/browser/output",
 
   "test_settings": {
@@ -10,11 +11,12 @@
         "start_process": true,
         "server_path": "./node_modules/.bin/chromedriver",
         "port": 9515,
-        "log_path": false
+        "log_path": "tests/browser/output"
       },
       "desiredCapabilities": {
         "browserName": "chrome"
-      }
+      },
+      "enable_fail_fast": true
     },
 
     "dev": {
@@ -23,10 +25,6 @@
 
     "prod": {
       "launch_url": "http://localhost:8000/index.html"
-    },
-
-    "inline": {
-      "launch_url": "http://localhost:8000/cyberchef.htm"
     }
 
   }

package.json

@@ -1,6 +1,6 @@
 {
   "name": "cyberchef",
-  "version": "8.24.2",
+  "version": "10.8.2",
   "description": "The Cyber Swiss Army Knife for encryption, encoding, compression and data analysis.",
   "author": "n1474335 <n1474335@gmail.com>",
   "homepage": "https://gchq.github.io/CyberChef",
@@ -27,123 +27,172 @@
     "type": "git",
     "url": "https://github.com/gchq/CyberChef/"
   },
-  "main": "build/node/CyberChef.js",
+  "main": "src/node/wrapper.js",
+  "exports": {
+    "import": "./src/node/index.mjs",
+    "require": "./src/node/wrapper.js"
+  },
   "bugs": "https://github.com/gchq/CyberChef/issues",
+  "browserslist": [
+    "Chrome >= 50",
+    "Firefox >= 38",
+    "node >= 16"
+  ],
   "devDependencies": {
-    "@babel/core": "^7.2.2",
-    "@babel/preset-env": "^7.2.3",
-    "autoprefixer": "^9.4.3",
-    "babel-eslint": "^10.0.1",
-    "babel-loader": "^8.0.4",
-    "babel-plugin-syntax-dynamic-import": "^6.18.0",
-    "bootstrap": "^4.2.1",
-    "chromedriver": "^2.45.0",
-    "colors": "^1.3.3",
-    "css-loader": "^2.1.0",
-    "eslint": "^5.12.1",
-    "exports-loader": "^0.7.0",
-    "file-loader": "^3.0.1",
-    "grunt": "^1.0.3",
-    "grunt-accessibility": "~6.0.0",
+    "@babel/core": "^7.23.9",
+    "@babel/eslint-parser": "^7.23.10",
+    "@babel/plugin-syntax-import-assertions": "^7.23.3",
+    "@babel/plugin-transform-runtime": "^7.23.9",
+    "@babel/preset-env": "^7.23.9",
+    "@babel/runtime": "^7.23.9",
+    "@codemirror/commands": "^6.3.3",
+    "@codemirror/language": "^6.10.1",
+    "@codemirror/search": "^6.5.5",
+    "@codemirror/state": "^6.4.0",
+    "@codemirror/view": "^6.23.1",
+    "autoprefixer": "^10.4.17",
+    "babel-loader": "^9.1.3",
+    "babel-plugin-dynamic-import-node": "^2.3.3",
+    "babel-plugin-transform-builtin-extend": "1.1.2",
+    "base64-loader": "^1.0.0",
+    "chromedriver": "^121.0.0",
+    "cli-progress": "^3.12.0",
+    "colors": "^1.4.0",
+    "copy-webpack-plugin": "^12.0.2",
+    "core-js": "^3.35.1",
+    "css-loader": "6.10.0",
+    "eslint": "^8.56.0",
+    "grunt": "^1.6.1",
     "grunt-chmod": "~1.1.1",
-    "grunt-concurrent": "^2.3.1",
-    "grunt-contrib-clean": "~2.0.0",
-    "grunt-contrib-connect": "^2.0.0",
+    "grunt-concurrent": "^3.0.0",
+    "grunt-contrib-clean": "~2.0.1",
+    "grunt-contrib-connect": "^4.0.0",
     "grunt-contrib-copy": "~1.0.0",
     "grunt-contrib-watch": "^1.1.0",
-    "grunt-eslint": "^21.0.0",
+    "grunt-eslint": "^24.3.0",
     "grunt-exec": "~3.0.0",
-    "grunt-jsdoc": "^2.3.0",
-    "grunt-webpack": "^3.1.3",
-    "html-webpack-plugin": "^3.2.0",
-    "imports-loader": "^0.8.0",
-    "ink-docstrap": "^1.3.2",
-    "jsdoc-babel": "^0.5.0",
-    "mini-css-extract-plugin": "^0.5.0",
-    "nightwatch": "^1.0.18",
-    "node-sass": "^4.11.0",
-    "postcss-css-variables": "^0.11.0",
-    "postcss-import": "^12.0.1",
-    "postcss-loader": "^3.0.0",
-    "prompt": "^1.0.0",
-    "sass-loader": "^7.1.0",
-    "sitemap": "^2.1.0",
-    "style-loader": "^0.23.1",
-    "url-loader": "^1.1.2",
-    "web-resource-inliner": "^4.2.1",
-    "webpack": "^4.28.3",
-    "webpack-bundle-analyzer": "^3.0.3",
-    "webpack-dev-server": "^3.1.14",
-    "webpack-node-externals": "^1.7.2",
-    "worker-loader": "^2.0.0"
+    "grunt-webpack": "^6.0.0",
+    "grunt-zip": "^1.0.0",
+    "html-webpack-plugin": "^5.6.0",
+    "imports-loader": "^5.0.0",
+    "mini-css-extract-plugin": "2.8.0",
+    "modify-source-webpack-plugin": "^3.0.0",
+    "nightwatch": "^3.4.0",
+    "postcss": "^8.4.33",
+    "postcss-css-variables": "^0.19.0",
+    "postcss-import": "^16.0.0",
+    "postcss-loader": "^8.1.0",
+    "prompt": "^1.3.0",
+    "sitemap": "^7.1.1",
+    "terser": "^5.27.0",
+    "webpack": "^5.90.1",
+    "webpack-bundle-analyzer": "^4.10.1",
+    "webpack-dev-server": "4.15.1",
+    "webpack-node-externals": "^3.0.0",
+    "worker-loader": "^3.0.8"
   },
   "dependencies": {
+    "@astronautlabs/amf": "^0.0.6",
+    "@babel/polyfill": "^7.12.1",
+    "@blu3r4y/lzma": "^2.3.3",
+    "@wavesenterprise/crypto-gost-js": "^2.1.0-RC1",
+    "argon2-browser": "^1.18.0",
     "arrive": "^2.4.1",
-    "babel-plugin-transform-builtin-extend": "1.1.2",
-    "babel-polyfill": "^6.26.0",
+    "avsc": "^5.7.7",
     "bcryptjs": "^2.4.3",
-    "bignumber.js": "^8.0.2",
-    "bootstrap-colorpicker": "^2.5.3",
-    "bootstrap-material-design": "^4.1.1",
-    "bson": "^4.0.1",
+    "bignumber.js": "^9.1.2",
+    "blakejs": "^1.2.1",
+    "bootstrap": "4.6.2",
+    "bootstrap-colorpicker": "^3.4.0",
+    "bootstrap-material-design": "^4.1.3",
+    "browserify-zlib": "^0.2.0",
+    "bson": "^4.7.2",
+    "buffer": "^6.0.3",
+    "cbor": "9.0.2",
     "chi-squared": "^1.1.0",
-    "crypto-api": "^0.8.3",
-    "crypto-js": "^3.1.9-1",
+    "codepage": "^1.15.0",
+    "crypto-api": "^0.8.5",
+    "crypto-browserify": "^3.12.0",
+    "crypto-js": "^4.2.0",
     "ctph.js": "0.0.5",
-    "diff": "^3.5.0",
-    "es6-promisify": "^6.0.1",
-    "escodegen": "^1.11.0",
-    "esmangle": "^1.0.1",
+    "d3": "7.8.5",
+    "d3-hexbin": "^0.2.2",
+    "diff": "^5.1.0",
+    "es6-promisify": "^7.0.0",
+    "escodegen": "^2.1.0",
     "esprima": "^4.0.1",
     "exif-parser": "^0.1.12",
-    "fernet": "^0.3.1",
-    "file-saver": "^2.0.0",
-    "geodesy": "^1.1.3",
-    "highlight.js": "^9.13.1",
-    "jimp": "^0.6.0",
-    "jquery": "^3.3.1",
+    "fernet": "^0.3.2",
+    "file-saver": "^2.0.5",
+    "flat": "^6.0.1",
+    "geodesy": "1.1.3",
+    "highlight.js": "^11.9.0",
+    "jimp": "^0.16.13",
+    "jquery": "3.7.1",
     "js-crc": "^0.2.0",
-    "js-sha3": "^0.8.0",
-    "jsesc": "^2.5.2",
-    "jsonpath": "^1.0.0",
-    "jsonwebtoken": "^8.4.0",
-    "jsqr": "^1.1.1",
-    "jsrsasign": "8.0.12",
-    "kbpgp": "^2.0.82",
-    "libyara-wasm": "0.0.11",
-    "lodash": "^4.17.11",
-    "loglevel": "^1.6.1",
+    "js-sha3": "^0.9.3",
+    "jsesc": "^3.0.2",
+    "json5": "^2.2.3",
+    "jsonpath-plus": "^8.0.0",
+    "jsonwebtoken": "8.5.1",
+    "jsqr": "^1.4.0",
+    "jsrsasign": "^11.1.0",
+    "kbpgp": "2.1.15",
+    "libbzip2-wasm": "0.0.4",
+    "libyara-wasm": "^1.2.1",
+    "lodash": "^4.17.21",
+    "loglevel": "^1.9.1",
     "loglevel-message-prefix": "^3.0.0",
-    "moment": "^2.23.0",
-    "moment-timezone": "^0.5.23",
+    "lz-string": "^1.5.0",
+    "lz4js": "^0.2.0",
+    "markdown-it": "^14.0.0",
+    "moment": "^2.30.1",
+    "moment-timezone": "^0.5.44",
     "ngeohash": "^0.6.3",
-    "node-forge": "^0.7.6",
+    "node-forge": "^1.3.1",
     "node-md6": "^0.1.0",
-    "notepack.io": "^2.2.0",
+    "nodom": "^2.4.0",
+    "notepack.io": "^3.0.1",
+    "ntlm": "^0.1.3",
     "nwmatcher": "^1.4.4",
-    "otp": "^0.1.3",
-    "popper.js": "^1.14.6",
+    "otp": "0.1.3",
+    "path": "^0.12.7",
+    "popper.js": "^1.16.1",
+    "process": "^0.11.10",
+    "protobufjs": "^7.2.6",
     "qr-image": "^3.2.0",
-    "scryptsy": "^2.0.0",
+    "reflect-metadata": "^0.2.1",
+    "rison": "^0.1.1",
+    "scryptsy": "^2.1.0",
     "snackbarjs": "^1.1.0",
-    "sortablejs": "^1.8.0-rc1",
-    "split.js": "^1.5.10",
-    "ssdeep.js": "0.0.2",
-    "ua-parser-js": "^0.7.19",
+    "sortablejs": "^1.15.2",
+    "split.js": "^1.6.5",
+    "ssdeep.js": "0.0.3",
+    "stream-browserify": "^3.0.0",
+    "tesseract.js": "5.0.4",
+    "ua-parser-js": "^1.0.37",
+    "unorm": "^1.6.0",
     "utf8": "^3.0.0",
     "vkbeautify": "^0.99.3",
-    "xmldom": "^0.1.27",
-    "xpath": "0.0.27",
-    "xregexp": "^4.2.4",
+    "xmldom": "^0.6.0",
+    "xpath": "0.0.34",
+    "xregexp": "^5.1.1",
     "zlibjs": "^0.3.1"
   },
   "scripts": {
-    "start": "grunt dev",
-    "build": "grunt prod",
-    "test": "grunt test",
-    "testui": "grunt testui",
-    "docs": "grunt docs",
-    "lint": "grunt lint",
-    "newop": "node --experimental-modules src/core/config/scripts/newOperation.mjs"
+    "start": "npx grunt dev",
+    "build": "npx grunt prod",
+    "node": "npx grunt node",
+    "repl": "node --experimental-modules --experimental-json-modules --experimental-specifier-resolution=node --no-experimental-fetch --no-warnings src/node/repl.mjs",
+    "test": "npx grunt configTests && node --experimental-modules --experimental-json-modules --no-warnings --no-deprecation --openssl-legacy-provider --no-experimental-fetch tests/node/index.mjs && node --experimental-modules --experimental-json-modules --no-warnings --no-deprecation --openssl-legacy-provider --no-experimental-fetch --trace-uncaught tests/operations/index.mjs",
+    "testnodeconsumer": "npx grunt testnodeconsumer",
+    "testui": "npx grunt testui",
+    "testuidev": "npx nightwatch --env=dev",
+    "lint": "npx grunt lint",
+    "postinstall": "npx grunt exec:fixCryptoApiImports && npx grunt exec:fixSnackbarMarkup",
+    "newop": "node --experimental-modules --experimental-json-modules src/core/config/scripts/newOperation.mjs",
+    "minor": "node --experimental-modules --experimental-json-modules src/core/config/scripts/newMinorVersion.mjs",
+    "getheapsize": "node -e 'console.log(`node heap limit = ${require(\"v8\").getHeapStatistics().heap_size_limit / (1024 * 1024)} Mb`)'",
+    "setheapsize": "export NODE_OPTIONS=--max_old_space_size=2048"
   }
 }

postcss.config.js

@@ -1,13 +1,7 @@
 module.exports = {
     plugins: [
         require("postcss-import"),
-        require("autoprefixer")({
-            browsers: [
-                "Chrome >= 40",
-                "Firefox >= 35",
-                "Edge >= 14"
-            ]
-        }),
+        require("autoprefixer"),
         require("postcss-css-variables")({
             preserve: true
         }),

src/core/Chef.mjs

@@ -4,9 +4,10 @@
  * @license Apache-2.0
  */
 
-import Dish from "./Dish";
-import Recipe from "./Recipe";
+import Dish from "./Dish.mjs";
+import Recipe from "./Recipe.mjs";
 import log from "loglevel";
+import { isWorkerEnvironment } from "./Utils.mjs";
 
 /**
  * The main controller for CyberChef.
@@ -26,10 +27,8 @@ class Chef {
      *
      * @param {string|ArrayBuffer} input - The input data as a string or ArrayBuffer
      * @param {Object[]} recipeConfig - The recipe configuration object
-     * @param {Object} options - The options object storing various user choices
-     * @param {boolean} options.attempHighlight - Whether or not to attempt highlighting
-     * @param {number} progress - The position in the recipe to start from
-     * @param {number} [step] - Whether to only execute one operation in the recipe
+     * @param {Object} [options={}] - The options object storing various user choices
+     * @param {string} [options.returnType] - What type to return the result as
      *
      * @returns {Object} response
      * @returns {string} response.result - The output of the recipe
@@ -38,46 +37,19 @@ class Chef {
      * @returns {number} response.duration - The number of ms it took to execute the recipe
      * @returns {number} response.error - The error object thrown by a failed operation (false if no error)
     */
-    async bake(input, recipeConfig, options, progress, step) {
+    async bake(input, recipeConfig, options={}) {
         log.debug("Chef baking");
-        const startTime = new Date().getTime(),
+        const startTime = Date.now(),
             recipe      = new Recipe(recipeConfig),
-            containsFc  = recipe.containsFlowControl(),
-            notUTF8     = options && options.hasOwnProperty("treatAsUtf8") && !options.treatAsUtf8;
-        let error = false;
-
-        if (containsFc && ENVIRONMENT_IS_WORKER()) self.setOption("attemptHighlight", false);
-
-        // Clean up progress
-        if (progress >= recipeConfig.length) {
-            progress = 0;
-        }
-
-        if (step) {
-            // Unset breakpoint on this step
-            recipe.setBreakpoint(progress, false);
-            // Set breakpoint on next step
-            recipe.setBreakpoint(progress + 1, true);
-        }
-
-        // If the previously run operation presented a different value to its
-        // normal output, we need to recalculate it.
-        if (recipe.lastOpPresented(progress)) {
+            containsFc  = recipe.containsFlowControl();
+        let error = false,
             progress = 0;
-        }
 
-        // If stepping with flow control, we have to start from the beginning
-        // but still want to skip all previous breakpoints
-        if (progress > 0 && containsFc) {
-            recipe.removeBreaksUpTo(progress);
-            progress = 0;
-        }
+        if (containsFc && isWorkerEnvironment()) self.setOption("attemptHighlight", false);
 
-        // If starting from scratch, load data
-        if (progress === 0) {
-            const type = input instanceof ArrayBuffer ? Dish.ARRAY_BUFFER : Dish.STRING;
-            this.dish.set(input, type);
-        }
+        // Load data
+        const type = input instanceof ArrayBuffer ? Dish.ARRAY_BUFFER : Dish.STRING;
+        this.dish.set(input, type);
 
         try {
             progress = await recipe.execute(this.dish, progress);
@@ -89,26 +61,22 @@ class Chef {
             progress = err.progress;
         }
 
-        // Depending on the size of the output, we may send it back as a string or an ArrayBuffer.
-        // This can prevent unnecessary casting as an ArrayBuffer can be easily downloaded as a file.
-        // The threshold is specified in KiB.
-        const threshold = (options.ioDisplayThreshold || 1024) * 1024;
-        const returnType = this.dish.size > threshold ? Dish.ARRAY_BUFFER : Dish.STRING;
-
         // Create a raw version of the dish, unpresented
         const rawDish = this.dish.clone();
 
         // Present the raw result
         await recipe.present(this.dish);
 
+        const returnType =
+            this.dish.type === Dish.HTML ? Dish.HTML :
+                options?.returnType ? options.returnType : Dish.ARRAY_BUFFER;
+
         return {
             dish: rawDish,
-            result: this.dish.type === Dish.HTML ?
-                await this.dish.get(Dish.HTML, notUTF8) :
-                await this.dish.get(returnType, notUTF8),
+            result: await this.dish.get(returnType),
             type: Dish.enumLookup(this.dish.type),
             progress: progress,
-            duration: new Date().getTime() - startTime,
+            duration: Date.now() - startTime,
             error: error
         };
     }
@@ -134,7 +102,7 @@ class Chef {
     silentBake(recipeConfig) {
         log.debug("Running silent bake");
 
-        const startTime = new Date().getTime(),
+        const startTime = Date.now(),
             recipe = new Recipe(recipeConfig),
             dish = new Dish();
 
@@ -143,7 +111,7 @@ class Chef {
         } catch (err) {
             // Suppress all errors
         }
-        return new Date().getTime() - startTime;
+        return Date.now() - startTime;
     }
 
 
@@ -170,7 +138,12 @@ class Chef {
             const func = direction === "forward" ? highlights[i].f : highlights[i].b;
 
             if (typeof func == "function") {
-                pos = func(pos, highlights[i].args);
+                try {
+                    pos = func(pos, highlights[i].args);
+                } catch (err) {
+                    // Throw away highlighting errors
+                    pos = [];
+                }
             }
         }
 
@@ -193,6 +166,18 @@ class Chef {
         return await newDish.get(type);
     }
 
+    /**
+     * Gets the title of a dish and returns it
+     *
+     * @param {Dish} dish
+     * @param {number} [maxLength=100]
+     * @returns {string}
+     */
+    async getDishTitle(dish, maxLength=100) {
+        const newDish = new Dish(dish);
+        return await newDish.getTitle(maxLength);
+    }
+
 }
 
 export default Chef;

src/core/ChefWorker.js

@@ -6,26 +6,19 @@
  * @license Apache-2.0
  */
 
-import "babel-polyfill";
-import Chef from "./Chef";
-import OperationConfig from "./config/OperationConfig.json";
-import OpModules from "./config/modules/OpModules";
-
-// Add ">" to the start of all log messages in the Chef Worker
+import Chef from "./Chef.mjs";
+import OperationConfig from "./config/OperationConfig.json" assert {type: "json"};
+import OpModules from "./config/modules/OpModules.mjs";
 import loglevelMessagePrefix from "loglevel-message-prefix";
 
-loglevelMessagePrefix(log, {
-    prefixes: [],
-    staticPrefixes: [">"],
-    prefixFormat: "%p"
-});
-
 
 // Set up Chef instance
 self.chef = new Chef();
 
 self.OpModules = OpModules;
 self.OperationConfig = OperationConfig;
+self.inputNum = -1;
+
 
 // Tell the app that the worker has loaded and is ready to operate
 self.postMessage({
@@ -36,6 +29,9 @@ self.postMessage({
 /**
  * Respond to message from parent thread.
  *
+ * inputNum is optional and only used for baking multiple inputs.
+ * Defaults to -1 when one isn't sent with the bake message.
+ *
  * Messages should have the following format:
  * {
  *     action: "bake" | "silentBake",
@@ -44,14 +40,15 @@ self.postMessage({
  *         recipeConfig: {[Object]},
  *         options: {Object},
  *         progress: {number},
- *         step: {boolean}
- *     } | undefined
+ *         step: {boolean},
+ *         [inputNum=-1]: {number}
+ *     }
  * }
  */
 self.addEventListener("message", function(e) {
     // Handle message
     const r = e.data;
-    log.debug("ChefWorker receiving command '" + r.action + "'");
+    log.debug(`Receiving command '${r.action}'`);
 
     switch (r.action) {
         case "bake":
@@ -63,6 +60,9 @@ self.addEventListener("message", function(e) {
         case "getDishAs":
             getDishAs(r.data);
             break;
+        case "getDishTitle":
+            getDishTitle(r.data);
+            break;
         case "docURL":
             // Used to set the URL of the current document so that scripts can be
             // imported into an inline worker.
@@ -78,6 +78,12 @@ self.addEventListener("message", function(e) {
         case "setLogLevel":
             log.setLevel(r.data, false);
             break;
+        case "setLogPrefix":
+            loglevelMessagePrefix(log, {
+                prefixes: [],
+                staticPrefixes: [r.data]
+            });
+            break;
         default:
             break;
     }
@@ -92,30 +98,38 @@ self.addEventListener("message", function(e) {
 async function bake(data) {
     // Ensure the relevant modules are loaded
     self.loadRequiredModules(data.recipeConfig);
-
     try {
+        self.inputNum = data.inputNum === undefined ? -1 : data.inputNum;
         const response = await self.chef.bake(
             data.input,          // The user's input
             data.recipeConfig,   // The configuration of the recipe
-            data.options,        // Options set by the user
-            data.progress,       // The current position in the recipe
-            data.step            // Whether or not to take one step or execute the whole recipe
+            data.options         // Options set by the user
         );
 
+        const transferable = (response.dish.value instanceof ArrayBuffer) ?
+            [response.dish.value] :
+            undefined;
+
         self.postMessage({
             action: "bakeComplete",
             data: Object.assign(response, {
-                id: data.id
+                id: data.id,
+                inputNum: data.inputNum,
+                bakeId: data.bakeId
             })
-        });
+        }, transferable);
+
     } catch (err) {
         self.postMessage({
             action: "bakeError",
-            data: Object.assign(err, {
-                id: data.id
-            })
+            data: {
+                error: err.message || err,
+                id: data.id,
+                inputNum: data.inputNum
+            }
         });
     }
+    self.inputNum = -1;
 }
 
 
@@ -137,13 +151,33 @@ function silentBake(data) {
  */
 async function getDishAs(data) {
     const value = await self.chef.getDishAs(data.dish, data.type);
-
+    const transferable = (data.type === "ArrayBuffer") ? [value] : undefined;
     self.postMessage({
         action: "dishReturned",
         data: {
             value: value,
             id: data.id
         }
+    }, transferable);
+}
+
+
+/**
+ * Gets the dish title
+ *
+ * @param {object} data
+ * @param {Dish} data.dish
+ * @param {number} data.maxLength
+ * @param {number} data.id
+ */
+async function getDishTitle(data) {
+    const title = await self.chef.getDishTitle(data.dish, data.maxLength);
+    self.postMessage({
+        action: "dishReturned",
+        data: {
+            value: title,
+            id: data.id
+        }
     });
 }
 
@@ -153,7 +187,7 @@ async function getDishAs(data) {
  *
  * @param {Object[]} recipeConfig
  * @param {string} direction
- * @param {Object} pos - The position object for the highlight.
+ * @param {Object[]} pos - The position object for the highlight.
  * @param {number} pos.start - The start offset.
  * @param {number} pos.end - The end offset.
  */
@@ -176,10 +210,10 @@ self.loadRequiredModules = function(recipeConfig) {
     recipeConfig.forEach(op => {
         const module = self.OperationConfig[op.op].module;
 
-        if (!OpModules.hasOwnProperty(module)) {
+        if (!(module in OpModules)) {
             log.info(`Loading ${module} module`);
             self.sendStatusMessage(`Loading ${module} module`);
-            self.importScripts(`${self.docURL}/${module}.js`);
+            self.importScripts(`${self.docURL}/modules/${module}.js`); // lgtm [js/client-side-unvalidated-url-redirection]
             self.sendStatusMessage("");
         }
     });
@@ -194,7 +228,28 @@ self.loadRequiredModules = function(recipeConfig) {
 self.sendStatusMessage = function(msg) {
     self.postMessage({
         action: "statusMessage",
-        data: msg
+        data: {
+            message: msg,
+            inputNum: self.inputNum
+        }
+    });
+};
+
+
+/**
+ * Send progress update to the app.
+ *
+ * @param {number} progress
+ * @param {number} total
+ */
+self.sendProgressMessage = function(progress, total) {
+    self.postMessage({
+        action: "progressMessage",
+        data: {
+            progress: progress,
+            total: total,
+            inputNum: self.inputNum
+        }
     });
 };
 

src/core/Dish.mjs

@@ -5,11 +5,22 @@
  * @license Apache-2.0
  */
 
-import Utils from "./Utils";
-import DishError from "./errors/DishError";
+import Utils, { isNodeEnvironment } from "./Utils.mjs";
+import DishError from "./errors/DishError.mjs";
 import BigNumber from "bignumber.js";
+import { detectFileType } from "./lib/FileType.mjs";
 import log from "loglevel";
 
+import DishByteArray from "./dishTypes/DishByteArray.mjs";
+import DishBigNumber from "./dishTypes/DishBigNumber.mjs";
+import DishFile from "./dishTypes/DishFile.mjs";
+import DishHTML from "./dishTypes/DishHTML.mjs";
+import DishJSON from "./dishTypes/DishJSON.mjs";
+import DishListFile from "./dishTypes/DishListFile.mjs";
+import DishNumber from "./dishTypes/DishNumber.mjs";
+import DishString from "./dishTypes/DishString.mjs";
+
+
 /**
  * The data being operated on by each operation.
  */
@@ -18,16 +29,27 @@ class Dish {
     /**
      * Dish constructor
      *
-     * @param {Dish} [dish=null] - A dish to clone
+     * @param {Dish || *} [dishOrInput=null] - A dish to clone OR an object
+     * literal to make into a dish
+     * @param {Enum} [type=null] (optional) - A type to accompany object
+     * literal input
      */
-    constructor(dish=null) {
-        this.value = [];
-        this.type = Dish.BYTE_ARRAY;
-
-        if (dish &&
-            dish.hasOwnProperty("value") &&
-            dish.hasOwnProperty("type")) {
-            this.set(dish.value, dish.type);
+    constructor(dishOrInput=null, type = null) {
+        this.value = new ArrayBuffer(0);
+        this.type = Dish.ARRAY_BUFFER;
+
+        // Case: dishOrInput is dish object
+        if (dishOrInput &&
+            Object.prototype.hasOwnProperty.call(dishOrInput, "value") &&
+            Object.prototype.hasOwnProperty.call(dishOrInput, "type")) {
+            this.set(dishOrInput.value, dishOrInput.type);
+        // input and type defined separately
+        } else if (dishOrInput && type !== null) {
+            this.set(dishOrInput, type);
+        // No type declared, so infer it.
+        } else if (dishOrInput) {
+            const inferredType = Dish.typeEnum(dishOrInput.constructor.name);
+            this.set(dishOrInput, inferredType);
         }
     }
 
@@ -56,6 +78,7 @@ class Dish {
             case "big number":
                 return Dish.BIG_NUMBER;
             case "json":
+            case "object": // object constructor name. To allow JSON input in node.
                 return Dish.JSON;
             case "file":
                 return Dish.FILE;
@@ -99,6 +122,42 @@ class Dish {
     }
 
 
+    /**
+     * Returns the value of the data in the type format specified.
+     *
+     * If running in a browser, get is asynchronous.
+     *
+     * @param {number} type - The data type of value, see Dish enums.
+     * @returns {* | Promise} - (Browser) A promise | (Node) value of dish in given type
+     */
+    get(type) {
+        if (typeof type === "string") {
+            type = Dish.typeEnum(type);
+        }
+
+        if (this.type !== type) {
+
+            // Node environment => _translate is sync
+            if (isNodeEnvironment()) {
+                this._translate(type);
+                return this.value;
+
+            // Browser environment => _translate is async
+            } else {
+                return new Promise((resolve, reject) => {
+                    this._translate(type)
+                        .then(() => {
+                            resolve(this.value);
+                        })
+                        .catch(reject);
+                });
+            }
+        }
+
+        return this.value;
+    }
+
+
     /**
      * Sets the data value and type and then validates them.
      *
@@ -117,124 +176,85 @@ class Dish {
         this.type = type;
 
         if (!this.valid()) {
-            const sample = Utils.truncate(JSON.stringify(this.value), 13);
+            const sample = Utils.truncate(JSON.stringify(this.value), 25);
             throw new DishError(`Data is not a valid ${Dish.enumLookup(type)}: ${sample}`);
         }
     }
 
-
     /**
-     * Returns the value of the data in the type format specified.
+     * Returns the Dish as the given type, without mutating the original dish.
+     *
+     * If running in a browser, get is asynchronous.
+     *
+     * @Node
      *
      * @param {number} type - The data type of value, see Dish enums.
-     * @param {boolean} [notUTF8=false] - Do not treat strings as UTF8.
-     * @returns {*} - The value of the output data.
+     * @returns {Dish | Promise} - (Browser) A promise | (Node) value of dish in given type
      */
-    async get(type, notUTF8=false) {
-        if (typeof type === "string") {
-            type = Dish.typeEnum(type);
-        }
-        if (this.type !== type) {
-            await this._translate(type, notUTF8);
+    presentAs(type) {
+        const clone = this.clone();
+        return clone.get(type);
+    }
+
+
+    /**
+     * Detects the MIME type of the current dish
+     * @returns {string}
+     */
+    detectDishType() {
+        const data = new Uint8Array(this.value.slice(0, 2048)),
+            types = detectFileType(data);
+
+        if (!types.length || !types[0].mime || !(types[0].mime === "text/plain")) {
+            return null;
+        } else {
+            return types[0].mime;
         }
-        return this.value;
     }
 
 
     /**
-     * Translates the data to the given type format.
+     * Returns the title of the data up to the specified length
      *
-     * @param {number} toType - The data type of value, see Dish enums.
-     * @param {boolean} [notUTF8=false] - Do not treat strings as UTF8.
+     * @param {number} maxLength - The maximum title length
+     * @returns {string}
      */
-    async _translate(toType, notUTF8=false) {
-        log.debug(`Translating Dish from ${Dish.enumLookup(this.type)} to ${Dish.enumLookup(toType)}`);
-        const byteArrayToStr = notUTF8 ? Utils.byteArrayToChars : Utils.byteArrayToUtf8;
+    async getTitle(maxLength) {
+        let title = "";
+        let cloned;
 
-        // Convert data to intermediate byteArray type
-        try {
-            switch (this.type) {
-                case Dish.STRING:
-                    this.value = this.value ? Utils.strToByteArray(this.value) : [];
-                    break;
-                case Dish.NUMBER:
-                    this.value = typeof this.value === "number" ? Utils.strToByteArray(this.value.toString()) : [];
-                    break;
-                case Dish.HTML:
-                    this.value = this.value ? Utils.strToByteArray(Utils.unescapeHtml(Utils.stripHtmlTags(this.value, true))) : [];
-                    break;
-                case Dish.ARRAY_BUFFER:
-                    // Array.from() would be nicer here, but it's slightly slower
-                    this.value = Array.prototype.slice.call(new Uint8Array(this.value));
-                    break;
-                case Dish.BIG_NUMBER:
-                    this.value = BigNumber.isBigNumber(this.value) ? Utils.strToByteArray(this.value.toFixed()) : [];
-                    break;
-                case Dish.JSON:
-                    this.value = this.value ? Utils.strToByteArray(JSON.stringify(this.value, null, 4)) : [];
-                    break;
-                case Dish.FILE:
-                    this.value = await Utils.readFile(this.value);
-                    this.value = Array.prototype.slice.call(this.value);
-                    break;
-                case Dish.LIST_FILE:
-                    this.value = await Promise.all(this.value.map(async f => Utils.readFile(f)));
-                    this.value = this.value.map(b => Array.prototype.slice.call(b));
-                    this.value = [].concat.apply([], this.value);
-                    break;
-                default:
-                    break;
-            }
-        } catch (err) {
-            throw new DishError(`Error translating from ${Dish.enumLookup(this.type)} to byteArray: ${err}`);
+        switch (this.type) {
+            case Dish.FILE:
+                title = this.value.name;
+                break;
+            case Dish.LIST_FILE:
+                title = `${this.value.length} file(s)`;
+                break;
+            case Dish.JSON:
+                title = "application/json";
+                break;
+            case Dish.NUMBER:
+            case Dish.BIG_NUMBER:
+                title = this.value.toString();
+                break;
+            case Dish.ARRAY_BUFFER:
+            case Dish.BYTE_ARRAY:
+                title = this.detectDishType();
+                if (title !== null) break;
+                // fall through if no mime type was detected
+            default:
+                try {
+                    cloned = this.clone();
+                    cloned.value = cloned.value.slice(0, 256);
+                    title = await cloned.get(Dish.STRING);
+                } catch (err) {
+                    log.error(`${Dish.enumLookup(this.type)} cannot be sliced. ${err}`);
+                }
         }
 
-        this.type = Dish.BYTE_ARRAY;
-
-        // Convert from byteArray to toType
-        try {
-            switch (toType) {
-                case Dish.STRING:
-                case Dish.HTML:
-                    this.value = this.value ? byteArrayToStr(this.value) : "";
-                    this.type = Dish.STRING;
-                    break;
-                case Dish.NUMBER:
-                    this.value = this.value ? parseFloat(byteArrayToStr(this.value)) : 0;
-                    this.type = Dish.NUMBER;
-                    break;
-                case Dish.ARRAY_BUFFER:
-                    this.value = new Uint8Array(this.value).buffer;
-                    this.type = Dish.ARRAY_BUFFER;
-                    break;
-                case Dish.BIG_NUMBER:
-                    try {
-                        this.value = new BigNumber(byteArrayToStr(this.value));
-                    } catch (err) {
-                        this.value = new BigNumber(NaN);
-                    }
-                    this.type = Dish.BIG_NUMBER;
-                    break;
-                case Dish.JSON:
-                    this.value = JSON.parse(byteArrayToStr(this.value));
-                    this.type = Dish.JSON;
-                    break;
-                case Dish.FILE:
-                    this.value = new File(this.value, "unknown");
-                    break;
-                case Dish.LIST_FILE:
-                    this.value = [new File(this.value, "unknown")];
-                    this.type = Dish.LIST_FILE;
-                    break;
-                default:
-                    break;
-            }
-        } catch (err) {
-            throw new DishError(`Error translating from byteArray to ${Dish.enumLookup(toType)}: ${err}`);
-        }
+        return title.slice(0, maxLength);
     }
 
-
     /**
      * Validates that the value is the type that has been specified.
      * May have to disable parts of BYTE_ARRAY validation if it effects performance.
@@ -244,7 +264,7 @@ class Dish {
     valid() {
         switch (this.type) {
             case Dish.BYTE_ARRAY:
-                if (!(this.value instanceof Array)) {
+                if (!(this.value instanceof Uint8Array) && !(this.value instanceof Array)) {
                     return false;
                 }
 
@@ -265,7 +285,21 @@ class Dish {
             case Dish.ARRAY_BUFFER:
                 return this.value instanceof ArrayBuffer;
             case Dish.BIG_NUMBER:
-                return BigNumber.isBigNumber(this.value);
+                if (BigNumber.isBigNumber(this.value)) return true;
+                /*
+                    If a BigNumber is passed between WebWorkers it is serialised as a JSON
+                    object with a coefficient (c), exponent (e) and sign (s). We detect this
+                    and reinitialise it as a BigNumber object.
+                */
+                if (Object.keys(this.value).sort().equals(["c", "e", "s"])) {
+                    const temp = new BigNumber();
+                    temp.c = this.value.c;
+                    temp.e = this.value.e;
+                    temp.s = this.value.s;
+                    this.value = temp;
+                    return true;
+                }
+                return false;
             case Dish.JSON:
                 // All values can be serialised in some manner, so we return true in all cases
                 return true;
@@ -372,6 +406,107 @@ class Dish {
         return newDish;
     }
 
+    /**
+     * Translates the data to the given type format.
+     *
+     * If running in the browser, _translate is asynchronous.
+     *
+     * @param {number} toType - The data type of value, see Dish enums.
+     * @returns {Promise || undefined}
+     */
+    _translate(toType) {
+        log.debug(`Translating Dish from ${Dish.enumLookup(this.type)} to ${Dish.enumLookup(toType)}`);
+
+        // Node environment => translate is sync
+        if (isNodeEnvironment()) {
+            this._toArrayBuffer();
+            this.type = Dish.ARRAY_BUFFER;
+            this._fromArrayBuffer(toType);
+
+        // Browser environment => translate is async
+        } else {
+            return new Promise((resolve, reject) => {
+                this._toArrayBuffer()
+                    .then(() => this.type = Dish.ARRAY_BUFFER)
+                    .then(() => {
+                        this._fromArrayBuffer(toType);
+                        resolve();
+                    })
+                    .catch(reject);
+            });
+        }
+
+    }
+
+    /**
+     * Convert this.value to an ArrayBuffer
+     *
+     * If running in a browser, _toByteArray is asynchronous.
+     *
+     * @returns {Promise || undefined}
+     */
+    _toArrayBuffer() {
+        // Using 'bind' here to allow this.value to be mutated within translation functions
+        const toByteArrayFuncs = {
+            browser: {
+                [Dish.STRING]:          () => Promise.resolve(DishString.toArrayBuffer.bind(this)()),
+                [Dish.NUMBER]:          () => Promise.resolve(DishNumber.toArrayBuffer.bind(this)()),
+                [Dish.HTML]:            () => Promise.resolve(DishHTML.toArrayBuffer.bind(this)()),
+                [Dish.ARRAY_BUFFER]:    () => Promise.resolve(),
+                [Dish.BIG_NUMBER]:      () => Promise.resolve(DishBigNumber.toArrayBuffer.bind(this)()),
+                [Dish.JSON]:            () => Promise.resolve(DishJSON.toArrayBuffer.bind(this)()),
+                [Dish.FILE]:            () => DishFile.toArrayBuffer.bind(this)(),
+                [Dish.LIST_FILE]:       () => Promise.resolve(DishListFile.toArrayBuffer.bind(this)()),
+                [Dish.BYTE_ARRAY]:      () => Promise.resolve(DishByteArray.toArrayBuffer.bind(this)()),
+            },
+            node: {
+                [Dish.STRING]:          () => DishString.toArrayBuffer.bind(this)(),
+                [Dish.NUMBER]:          () => DishNumber.toArrayBuffer.bind(this)(),
+                [Dish.HTML]:            () => DishHTML.toArrayBuffer.bind(this)(),
+                [Dish.ARRAY_BUFFER]:    () => {},
+                [Dish.BIG_NUMBER]:      () => DishBigNumber.toArrayBuffer.bind(this)(),
+                [Dish.JSON]:            () => DishJSON.toArrayBuffer.bind(this)(),
+                [Dish.FILE]:            () => DishFile.toArrayBuffer.bind(this)(),
+                [Dish.LIST_FILE]:       () => DishListFile.toArrayBuffer.bind(this)(),
+                [Dish.BYTE_ARRAY]:      () => DishByteArray.toArrayBuffer.bind(this)(),
+            }
+        };
+
+        try {
+            return toByteArrayFuncs[isNodeEnvironment() && "node" || "browser"][this.type]();
+        } catch (err) {
+            throw new DishError(`Error translating from ${Dish.enumLookup(this.type)} to ArrayBuffer: ${err}`);
+        }
+    }
+
+    /**
+     * Convert this.value to the given type from ArrayBuffer
+     *
+     * @param {number} toType - the Dish enum to convert to
+    */
+    _fromArrayBuffer(toType) {
+
+        // Using 'bind' here to allow this.value to be mutated within translation functions
+        const toTypeFunctions = {
+            [Dish.STRING]:          () => DishString.fromArrayBuffer.bind(this)(),
+            [Dish.NUMBER]:          () => DishNumber.fromArrayBuffer.bind(this)(),
+            [Dish.HTML]:            () => DishHTML.fromArrayBuffer.bind(this)(),
+            [Dish.ARRAY_BUFFER]:    () => {},
+            [Dish.BIG_NUMBER]:      () => DishBigNumber.fromArrayBuffer.bind(this)(),
+            [Dish.JSON]:            () => DishJSON.fromArrayBuffer.bind(this)(),
+            [Dish.FILE]:            () => DishFile.fromArrayBuffer.bind(this)(),
+            [Dish.LIST_FILE]:       () => DishListFile.fromArrayBuffer.bind(this)(),
+            [Dish.BYTE_ARRAY]:      () => DishByteArray.fromArrayBuffer.bind(this)(),
+        };
+
+        try {
+            toTypeFunctions[toType]();
+            this.type = toType;
+        } catch (err) {
+            throw new DishError(`Error translating from ArrayBuffer to ${Dish.enumLookup(toType)}: ${err}`);
+        }
+    }
+
 }
 
 

src/core/Ingredient.mjs

@@ -4,8 +4,8 @@
  * @license Apache-2.0
  */
 
-import Utils from "./Utils";
-import {fromHex} from "./lib/Hex";
+import Utils from "./Utils.mjs";
+import {fromHex} from "./lib/Hex.mjs";
 
 /**
  * The arguments to operations.
@@ -27,6 +27,10 @@ class Ingredient {
         this.toggleValues = [];
         this.target = null;
         this.defaultIndex = 0;
+        this.maxLength = null;
+        this.min = null;
+        this.max = null;
+        this.step = 1;
 
         if (ingredientConfig) {
             this._parseConfig(ingredientConfig);
@@ -50,6 +54,10 @@ class Ingredient {
         this.toggleValues = ingredientConfig.toggleValues;
         this.target = typeof ingredientConfig.target !== "undefined" ? ingredientConfig.target : null;
         this.defaultIndex = typeof ingredientConfig.defaultIndex !== "undefined" ? ingredientConfig.defaultIndex : 0;
+        this.maxLength = ingredientConfig.maxLength || null;
+        this.min = ingredientConfig.min;
+        this.max = ingredientConfig.max;
+        this.step = ingredientConfig.step;
     }
 
 
@@ -107,6 +115,7 @@ class Ingredient {
                     return data;
                 }
             case "number":
+                if (data === null) return data;
                 number = parseFloat(data);
                 if (isNaN(number)) {
                     const sample = Utils.truncate(data.toString(), 10);

src/core/Operation.mjs

@@ -4,8 +4,8 @@
  * @license Apache-2.0
  */
 
-import Dish from "./Dish";
-import Ingredient from "./Ingredient";
+import Dish from "./Dish.mjs";
+import Ingredient from "./Ingredient.mjs";
 
 /**
  * The Operation specified by the user to be run.
@@ -184,6 +184,10 @@ class Operation {
             if (ing.disabled) conf.disabled = ing.disabled;
             if (ing.target) conf.target = ing.target;
             if (ing.defaultIndex) conf.defaultIndex = ing.defaultIndex;
+            if (ing.maxLength) conf.maxLength = ing.maxLength;
+            if (typeof ing.min === "number") conf.min = ing.min;
+            if (typeof ing.max === "number") conf.max = ing.max;
+            if (ing.step) conf.step = ing.step;
             return conf;
         });
     }

src/core/Recipe.mjs

@@ -4,11 +4,12 @@
  * @license Apache-2.0
  */
 
-import OperationConfig from "./config/OperationConfig.json";
-import OperationError from "./errors/OperationError";
-import Operation from "./Operation";
-import DishError from "./errors/DishError";
+import OperationConfig from "./config/OperationConfig.json" assert {type: "json"};
+import OperationError from "./errors/OperationError.mjs";
+import Operation from "./Operation.mjs";
+import DishError from "./errors/DishError.mjs";
 import log from "loglevel";
+import { isWorkerEnvironment } from "./Utils.mjs";
 
 // Cache container for modules
 let modules = null;
@@ -45,7 +46,7 @@ class Recipe  {
                 module: OperationConfig[c.op].module,
                 ingValues: c.args,
                 breakpoint: c.breakpoint,
-                disabled: c.disabled,
+                disabled: c.disabled || c.op === "Comment",
             });
         });
     }
@@ -61,7 +62,7 @@ class Recipe  {
         if (!modules) {
             // Using Webpack Magic Comments to force the dynamic import to be included in the main chunk
             // https://webpack.js.org/api/module-methods/
-            modules = await import(/* webpackMode: "eager" */ "./config/modules/OpModules");
+            modules = await import(/* webpackMode: "eager" */ "./config/modules/OpModules.mjs");
             modules = modules.default;
         }
 
@@ -200,7 +201,12 @@ class Recipe  {
 
             try {
                 input = await dish.get(op.inputType);
-                log.debug("Executing operation");
+                log.debug(`Executing operation '${op.name}'`);
+
+                if (isWorkerEnvironment()) {
+                    self.sendStatusMessage(`Baking... (${i+1}/${this.opList.length})`);
+                    self.sendProgressMessage(i + 1, this.opList.length);
+                }
 
                 if (op.flowControl) {
                     // Package up the current state
@@ -224,14 +230,12 @@ class Recipe  {
                 this.lastRunOp = op;
             } catch (err) {
                 // Return expected errors as output
-                if (err instanceof OperationError ||
-                    (err.type && err.type === "OperationError")) {
+                if (err instanceof OperationError || err?.type === "OperationError") {
                     // Cannot rely on `err instanceof OperationError` here as extending
                     // native types is not fully supported yet.
                     dish.set(err.message, "string");
                     return i;
-                } else if (err instanceof DishError ||
-                    (err.type && err.type === "DishError")) {
+                } else if (err instanceof DishError || err?.type === "DishError") {
                     dish.set(err.message, "string");
                     return i;
                 } else {

src/core/config/Categories.json

@@ -18,15 +18,19 @@
             "From Binary",
             "To Octal",
             "From Octal",
-            "To Base64",
-            "From Base64",
-            "Show Base64 offsets",
             "To Base32",
             "From Base32",
+            "To Base45",
+            "From Base45",
             "To Base58",
             "From Base58",
             "To Base62",
             "From Base62",
+            "To Base64",
+            "From Base64",
+            "Show Base64 offsets",
+            "To Base92",
+            "From Base92",
             "To Base85",
             "From Base85",
             "To Base",
@@ -39,10 +43,13 @@
             "URL Decode",
             "Escape Unicode Characters",
             "Unescape Unicode Characters",
+            "Normalise Unicode",
             "To Quoted Printable",
             "From Quoted Printable",
             "To Punycode",
             "From Punycode",
+            "AMF Encode",
+            "AMF Decode",
             "To Hex Content",
             "From Hex Content",
             "PEM to Hex",
@@ -59,7 +66,13 @@
             "From Braille",
             "Parse TLV",
             "CSV to JSON",
-            "JSON to CSV"
+            "JSON to CSV",
+            "Avro to JSON",
+            "CBOR Encode",
+            "CBOR Decode",
+            "Caret/M-decode",
+            "Rison Encode",
+            "Rison Decode"
         ]
     },
     {
@@ -75,28 +88,53 @@
             "Triple DES Decrypt",
             "Fernet Encrypt",
             "Fernet Decrypt",
+            "LS47 Encrypt",
+            "LS47 Decrypt",
             "RC2 Encrypt",
             "RC2 Decrypt",
             "RC4",
             "RC4 Drop",
+            "ChaCha",
+            "Rabbit",
+            "SM4 Encrypt",
+            "SM4 Decrypt",
+            "GOST Encrypt",
+            "GOST Decrypt",
+            "GOST Sign",
+            "GOST Verify",
+            "GOST Key Wrap",
+            "GOST Key Unwrap",
             "ROT13",
+            "ROT13 Brute Force",
             "ROT47",
+            "ROT47 Brute Force",
+            "ROT8000",
             "XOR",
             "XOR Brute Force",
             "Vigenère Encode",
             "Vigenère Decode",
             "To Morse Code",
             "From Morse Code",
+            "Bacon Cipher Encode",
+            "Bacon Cipher Decode",
             "Bifid Cipher Encode",
             "Bifid Cipher Decode",
+            "Caesar Box Cipher",
             "Affine Cipher Encode",
             "Affine Cipher Decode",
             "A1Z26 Cipher Encode",
             "A1Z26 Cipher Decode",
+            "Rail Fence Cipher Encode",
+            "Rail Fence Cipher Decode",
             "Atbash Cipher",
+            "CipherSaber2 Encrypt",
+            "CipherSaber2 Decrypt",
+            "Cetacean Cipher Encode",
+            "Cetacean Cipher Decode",
             "Substitute",
             "Derive PBKDF2 key",
             "Derive EVP key",
+            "Derive HKDF key",
             "Bcrypt",
             "Scrypt",
             "JWT Sign",
@@ -104,7 +142,16 @@
             "JWT Decode",
             "Citrix CTX1 Encode",
             "Citrix CTX1 Decode",
-            "Pseudo-Random Number Generator"
+            "AES Key Wrap",
+            "AES Key Unwrap",
+            "Pseudo-Random Number Generator",
+            "Enigma",
+            "Bombe",
+            "Multiple Bombe",
+            "Typex",
+            "Lorenz",
+            "Colossus",
+            "SIGABA"
         ]
     },
     {
@@ -119,8 +166,15 @@
             "Generate PGP Key Pair",
             "PGP Encrypt",
             "PGP Decrypt",
+            "PGP Verify",
             "PGP Encrypt and Sign",
-            "PGP Decrypt and Verify"
+            "PGP Decrypt and Verify",
+            "Generate RSA Key Pair",
+            "RSA Sign",
+            "RSA Verify",
+            "RSA Encrypt",
+            "RSA Decrypt",
+            "Parse SSH Host Key"
         ]
     },
     {
@@ -150,7 +204,8 @@
             "Bit shift right",
             "Rotate left",
             "Rotate right",
-            "ROT13"
+            "ROT13",
+            "ROT8000"
         ]
     },
     {
@@ -164,15 +219,27 @@
             "Parse IP range",
             "Parse IPv6 address",
             "Parse IPv4 header",
+            "Parse TCP",
+            "Parse UDP",
+            "Parse SSH Host Key",
             "Parse URI",
             "URL Encode",
             "URL Decode",
+            "Protobuf Decode",
+            "Protobuf Encode",
+            "VarInt Encode",
+            "VarInt Decode",
+            "JA3 Fingerprint",
+            "JA3S Fingerprint",
+            "HASSH Client Fingerprint",
+            "HASSH Server Fingerprint",
             "Format MAC addresses",
             "Change IP format",
             "Group IP addresses",
             "Encode NetBIOS Name",
             "Decode NetBIOS Name",
-            "Defang URL"
+            "Defang URL",
+            "Defang IP Addresses"
         ]
     },
     {
@@ -180,8 +247,10 @@
         "ops": [
             "Encode text",
             "Decode text",
+            "Unicode Text Format",
             "Remove Diacritics",
-            "Unescape Unicode Characters"
+            "Unescape Unicode Characters",
+            "Convert to NATO alphabet"
         ]
     },
     {
@@ -192,13 +261,16 @@
             "Remove null bytes",
             "To Upper case",
             "To Lower case",
-			"To Case Insensitive Regex",
-			"From Case Insensitive Regex",
+            "Swap case",
+            "To Case Insensitive Regex",
+            "From Case Insensitive Regex",
             "Add line numbers",
             "Remove line numbers",
+            "Get All Casings",
             "To Table",
             "Reverse",
             "Sort",
+            "Shuffle",
             "Unique",
             "Split",
             "Filter",
@@ -211,21 +283,26 @@
             "Pad lines",
             "Find / Replace",
             "Regular expression",
+            "Fuzzy Match",
             "Offset checker",
             "Hamming Distance",
+            "Levenshtein Distance",
             "Convert distance",
             "Convert area",
             "Convert mass",
             "Convert speed",
             "Convert data units",
             "Convert co-ordinate format",
+            "Show on map",
             "Parse UNIX file permissions",
+            "Parse ObjectID timestamp",
             "Swap endianness",
             "Parse colour code",
             "Escape string",
             "Unescape string",
             "Pseudo-Random Number Generator",
-            "Sleep"
+            "Sleep",
+            "File Tree"
         ]
     },
     {
@@ -238,6 +315,7 @@
             "Windows Filetime to UNIX Timestamp",
             "UNIX Timestamp to Windows Filetime",
             "Extract dates",
+            "Get Time",
             "Sleep"
         ]
     },
@@ -256,7 +334,10 @@
             "XPath expression",
             "JPath expression",
             "CSS selector",
-            "Extract EXIF"
+            "Extract EXIF",
+            "Extract ID3",
+            "Extract Files",
+            "RAKE"
         ]
     },
     {
@@ -271,8 +352,16 @@
             "Zip",
             "Unzip",
             "Bzip2 Decompress",
+            "Bzip2 Compress",
             "Tar",
-            "Untar"
+            "Untar",
+            "LZString Decompress",
+            "LZString Compress",
+            "LZMA Decompress",
+            "LZMA Compress",
+            "LZ4 Decompress",
+            "LZ4 Compress",
+            "LZNT1 Decompress"
         ]
     },
     {
@@ -288,26 +377,39 @@
             "SHA1",
             "SHA2",
             "SHA3",
+            "SM3",
+            "MurmurHash3",
             "Keccak",
             "Shake",
             "RIPEMD",
             "HAS-160",
             "Whirlpool",
             "Snefru",
+            "BLAKE2b",
+            "BLAKE2s",
+            "GOST Hash",
+            "Streebog",
             "SSDEEP",
             "CTPH",
             "Compare SSDEEP hashes",
             "Compare CTPH hashes",
             "HMAC",
+            "CMAC",
             "Bcrypt",
             "Bcrypt compare",
             "Bcrypt parse",
+            "Argon2",
+            "Argon2 compare",
             "Scrypt",
+            "NT Hash",
+            "LM Hash",
             "Fletcher-8 Checksum",
             "Fletcher-16 Checksum",
             "Fletcher-32 Checksum",
             "Fletcher-64 Checksum",
             "Adler-32 Checksum",
+            "Luhn Checksum",
+            "CRC-8 Checksum",
             "CRC-16 Checksum",
             "CRC-32 Checksum",
             "TCP/IP Checksum"
@@ -342,7 +444,8 @@
             "BSON serialise",
             "BSON deserialise",
             "To MessagePack",
-            "From MessagePack"
+            "From MessagePack",
+            "Render Markdown"
         ]
     },
     {
@@ -350,8 +453,15 @@
         "ops": [
             "Detect File Type",
             "Scan for Embedded Files",
+            "Extract Files",
+            "YARA Rules",
             "Remove EXIF",
-            "Extract EXIF"
+            "Extract EXIF",
+            "Extract RGBA",
+            "View Bit Plane",
+            "Randomize Colour Palette",
+            "Extract LSB",
+            "ELF Info"
         ]
     },
     {
@@ -359,9 +469,32 @@
         "ops": [
             "Render Image",
             "Play Media",
+            "Generate Image",
+            "Optical Character Recognition",
             "Remove EXIF",
             "Extract EXIF",
-            "Split Colour Channels"
+            "Split Colour Channels",
+            "Rotate Image",
+            "Resize Image",
+            "Blur Image",
+            "Dither Image",
+            "Invert Image",
+            "Flip Image",
+            "Crop Image",
+            "Image Brightness / Contrast",
+            "Image Opacity",
+            "Image Filter",
+            "Contain Image",
+            "Cover Image",
+            "Image Hue/Saturation/Lightness",
+            "Sharpen Image",
+            "Normalise Image",
+            "Convert Image Format",
+            "Add Text To Image",
+            "Hex Density chart",
+            "Scatter chart",
+            "Series chart",
+            "Heatmap chart"
         ]
     },
     {
@@ -369,15 +502,19 @@
         "ops": [
             "Entropy",
             "Frequency distribution",
+            "Index of Coincidence",
             "Chi Square",
+            "P-list Viewer",
             "Disassemble x86",
             "Pseudo-Random Number Generator",
+            "Generate De Bruijn Sequence",
             "Generate UUID",
             "Generate TOTP",
             "Generate HOTP",
             "Generate QR Code",
             "Parse QR Code",
             "Haversine distance",
+            "HTML To Text",
             "Generate Lorem Ipsum",
             "Numberwang",
             "XKCD Random Number"

src/core/config/scripts/generateConfig.mjs

@@ -9,12 +9,12 @@
  * @license Apache-2.0
  */
 
-/*eslint no-console: ["off"] */
+/* eslint no-console: ["off"] */
 
 import path from "path";
 import fs  from "fs";
 import process from "process";
-import * as Ops from "../../operations/index";
+import * as Ops from "../../operations/index.mjs";
 
 const dir = path.join(process.cwd() + "/src/core/config/");
 if (!fs.existsSync(dir)) {
@@ -42,14 +42,11 @@ for (const opObj in Ops) {
         outputType:  op.presentType,
         flowControl: op.flowControl,
         manualBake:  op.manualBake,
-        args:        op.args
+        args:        op.args,
+        checks:      op.checks
     };
 
-    if (op.hasOwnProperty("patterns")) {
-        operationConfig[op.name].patterns = op.patterns;
-    }
-
-    if (!modules.hasOwnProperty(op.module))
+    if (!(op.module in modules))
         modules[op.module] = {};
     modules[op.module][op.name] = opObj;
 }
@@ -84,7 +81,7 @@ for (const module in modules) {
 
     for (const opName in modules[module]) {
         const objName = modules[module][opName];
-        code += `import ${objName} from "../../operations/${objName}";\n`;
+        code += `import ${objName} from "../../operations/${objName}.mjs";\n`;
     }
 
     code += `
@@ -124,7 +121,7 @@ let opModulesCode = `/**
 `;
 
 for (const module in modules) {
-    opModulesCode += `import ${module}Module from "./${module}";\n`;
+    opModulesCode += `import ${module}Module from "./${module}.mjs";\n`;
 }
 
 opModulesCode += `

src/core/config/scripts/generateOpsIndex.mjs

@@ -7,7 +7,7 @@
  * @license Apache-2.0
  */
 
-/*eslint no-console: ["off"] */
+/* eslint no-console: ["off"] */
 
 import path from "path";
 import fs  from "fs";
@@ -39,7 +39,7 @@ let code = `/**
 `;
 
 opObjs.forEach(obj => {
-    code += `import ${obj} from "./${obj}";\n`;
+    code += `import ${obj} from "./${obj}.mjs";\n`;
 });
 
 code += `

src/core/config/scripts/newMinorVersion.mjs

@@ -0,0 +1,144 @@
+/**
+ * This script updates the CHANGELOG when a new minor version is created.
+ *
+ * @author n1474335 [n1474335@gmail.com]
+ * @copyright Crown Copyright 2022
+ * @license Apache-2.0
+ */
+
+/* eslint no-console: ["off"] */
+
+import prompt from "prompt";
+import colors from "colors";
+import path from "path";
+import fs  from "fs";
+import process from "process";
+
+const dir = path.join(process.cwd() + "/src/core/config/");
+if (!fs.existsSync(dir)) {
+    console.log("\nCWD: " + process.cwd());
+    console.log("Error: newMinorVersion.mjs should be run from the project root");
+    console.log("Example> node --experimental-modules src/core/config/scripts/newMinorVersion.mjs");
+    process.exit(1);
+}
+
+let changelogData = fs.readFileSync(path.join(process.cwd(), "CHANGELOG.md"), "utf8");
+const lastVersion = changelogData.match(/## Details\s+### \[(\d+)\.(\d+)\.(\d+)\]/);
+const newVersion = [
+    parseInt(lastVersion[1], 10),
+    parseInt(lastVersion[2], 10) + 1,
+    0
+];
+
+let knownContributors = changelogData.match(/^\[@([^\]]+)\]/gm);
+knownContributors = knownContributors.map(c => c.slice(2, -1));
+
+const date = (new Date()).toISOString().split("T")[0];
+
+const schema = {
+    properties: {
+        message: {
+            description: "A short but descriptive summary of a feature in this version",
+            example: "Added 'Op name' operation",
+            prompt: "Feature description",
+            type: "string",
+            required: true,
+        },
+        author: {
+            description: "The author of the feature (only one supported, edit manually to add more)",
+            example: "n1474335",
+            prompt: "Author",
+            type: "string",
+            default: "n1474335"
+        },
+        id: {
+            description: "The PR number or full commit hash for this feature.",
+            example: "1200",
+            prompt: "Pull request or commit ID",
+            type: "string"
+        },
+        another: {
+            description: "y/n",
+            example: "y",
+            prompt: "Add another feature?",
+            type: "string",
+            pattern: /^[yn]$/,
+        }
+    }
+};
+
+// Build schema
+for (const prop in schema.properties) {
+    const p = schema.properties[prop];
+    p.description = "\n" + colors.white(p.description) + colors.cyan("\nExample: " + p.example) + "\n" + colors.green(p.prompt);
+}
+
+prompt.message = "";
+prompt.delimiter = ":".green;
+
+const features = [];
+const authors = [];
+const prIDs = [];
+const commitIDs = [];
+
+prompt.start();
+
+const getFeature = function() {
+    prompt.get(schema, (err, result) => {
+        if (err) {
+            console.log("\nExiting script.");
+            process.exit(0);
+        }
+
+        features.push(result);
+
+        if (result.another === "y") {
+            getFeature();
+        } else {
+            let message = `### [${newVersion[0]}.${newVersion[1]}.${newVersion[2]}] - ${date}\n`;
+
+            features.forEach(feature => {
+                const id = feature.id.length > 10 ? feature.id.slice(0, 7) : "#" + feature.id;
+                message += `- ${feature.message} [@${feature.author}] | [${id}]\n`;
+
+                if (!knownContributors.includes(feature.author)) {
+                    authors.push(`[@${feature.author}]: https://github.com/${feature.author}`);
+                }
+
+                if (feature.id.length > 10) {
+                    commitIDs.push(`[${id}]: https://github.com/gchq/CyberChef/commit/${feature.id}`);
+                } else {
+                    prIDs.push(`[#${feature.id}]: https://github.com/gchq/CyberChef/pull/${feature.id}`);
+                }
+            });
+
+            // Message
+            changelogData = changelogData.replace(/## Details\n\n/, "## Details\n\n" + message + "\n");
+
+            // Tag
+            const newTag = `[${newVersion[0]}.${newVersion[1]}.${newVersion[2]}]: https://github.com/gchq/CyberChef/releases/tag/v${newVersion[0]}.${newVersion[1]}.${newVersion[2]}\n`;
+            changelogData = changelogData.replace(/\n\n(\[\d+\.\d+\.\d+\]: https)/, "\n\n" + newTag + "$1");
+
+            // Author
+            authors.forEach(author => {
+                changelogData = changelogData.replace(/(\n\[@[^\]]+\]: https:\/\/github\.com\/[^\n]+\n)\n/, "$1" + author + "\n\n");
+            });
+
+            // Commit IDs
+            commitIDs.forEach(commitID => {
+                changelogData = changelogData.replace(/(\n\[[^\].]+\]: https:\/\/github.com\/gchq\/CyberChef\/commit\/[^\n]+\n)\n/, "$1" + commitID + "\n\n");
+            });
+
+            // PR IDs
+            prIDs.forEach(prID => {
+                changelogData = changelogData.replace(/(\n\[#[^\]]+\]: https:\/\/github.com\/gchq\/CyberChef\/pull\/[^\n]+\n)\n*$/, "$1" + prID + "\n\n");
+            });
+
+            fs.writeFileSync(path.join(process.cwd(), "CHANGELOG.md"), changelogData);
+
+            console.log("Written CHANGELOG.md\nCommit changes and then run `npm version minor`.");
+        }
+    });
+};
+
+getFeature();

src/core/config/scripts/newOperation.mjs

@@ -6,14 +6,14 @@
  * @license Apache-2.0
  */
 
-/*eslint no-console: ["off"] */
+/* eslint no-console: ["off"] */
 
 import prompt from "prompt";
 import colors from "colors";
 import process from "process";
 import fs from "fs";
 import path from "path";
-import EscapeString from "../../operations/EscapeString";
+import EscapeString from "../../operations/EscapeString.mjs";
 
 
 const dir = path.join(process.cwd() + "/src/core/operations/");
@@ -121,7 +121,7 @@ prompt.get(schema, (err, result) => {
 
     const moduleName = result.opName.replace(/\w\S*/g, txt => {
         return txt.charAt(0).toUpperCase() + txt.substr(1);
-    }).replace(/[\s-()/./]/g, "");
+    }).replace(/[\s-()./]/g, "");
 
 
     const template = `/**
@@ -130,8 +130,8 @@ prompt.get(schema, (err, result) => {
  * @license Apache-2.0
  */
 
-import Operation from "../Operation";
-import OperationError from "../errors/OperationError";
+import Operation from "../Operation.mjs";
+import OperationError from "../errors/OperationError.mjs";
 
 /**
  * ${result.opName} operation
@@ -208,7 +208,7 @@ ${result.highlight ? `
 export default ${moduleName};
 `;
 
-    //console.log(template);
+    // console.log(template);
 
     const filename = path.join(dir, `./${moduleName}.mjs`);
     if (fs.existsSync(filename)) {

src/core/dishTypes/DishBigNumber.mjs

@@ -0,0 +1,38 @@
+/**
+ * @author d98762625 [d98762625@gmail.com]
+ * @copyright Crown Copyright 2019
+ * @license Apache-2.0
+ */
+
+import DishType from "./DishType.mjs";
+import Utils from "../Utils.mjs";
+import BigNumber from "bignumber.js";
+
+/**
+ * translation methods for BigNumber Dishes
+ */
+class DishBigNumber extends DishType {
+
+    /**
+     * convert the given value to a ArrayBuffer
+     * @param {BigNumber} value
+     */
+    static toArrayBuffer() {
+        DishBigNumber.checkForValue(this.value);
+        this.value = BigNumber.isBigNumber(this.value) ? Utils.strToArrayBuffer(this.value.toFixed()) : new ArrayBuffer;
+    }
+
+    /**
+     * convert the given value from a ArrayBuffer
+     */
+    static fromArrayBuffer() {
+        DishBigNumber.checkForValue(this.value);
+        try {
+            this.value = new BigNumber(Utils.arrayBufferToStr(this.value));
+        } catch (err) {
+            this.value = new BigNumber(NaN);
+        }
+    }
+}
+
+export default DishBigNumber;

src/core/dishTypes/DishByteArray.mjs

@@ -0,0 +1,31 @@
+/**
+ * @author d98762625 [d98762625@gmail.com]
+ * @copyright Crown Copyright 2019
+ * @license Apache-2.0
+ */
+
+import DishType from "./DishType.mjs";
+
+/**
+ * Translation methods for ArrayBuffer Dishes
+ */
+class DishByteArray extends DishType {
+
+    /**
+     * convert the given value to a ArrayBuffer
+     */
+    static toArrayBuffer() {
+        DishByteArray.checkForValue(this.value);
+        this.value = new Uint8Array(this.value).buffer;
+    }
+
+    /**
+     * convert the given value from a ArrayBuffer
+     */
+    static fromArrayBuffer() {
+        DishByteArray.checkForValue(this.value);
+        this.value = Array.prototype.slice.call(new Uint8Array(this.value));
+    }
+}
+
+export default DishByteArray;

src/core/dishTypes/DishFile.mjs

@@ -0,0 +1,42 @@
+/**
+ * @author d98762625 [d98762625@gmail.com]
+ * @copyright Crown Copyright 2019
+ * @license Apache-2.0
+ */
+
+import DishType from "./DishType.mjs";
+import Utils, { isNodeEnvironment } from "../Utils.mjs";
+
+/**
+ * Translation methods for file Dishes
+ */
+class DishFile extends DishType {
+
+    /**
+     * convert the given value to an ArrayBuffer
+     * @param {File} value
+     */
+    static toArrayBuffer() {
+        DishFile.checkForValue(this.value);
+        if (isNodeEnvironment()) {
+            this.value = Utils.readFileSync(this.value);
+        } else {
+            return new Promise((resolve, reject) => {
+                Utils.readFile(this.value)
+                    .then(v => this.value = v.buffer)
+                    .then(resolve)
+                    .catch(reject);
+            });
+        }
+    }
+
+    /**
+     * convert the given value from an ArrayBuffer
+     */
+    static fromArrayBuffer() {
+        DishFile.checkForValue(this.value);
+        this.value = new File(this.value, "unknown");
+    }
+}
+
+export default DishFile;

src/core/dishTypes/DishHTML.mjs

@@ -0,0 +1,26 @@
+/**
+ * @author d98762625 [d98762625@gmail.com]
+ * @copyright Crown Copyright 2019
+ * @license Apache-2.0
+ */
+
+import DishString from "./DishString.mjs";
+import Utils from "../Utils.mjs";
+
+/**
+ * Translation methods for HTML Dishes
+ */
+class DishHTML extends DishString {
+
+    /**
+     * convert the given value to a ArrayBuffer
+     * @param {String} value
+     */
+    static toArrayBuffer() {
+        DishHTML.checkForValue(this.value);
+        this.value = this.value ? Utils.strToArrayBuffer(Utils.unescapeHtml(Utils.stripHtmlTags(this.value, true))) : new ArrayBuffer;
+    }
+
+}
+
+export default DishHTML;

src/core/dishTypes/DishJSON.mjs

@@ -0,0 +1,32 @@
+/**
+ * @author d98762625 [d98762625@gmail.com]
+ * @copyright Crown Copyright 2019
+ * @license Apache-2.0
+ */
+
+import DishType from "./DishType.mjs";
+import Utils from "../Utils.mjs";
+
+/**
+ * Translation methods for JSON dishes
+ */
+class DishJSON extends DishType {
+
+    /**
+     * convert the given value to a ArrayBuffer
+     */
+    static toArrayBuffer() {
+        DishJSON.checkForValue(this.value);
+        this.value = this.value !== undefined ? Utils.strToArrayBuffer(JSON.stringify(this.value, null, 4)) : new ArrayBuffer;
+    }
+
+    /**
+     * convert the given value from a ArrayBuffer
+     */
+    static fromArrayBuffer() {
+        DishJSON.checkForValue(this.value);
+        this.value = JSON.parse(Utils.arrayBufferToStr(this.value));
+    }
+}
+
+export default DishJSON;

src/core/dishTypes/DishListFile.mjs

@@ -0,0 +1,80 @@
+/**
+ * @author d98762625 [d98762625@gmail.com]
+ * @copyright Crown Copyright 2019
+ * @license Apache-2.0
+ */
+
+import DishType from "./DishType.mjs";
+import Utils, { isNodeEnvironment } from "../Utils.mjs";
+
+
+/**
+ * Translation methods for ListFile Dishes
+ */
+class DishListFile extends DishType {
+
+    /**
+     * convert the given value to a ArrayBuffer
+     */
+    static async toArrayBuffer() {
+        DishListFile.checkForValue(this.value);
+
+        if (isNodeEnvironment()) {
+            this.value = this.value.map(file => Uint8Array.from(file.data));
+        } else {
+            this.value = await DishListFile.concatenateTypedArraysWithTypedElements(...this.value);
+        }
+    }
+
+    /**
+     * convert the given value from a ArrayBuffer
+     */
+    static fromArrayBuffer() {
+        DishListFile.checkForValue(this.value);
+        this.value = [new File(this.value, "unknown")];
+    }
+
+    /**
+     * Concatenates a list of typed elements together.
+     *
+     * @param {Uint8Array[]} arrays
+     * @returns {Uint8Array}
+     */
+    static async concatenateTypedArraysWithTypedElements(...arrays) {
+        let totalLength = 0;
+        for (const arr of arrays) {
+            totalLength += arr.size;
+        }
+        const myArray = new Uint8Array(totalLength);
+
+        let offset = 0;
+        for (const arr of arrays) {
+            const data = await Utils.readFile(arr);
+            myArray.set(data, offset);
+            offset += data.length;
+        }
+        return myArray;
+    }
+
+    /**
+     * Concatenates a list of Uint8Arrays together
+     *
+     * @param {Uint8Array[]} arrays
+     * @returns {Uint8Array}
+     */
+    static concatenateTypedArrays(...arrays) {
+        let totalLength = 0;
+        for (const arr of arrays) {
+            totalLength += arr.length;
+        }
+        const result = new Uint8Array(totalLength);
+        let offset = 0;
+        for (const arr of arrays) {
+            result.set(arr, offset);
+            offset += arr.length;
+        }
+        return result;
+    }
+}
+
+export default DishListFile;

src/core/dishTypes/DishNumber.mjs

@@ -0,0 +1,33 @@
+/**
+ * @author d98762625 [d98762625@gmail.com]
+ * @copyright Crown Copyright 2019
+ * @license Apache-2.0
+ */
+
+
+import DishType from "./DishType.mjs";
+import Utils from "../Utils.mjs";
+
+/**
+ * Translation methods for number dishes
+ */
+class DishNumber extends DishType {
+
+    /**
+     * convert the given value to a ArrayBuffer
+     */
+    static toArrayBuffer() {
+        DishNumber.checkForValue(this.value);
+        this.value = typeof this.value === "number" ? Utils.strToArrayBuffer(this.value.toString()) : new ArrayBuffer;
+    }
+
+    /**
+     * convert the given value from a ArrayBuffer
+     */
+    static fromArrayBuffer() {
+        DishNumber.checkForValue(this.value);
+        this.value = this.value ? parseFloat(Utils.arrayBufferToStr(this.value)) : 0;
+    }
+}
+
+export default DishNumber;

src/core/dishTypes/DishString.mjs

@@ -0,0 +1,33 @@
+/**
+ * @author d98762625 [d98762625@gmail.com]
+ * @copyright Crown Copyright 2019
+ * @license Apache-2.0
+ */
+
+
+import DishType from "./DishType.mjs";
+import Utils from "../Utils.mjs";
+
+/**
+ * Translation methods for string dishes
+ */
+class DishString extends DishType {
+
+    /**
+     * convert the given value to a ArrayBuffer
+     */
+    static toArrayBuffer() {
+        DishString.checkForValue(this.value);
+        this.value = this.value ? Utils.strToArrayBuffer(this.value) : new ArrayBuffer;
+    }
+
+    /**
+     * convert the given value from a ArrayBuffer
+     */
+    static fromArrayBuffer() {
+        DishString.checkForValue(this.value);
+        this.value = this.value ? Utils.arrayBufferToStr(this.value) : "";
+    }
+}
+
+export default DishString;

src/core/dishTypes/DishType.mjs

@@ -0,0 +1,38 @@
+/**
+ * @author d98762625 [d98762625@gmail.com]
+ * @copyright Crown Copyright 2019
+ * @license Apache-2.0
+ */
+
+
+/**
+ * Abstract class for dish translation methods
+ */
+class DishType {
+
+    /**
+     * Warn translations dont work without value from bind
+     */
+    static checkForValue(value) {
+        if (value === undefined) {
+            throw new Error("only use translation methods with .bind");
+        }
+    }
+
+    /**
+     * convert the given value to a ArrayBuffer
+     * @param {*} value
+     */
+    static toArrayBuffer() {
+        throw new Error("toArrayBuffer has not been implemented");
+    }
+
+    /**
+     * convert the given value from a ArrayBuffer
+     */
+    static fromArrayBuffer() {
+        throw new Error("fromArrayBuffer has not been implemented");
+    }
+}
+
+export default DishType;

src/core/dishTypes/index.mjs

@@ -0,0 +1,26 @@
+/**
+ * @author d98762625 [d98762625@gmail.com]
+ * @copyright Crown Copyright 2019
+ * @license Apache-2.0
+ */
+
+
+import DishByteArray from "./DishByteArray.mjs";
+import DishBigNumber from "./DishBigNumber.mjs";
+import DishFile from "./DishFile.mjs";
+import DishHTML from "./DishHTML.mjs";
+import DishJSON from "./DishJSON.mjs";
+import DishListFile from "./DishListFile.mjs";
+import DishNumber from "./DishNumber.mjs";
+import DishString from "./DishString.mjs";
+
+export {
+    DishByteArray,
+    DishBigNumber,
+    DishFile,
+    DishHTML,
+    DishJSON,
+    DishListFile,
+    DishNumber,
+    DishString,
+};

src/core/errors/ExcludedOperationError.mjs

@@ -0,0 +1,25 @@
+/**
+ * Custom error type for handling operation that isnt included in node.js API
+ *
+ * @author d98762625 [d98762625@gmail.com]
+ * @copyright Crown Copyright 2018
+ * @license Apache-2.0
+ */
+class ExcludedOperationError extends Error {
+    /**
+     * Standard error constructor. Adds no new behaviour.
+     *
+     * @param args - Standard error args
+     */
+    constructor(...args) {
+        super(...args);
+
+        this.type = "ExcludedOperationError";
+
+        if (Error.captureStackTrace) {
+            Error.captureStackTrace(this, ExcludedOperationError);
+        }
+    }
+}
+
+export default ExcludedOperationError;

src/core/errors/index.mjs

@@ -0,0 +1,9 @@
+import OperationError from "./OperationError.mjs";
+import DishError from "./DishError.mjs";
+import ExcludedOperationError from "./ExcludedOperationError.mjs";
+
+export {
+    OperationError,
+    DishError,
+    ExcludedOperationError,
+};

src/core/lib/Arithmetic.mjs

@@ -5,7 +5,7 @@
  * @license Apache-2.0
  */
 
-import Utils from "../Utils";
+import Utils from "../Utils.mjs";
 import BigNumber from "bignumber.js";
 
 
@@ -109,7 +109,7 @@ export function mean(data) {
  */
 export function median(data) {
     if ((data.length % 2) === 0 && data.length > 0) {
-        data.sort(function(a, b){
+        data.sort(function(a, b) {
             return a.minus(b);
         });
         const first = data[Math.floor(data.length / 2)];

src/core/lib/BCD.mjs

@@ -22,7 +22,7 @@ export const ENCODING_SCHEME = [
 /**
  * Lookup table for the binary value of each digit representation.
  *
- * I wrote a very nice algorithm to generate 8 4 2 1 encoding programatically,
+ * I wrote a very nice algorithm to generate 8 4 2 1 encoding programmatically,
  * but unfortunately it's much easier (if less elegant) to use lookup tables
  * when supporting multiple encoding schemes.
  *

src/core/lib/Bacon.mjs

@@ -0,0 +1,66 @@
+/**
+ * Bacon Cipher resources.
+ *
+ * @author Karsten Silkenbäumer [github.com/kassi]
+ * @copyright Karsten Silkenbäumer 2019
+ * @license Apache-2.0
+ */
+
+/**
+ * Bacon definitions.
+ */
+export const BACON_ALPHABETS = {
+    "Standard (I=J and U=V)": {
+        alphabet: "ABCDEFGHIKLMNOPQRSTUWXYZ",
+        codes: [0, 1, 2, 3, 4, 5, 6, 7, 8, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 19, 20, 21, 22, 23]
+    },
+    "Complete": {
+        alphabet: "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+    }
+};
+export const BACON_TRANSLATION_01 = "0/1";
+export const BACON_TRANSLATION_AB = "A/B";
+export const BACON_TRANSLATION_CASE = "Case";
+export const BACON_TRANSLATION_AMNZ = "A-M/N-Z first letter";
+export const BACON_TRANSLATIONS = [
+    BACON_TRANSLATION_01,
+    BACON_TRANSLATION_AB,
+    BACON_TRANSLATION_CASE,
+    BACON_TRANSLATION_AMNZ,
+];
+export const BACON_TRANSLATIONS_FOR_ENCODING = [
+    BACON_TRANSLATION_01,
+    BACON_TRANSLATION_AB
+];
+export const BACON_CLEARER_MAP = {
+    [BACON_TRANSLATION_01]: /[^01]/g,
+    [BACON_TRANSLATION_AB]: /[^ABab]/g,
+    [BACON_TRANSLATION_CASE]: /[^A-Za-z]/g,
+};
+export const BACON_NORMALIZE_MAP = {
+    [BACON_TRANSLATION_AB]: {
+        "A": "0",
+        "B": "1",
+        "a": "0",
+        "b": "1"
+    },
+};
+
+/**
+ * Swaps zeros to ones and ones to zeros.
+ *
+ * @param {string} data
+ * @returns {string}
+ *
+ * @example
+ * // returns "11001 01010"
+ * swapZeroAndOne("00110 10101");
+ */
+export function swapZeroAndOne(string) {
+    return string.replace(/[01]/g, function (c) {
+        return {
+            "0": "1",
+            "1": "0"
+        }[c];
+    });
+}

src/core/lib/Base45.mjs

@@ -0,0 +1,27 @@
+/**
+ * Base45 resources.
+ *
+ * @author Thomas Weißschuh [thomas@t-8ch.de]
+ * @copyright Crown Copyright 2021
+ * @license Apache-2.0
+ */
+
+/**
+ * Highlight to Base45
+ */
+export function highlightToBase45(pos, args) {
+    pos[0].start = Math.floor(pos[0].start / 2) * 3;
+    pos[0].end = Math.ceil(pos[0].end / 2) * 3;
+    return pos;
+}
+
+/**
+ * Highlight from Base45
+ */
+export function highlightFromBase45(pos, args) {
+    pos[0].start = Math.floor(pos[0].start / 3) * 2;
+    pos[0].end = Math.ceil(pos[0].end / 3) * 2;
+    return pos;
+}
+
+export const ALPHABET = "0-9A-Z $%*+\\-./:";

src/core/lib/Base58.mjs

@@ -6,13 +6,13 @@
  * @license Apache-2.0
  */
 
-import Utils from "../Utils";
-
+import Utils from "../Utils.mjs";
+import OperationError from "../errors/OperationError.mjs";
 
 /**
  * Base64's the input byte array using the given alphabet, returning a string.
  *
- * @param {byteArray|Uint8Array|string} data
+ * @param {byteArray|Uint8Array|ArrayBuffer|string} data
  * @param {string} [alphabet="A-Za-z0-9+/="]
  * @returns {string}
  *
@@ -26,10 +26,16 @@ import Utils from "../Utils";
 export function toBase64(data, alphabet="A-Za-z0-9+/=") {
     if (!data) return "";
     if (typeof data == "string") {
-        data = Utils.strToByteArray(data);
+        data = Utils.strToArrayBuffer(data);
+    }
+    if (data instanceof ArrayBuffer) {
+        data = new Uint8Array(data);
     }
 
     alphabet = Utils.expandAlphRange(alphabet).join("");
+    if (alphabet.length !== 64 && alphabet.length !== 65) { // Allow for padding
+        throw new OperationError(`Invalid Base64 alphabet length (${alphabet.length}): ${alphabet}`);
+    }
 
     let output = "",
         chr1, chr2, chr3,
@@ -63,7 +69,7 @@ export function toBase64(data, alphabet="A-Za-z0-9+/=") {
 /**
  * UnBase64's the input string using the given alphabet, returning a byte array.
  *
- * @param {byteArray} data
+ * @param {string} data
  * @param {string} [alphabet="A-Za-z0-9+/="]
  * @param {string} [returnType="string"] - Either "string" or "byteArray"
  * @param {boolean} [removeNonAlphChars=true]
@@ -76,7 +82,7 @@ export function toBase64(data, alphabet="A-Za-z0-9+/=") {
  * // returns [72, 101, 108, 108, 111]
  * fromBase64("SGVsbG8=", null, "byteArray");
  */
-export function fromBase64(data, alphabet="A-Za-z0-9+/=", returnType="string", removeNonAlphChars=true) {
+export function fromBase64(data, alphabet="A-Za-z0-9+/=", returnType="string", removeNonAlphChars=true, strictMode=false) {
     if (!data) {
         return returnType === "string" ? "" : [];
     }
@@ -84,36 +90,67 @@ export function fromBase64(data, alphabet="A-Za-z0-9+/=", returnType="string", r
     alphabet = alphabet || "A-Za-z0-9+/=";
     alphabet = Utils.expandAlphRange(alphabet).join("");
 
-    const output = [];
-    let chr1, chr2, chr3,
-        enc1, enc2, enc3, enc4,
-        i = 0;
+    // Confirm alphabet is a valid length
+    if (alphabet.length !== 64 && alphabet.length !== 65) { // Allow for padding
+        throw new OperationError(`Error: Base64 alphabet should be 64 characters long, or 65 with a padding character. Found ${alphabet.length}: ${alphabet}`);
+    }
 
+    // Remove non-alphabet characters
     if (removeNonAlphChars) {
         const re = new RegExp("[^" + alphabet.replace(/[[\]\\\-^$]/g, "\\$&") + "]", "g");
         data = data.replace(re, "");
     }
 
-    while (i < data.length) {
-        enc1 = alphabet.indexOf(data.charAt(i++));
-        enc2 = alphabet.indexOf(data.charAt(i++) || "=");
-        enc3 = alphabet.indexOf(data.charAt(i++) || "=");
-        enc4 = alphabet.indexOf(data.charAt(i++) || "=");
+    if (strictMode) {
+        // Check for incorrect lengths (even without padding)
+        if (data.length % 4 === 1) {
+            throw new OperationError(`Error: Invalid Base64 input length (${data.length}). Cannot be 4n+1, even without padding chars.`);
+        }
+
+        if (alphabet.length === 65) { // Padding character included
+            const pad = alphabet.charAt(64);
+            const padPos = data.indexOf(pad);
+            if (padPos >= 0) {
+                // Check that the padding character is only used at the end and maximum of twice
+                if (padPos < data.length - 2 || data.charAt(data.length - 1) !== pad) {
+                    throw new OperationError(`Error: Base64 padding character (${pad}) not used in the correct place.`);
+                }
+
+                // Check that input is padded to the correct length
+                if (data.length % 4 !== 0) {
+                    throw new OperationError("Error: Base64 not padded to a multiple of 4.");
+                }
+            }
+        }
+    }
 
-        enc2 = enc2 === -1 ? 64 : enc2;
-        enc3 = enc3 === -1 ? 64 : enc3;
-        enc4 = enc4 === -1 ? 64 : enc4;
+    const output = [];
+    let chr1, chr2, chr3,
+        enc1, enc2, enc3, enc4,
+        i = 0;
+
+    while (i < data.length) {
+        // Including `|| null` forces empty strings to null so that indexOf returns -1 instead of 0
+        enc1 = alphabet.indexOf(data.charAt(i++) || null);
+        enc2 = alphabet.indexOf(data.charAt(i++) || null);
+        enc3 = alphabet.indexOf(data.charAt(i++) || null);
+        enc4 = alphabet.indexOf(data.charAt(i++) || null);
+
+        if (strictMode && (enc1 < 0 || enc2 < 0 || enc3 < 0 || enc4 < 0)) {
+            throw new OperationError("Error: Base64 input contains non-alphabet char(s)");
+        }
 
         chr1 = (enc1 << 2) | (enc2 >> 4);
         chr2 = ((enc2 & 15) << 4) | (enc3 >> 2);
         chr3 = ((enc3 & 3) << 6) | enc4;
 
-        output.push(chr1);
-
-        if (enc3 !== 64) {
+        if (chr1 >= 0 && chr1 < 256) {
+            output.push(chr1);
+        }
+        if (chr2 >= 0 && chr2 < 256 && enc3 !== 64) {
             output.push(chr2);
         }
-        if (enc4 !== 64) {
+        if (chr3 >= 0 && chr3 < 256 && enc4 !== 64) {
             output.push(chr3);
         }
     }
@@ -139,4 +176,8 @@ export const ALPHABET_OPTIONS = [
     {name: "BinHex: !-,-0-689@A-NP-VX-Z[`a-fh-mp-r", value: "!-,-0-689@A-NP-VX-Z[`a-fh-mp-r"},
     {name: "ROT13: N-ZA-Mn-za-m0-9+/=", value: "N-ZA-Mn-za-m0-9+/="},
     {name: "UNIX crypt: ./0-9A-Za-z", value: "./0-9A-Za-z"},
+    {name: "Atom128: /128GhIoPQROSTeUbADfgHijKLM+n0pFWXY456xyzB7=39VaqrstJklmNuZvwcdEC", value: "/128GhIoPQROSTeUbADfgHijKLM+n0pFWXY456xyzB7=39VaqrstJklmNuZvwcdEC"},
+    {name: "Megan35: 3GHIJKLMNOPQRSTUb=cdefghijklmnopWXYZ/12+406789VaqrstuvwxyzABCDEF5", value: "3GHIJKLMNOPQRSTUb=cdefghijklmnopWXYZ/12+406789VaqrstuvwxyzABCDEF5"},
+    {name: "Zong22: ZKj9n+yf0wDVX1s/5YbdxSo=ILaUpPBCHg8uvNO4klm6iJGhQ7eFrWczAMEq3RTt2", value: "ZKj9n+yf0wDVX1s/5YbdxSo=ILaUpPBCHg8uvNO4klm6iJGhQ7eFrWczAMEq3RTt2"},
+    {name: "Hazz15: HNO4klm6ij9n+J2hyf0gzA8uvwDEq3X1Q7ZKeFrWcVTts/MRGYbdxSo=ILaUpPBC5", value: "HNO4klm6ij9n+J2hyf0gzA8uvwDEq3X1Q7ZKeFrWcVTts/MRGYbdxSo=ILaUpPBC5"}
 ];

src/core/lib/Base85.mjs

@@ -1,3 +1,5 @@
+import Utils from "../Utils.mjs";
+
 /**
  * Base85 resources.
  *
@@ -20,7 +22,7 @@ export const ALPHABET_OPTIONS = [
     },
     {
         name: "IPv6",
-        value: "0-9A-Za-z!#$%&()*+\\-;<=>?@^_`{|~}",
+        value: "0-9A-Za-z!#$%&()*+\\-;<=>?@^_`{|}~",
     }
 ];
 
@@ -32,13 +34,12 @@ export const ALPHABET_OPTIONS = [
  * @returns {string}
  */
 export function alphabetName(alphabet) {
-    alphabet = alphabet.replace("'", "&apos;");
-    alphabet = alphabet.replace("\"", "&quot;");
-    alphabet = alphabet.replace("\\", "&bsol;");
+    alphabet = escape(alphabet);
     let name;
 
     ALPHABET_OPTIONS.forEach(function(a) {
-        if (escape(alphabet) === escape(a.value)) name = a.name;
+        const expanded = Utils.expandAlphRange(a.value).join("");
+        if (alphabet === escape(expanded)) name = a.name;
     });
 
     return name;

src/core/lib/Base92.mjs

@@ -0,0 +1,44 @@
+/**
+ * Base92 resources.
+ *
+ * @author sg5506844 [sg5506844@gmail.com]
+ * @copyright Crown Copyright 2021
+ * @license Apache-2.0
+ */
+
+import OperationError from "../errors/OperationError.mjs";
+
+/**
+ * Base92 alphabet char
+ *
+ * @param {number} val
+ * @returns {number}
+ */
+export function base92Chr(val) {
+    if (val < 0 || val >= 91) {
+        throw new OperationError("Invalid value");
+    }
+    if (val === 0)
+        return "!".charCodeAt(0);
+    else if (val <= 61)
+        return "#".charCodeAt(0) + val - 1;
+    else
+        return "a".charCodeAt(0) + val - 62;
+}
+
+/**
+ * Base92 alphabet ord
+ *
+ * @param {string} val
+ * @returns {number}
+ */
+export function base92Ord(val) {
+    if (val === "!")
+        return 0;
+    else if ("#" <= val && val <= "_")
+        return val.charCodeAt(0) - "#".charCodeAt(0) + 1;
+    else if ("a" <= val && val <= "}")
+        return val.charCodeAt(0) - "a".charCodeAt(0) + 62;
+    throw new OperationError(`${val} is not a base92 character`);
+}
+

src/core/lib/Binary.mjs

@@ -6,39 +6,46 @@
  * @license Apache-2.0
  */
 
-import Utils from "../Utils";
+import Utils from "../Utils.mjs";
+import OperationError from "../errors/OperationError.mjs";
 
 
 /**
  * Convert a byte array into a binary string.
  *
- * @param {Uint8Array|byteArray} data
+ * @param {Uint8Array|byteArray|number} data
  * @param {string} [delim="Space"]
  * @param {number} [padding=8]
  * @returns {string}
  *
  * @example
- * // returns "00010000 00100000 00110000"
+ * // returns "00001010 00010100 00011110"
  * toBinary([10,20,30]);
  *
- * // returns "00010000 00100000 00110000"
- * toBinary([10,20,30], ":");
+ * // returns "00001010:00010100:00011110"
+ * toBinary([10,20,30], "Colon");
+ *
+ * // returns "1010:10100:11110"
+ * toBinary([10,20,30], "Colon", 0);
  */
 export function toBinary(data, delim="Space", padding=8) {
-    if (!data) return "";
+    if (data === undefined || data === null)
+        throw new OperationError("Unable to convert to binary: Empty input data enocuntered");
 
     delim = Utils.charRep(delim);
     let output = "";
 
-    for (let i = 0; i < data.length; i++) {
-        output += data[i].toString(2).padStart(padding, "0") + delim;
-    }
-
-    if (delim.length) {
-        return output.slice(0, -delim.length);
+    if (data.length) { // array
+        for (let i = 0; i < data.length; i++) {
+            output += data[i].toString(2).padStart(padding, "0");
+            if (i !== data.length - 1) output += delim;
+        }
+    } else if (typeof data === "number") { // Single value
+        return data.toString(2).padStart(padding, "0");
     } else {
-        return output;
+        return "";
     }
+    return output;
 }
 
 
@@ -52,12 +59,15 @@ export function toBinary(data, delim="Space", padding=8) {
  *
  * @example
  * // returns [10,20,30]
- * fromBinary("00010000 00100000 00110000");
+ * fromBinary("00001010 00010100 00011110");
  *
  * // returns [10,20,30]
- * fromBinary("00010000:00100000:00110000", "Colon");
+ * fromBinary("00001010:00010100:00011110", "Colon");
  */
 export function fromBinary(data, delim="Space", byteLen=8) {
+    if (byteLen < 1 || Math.round(byteLen) !== byteLen)
+        throw new OperationError("Byte length must be a positive integer");
+
     const delimRegex = Utils.regexRep(delim);
     data = data.replace(delimRegex, "");
 

src/core/lib/BitwiseOp.mjs

@@ -9,7 +9,7 @@
 /**
  * Runs bitwise operations across the input data.
  *
- * @param {byteArray} input
+ * @param {byteArray|Uint8Array} input
  * @param {byteArray} key
  * @param {function} func - The bitwise calculation to carry out
  * @param {boolean} nullPreserving
@@ -34,10 +34,10 @@ export function bitOp (input, key, func, nullPreserving, scheme) {
             !(nullPreserving && (o === 0 || o === k))) {
             switch (scheme) {
                 case "Input differential":
-                    key[i % key.length] = x;
+                    key[i % key.length] = o;
                     break;
                 case "Output differential":
-                    key[i % key.length] = o;
+                    key[i % key.length] = x;
                     break;
             }
         }

src/core/lib/Blowfish.mjs

@@ -0,0 +1,436 @@
+/**
+  Blowfish.js from Dojo Toolkit 1.8.1 (https://github.com/dojo/dojox/tree/1.8/encoding)
+  Extracted by Sladex (xslade@gmail.com)
+  Shoehorned into working with mjs for CyberChef by Matt C (matt@artemisbot.uk)
+  Refactored and implemented modes support by cbeuw (cbeuw.andy@gmail.com)
+
+  @license BSD
+  ========================================================================
+  The "New" BSD License:
+  **********************
+
+  Copyright (c) 2005-2016, The Dojo Foundation
+  All rights reserved.
+
+  Redistribution and use in source and binary forms, with or without
+  modification, are permitted provided that the following conditions are met:
+
+  * Redistributions of source code must retain the above copyright notice, this
+    list of conditions and the following disclaimer.
+  * Redistributions in binary form must reproduce the above copyright notice,
+    this list of conditions and the following disclaimer in the documentation
+    and/or other materials provided with the distribution.
+  * Neither the name of the Dojo Foundation nor the names of its contributors
+    may be used to endorse or promote products derived from this software
+    without specific prior written permission.
+
+  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+  ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+  WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+  DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+  FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+  DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+  SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+  CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+  OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+const crypto = {};
+
+import forge from "node-forge";
+
+
+/* dojo-release-1.8.1/dojo/_base/lang.js.uncompressed.js */
+
+const lang = {};
+lang.isString = function(it) {
+	// summary:
+	//		Return true if it is a String
+	// it: anything
+	//		Item to test.
+    return (typeof it == "string" || it instanceof String); // Boolean
+};
+
+
+/* dojo-release-1.8.1/dojo/_base/array.js.uncompressed.js */
+
+const arrayUtil = {};
+arrayUtil.map = function(arr, callback, thisObject, Ctr) {
+	// summary:
+	//		applies callback to each element of arr and returns
+	//		an Array with the results
+	// arr: Array|String
+	//		the array to iterate on. If a string, operates on
+	//		individual characters.
+	// callback: Function
+	//		a function is invoked with three arguments, (item, index,
+	//		array),	 and returns a value
+	// thisObject: Object?
+	//		may be used to scope the call to callback
+	// returns: Array
+	// description:
+	//		This function corresponds to the JavaScript 1.6 Array.map() method, with one difference: when
+	//		run over sparse arrays, this implementation passes the "holes" in the sparse array to
+	//		the callback function with a value of undefined. JavaScript 1.6's map skips the holes in the sparse array.
+	//		For more details, see:
+	//		https://developer.mozilla.org/en/Core_JavaScript_1.5_Reference/Objects/Array/map
+	// example:
+	//	| // returns [2, 3, 4, 5]
+	//	| array.map([1, 2, 3, 4], function(item){ return item+1 });
+
+	// TODO: why do we have a non-standard signature here? do we need "Ctr"?
+    let i = 0;
+    const l = arr && arr.length || 0, out = new (Ctr || Array)(l);
+    if (l && typeof arr == "string") arr = arr.split("");
+    if (thisObject) {
+        for (; i < l; ++i) {
+            out[i] = callback.call(thisObject, arr[i], i, arr);
+        }
+    } else {
+        for (; i < l; ++i) {
+            out[i] = callback(arr[i], i, arr);
+        }
+    }
+    return out; // Array
+};
+
+/* dojo-release-1.8.1/dojox/encoding/crypto/Blowfish.js.uncompressed.js */
+
+/*	Blowfish
+ *	Created based on the C# implementation by Marcus Hahn (http://www.hotpixel.net/)
+ *	Unsigned math based on Paul Johnstone and Peter Wood patches.
+ *	2005-12-08
+ */
+const boxes={
+    p: [
+        0x243f6a88, 0x85a308d3, 0x13198a2e, 0x03707344, 0xa4093822, 0x299f31d0, 0x082efa98, 0xec4e6c89,
+        0x452821e6, 0x38d01377, 0xbe5466cf, 0x34e90c6c, 0xc0ac29b7, 0xc97c50dd, 0x3f84d5b5, 0xb5470917,
+        0x9216d5d9, 0x8979fb1b
+    ],
+    s0: [
+        0xd1310ba6, 0x98dfb5ac, 0x2ffd72db, 0xd01adfb7, 0xb8e1afed, 0x6a267e96, 0xba7c9045, 0xf12c7f99,
+        0x24a19947, 0xb3916cf7, 0x0801f2e2, 0x858efc16, 0x636920d8, 0x71574e69, 0xa458fea3, 0xf4933d7e,
+        0x0d95748f, 0x728eb658, 0x718bcd58, 0x82154aee, 0x7b54a41d, 0xc25a59b5, 0x9c30d539, 0x2af26013,
+        0xc5d1b023, 0x286085f0, 0xca417918, 0xb8db38ef, 0x8e79dcb0, 0x603a180e, 0x6c9e0e8b, 0xb01e8a3e,
+        0xd71577c1, 0xbd314b27, 0x78af2fda, 0x55605c60, 0xe65525f3, 0xaa55ab94, 0x57489862, 0x63e81440,
+        0x55ca396a, 0x2aab10b6, 0xb4cc5c34, 0x1141e8ce, 0xa15486af, 0x7c72e993, 0xb3ee1411, 0x636fbc2a,
+        0x2ba9c55d, 0x741831f6, 0xce5c3e16, 0x9b87931e, 0xafd6ba33, 0x6c24cf5c, 0x7a325381, 0x28958677,
+        0x3b8f4898, 0x6b4bb9af, 0xc4bfe81b, 0x66282193, 0x61d809cc, 0xfb21a991, 0x487cac60, 0x5dec8032,
+        0xef845d5d, 0xe98575b1, 0xdc262302, 0xeb651b88, 0x23893e81, 0xd396acc5, 0x0f6d6ff3, 0x83f44239,
+        0x2e0b4482, 0xa4842004, 0x69c8f04a, 0x9e1f9b5e, 0x21c66842, 0xf6e96c9a, 0x670c9c61, 0xabd388f0,
+        0x6a51a0d2, 0xd8542f68, 0x960fa728, 0xab5133a3, 0x6eef0b6c, 0x137a3be4, 0xba3bf050, 0x7efb2a98,
+        0xa1f1651d, 0x39af0176, 0x66ca593e, 0x82430e88, 0x8cee8619, 0x456f9fb4, 0x7d84a5c3, 0x3b8b5ebe,
+        0xe06f75d8, 0x85c12073, 0x401a449f, 0x56c16aa6, 0x4ed3aa62, 0x363f7706, 0x1bfedf72, 0x429b023d,
+        0x37d0d724, 0xd00a1248, 0xdb0fead3, 0x49f1c09b, 0x075372c9, 0x80991b7b, 0x25d479d8, 0xf6e8def7,
+        0xe3fe501a, 0xb6794c3b, 0x976ce0bd, 0x04c006ba, 0xc1a94fb6, 0x409f60c4, 0x5e5c9ec2, 0x196a2463,
+        0x68fb6faf, 0x3e6c53b5, 0x1339b2eb, 0x3b52ec6f, 0x6dfc511f, 0x9b30952c, 0xcc814544, 0xaf5ebd09,
+        0xbee3d004, 0xde334afd, 0x660f2807, 0x192e4bb3, 0xc0cba857, 0x45c8740f, 0xd20b5f39, 0xb9d3fbdb,
+        0x5579c0bd, 0x1a60320a, 0xd6a100c6, 0x402c7279, 0x679f25fe, 0xfb1fa3cc, 0x8ea5e9f8, 0xdb3222f8,
+        0x3c7516df, 0xfd616b15, 0x2f501ec8, 0xad0552ab, 0x323db5fa, 0xfd238760, 0x53317b48, 0x3e00df82,
+        0x9e5c57bb, 0xca6f8ca0, 0x1a87562e, 0xdf1769db, 0xd542a8f6, 0x287effc3, 0xac6732c6, 0x8c4f5573,
+        0x695b27b0, 0xbbca58c8, 0xe1ffa35d, 0xb8f011a0, 0x10fa3d98, 0xfd2183b8, 0x4afcb56c, 0x2dd1d35b,
+        0x9a53e479, 0xb6f84565, 0xd28e49bc, 0x4bfb9790, 0xe1ddf2da, 0xa4cb7e33, 0x62fb1341, 0xcee4c6e8,
+        0xef20cada, 0x36774c01, 0xd07e9efe, 0x2bf11fb4, 0x95dbda4d, 0xae909198, 0xeaad8e71, 0x6b93d5a0,
+        0xd08ed1d0, 0xafc725e0, 0x8e3c5b2f, 0x8e7594b7, 0x8ff6e2fb, 0xf2122b64, 0x8888b812, 0x900df01c,
+        0x4fad5ea0, 0x688fc31c, 0xd1cff191, 0xb3a8c1ad, 0x2f2f2218, 0xbe0e1777, 0xea752dfe, 0x8b021fa1,
+        0xe5a0cc0f, 0xb56f74e8, 0x18acf3d6, 0xce89e299, 0xb4a84fe0, 0xfd13e0b7, 0x7cc43b81, 0xd2ada8d9,
+        0x165fa266, 0x80957705, 0x93cc7314, 0x211a1477, 0xe6ad2065, 0x77b5fa86, 0xc75442f5, 0xfb9d35cf,
+        0xebcdaf0c, 0x7b3e89a0, 0xd6411bd3, 0xae1e7e49, 0x00250e2d, 0x2071b35e, 0x226800bb, 0x57b8e0af,
+        0x2464369b, 0xf009b91e, 0x5563911d, 0x59dfa6aa, 0x78c14389, 0xd95a537f, 0x207d5ba2, 0x02e5b9c5,
+        0x83260376, 0x6295cfa9, 0x11c81968, 0x4e734a41, 0xb3472dca, 0x7b14a94a, 0x1b510052, 0x9a532915,
+        0xd60f573f, 0xbc9bc6e4, 0x2b60a476, 0x81e67400, 0x08ba6fb5, 0x571be91f, 0xf296ec6b, 0x2a0dd915,
+        0xb6636521, 0xe7b9f9b6, 0xff34052e, 0xc5855664, 0x53b02d5d, 0xa99f8fa1, 0x08ba4799, 0x6e85076a
+    ],
+    s1: [
+        0x4b7a70e9, 0xb5b32944, 0xdb75092e, 0xc4192623, 0xad6ea6b0, 0x49a7df7d, 0x9cee60b8, 0x8fedb266,
+        0xecaa8c71, 0x699a17ff, 0x5664526c, 0xc2b19ee1, 0x193602a5, 0x75094c29, 0xa0591340, 0xe4183a3e,
+        0x3f54989a, 0x5b429d65, 0x6b8fe4d6, 0x99f73fd6, 0xa1d29c07, 0xefe830f5, 0x4d2d38e6, 0xf0255dc1,
+        0x4cdd2086, 0x8470eb26, 0x6382e9c6, 0x021ecc5e, 0x09686b3f, 0x3ebaefc9, 0x3c971814, 0x6b6a70a1,
+        0x687f3584, 0x52a0e286, 0xb79c5305, 0xaa500737, 0x3e07841c, 0x7fdeae5c, 0x8e7d44ec, 0x5716f2b8,
+        0xb03ada37, 0xf0500c0d, 0xf01c1f04, 0x0200b3ff, 0xae0cf51a, 0x3cb574b2, 0x25837a58, 0xdc0921bd,
+        0xd19113f9, 0x7ca92ff6, 0x94324773, 0x22f54701, 0x3ae5e581, 0x37c2dadc, 0xc8b57634, 0x9af3dda7,
+        0xa9446146, 0x0fd0030e, 0xecc8c73e, 0xa4751e41, 0xe238cd99, 0x3bea0e2f, 0x3280bba1, 0x183eb331,
+        0x4e548b38, 0x4f6db908, 0x6f420d03, 0xf60a04bf, 0x2cb81290, 0x24977c79, 0x5679b072, 0xbcaf89af,
+        0xde9a771f, 0xd9930810, 0xb38bae12, 0xdccf3f2e, 0x5512721f, 0x2e6b7124, 0x501adde6, 0x9f84cd87,
+        0x7a584718, 0x7408da17, 0xbc9f9abc, 0xe94b7d8c, 0xec7aec3a, 0xdb851dfa, 0x63094366, 0xc464c3d2,
+        0xef1c1847, 0x3215d908, 0xdd433b37, 0x24c2ba16, 0x12a14d43, 0x2a65c451, 0x50940002, 0x133ae4dd,
+        0x71dff89e, 0x10314e55, 0x81ac77d6, 0x5f11199b, 0x043556f1, 0xd7a3c76b, 0x3c11183b, 0x5924a509,
+        0xf28fe6ed, 0x97f1fbfa, 0x9ebabf2c, 0x1e153c6e, 0x86e34570, 0xeae96fb1, 0x860e5e0a, 0x5a3e2ab3,
+        0x771fe71c, 0x4e3d06fa, 0x2965dcb9, 0x99e71d0f, 0x803e89d6, 0x5266c825, 0x2e4cc978, 0x9c10b36a,
+        0xc6150eba, 0x94e2ea78, 0xa5fc3c53, 0x1e0a2df4, 0xf2f74ea7, 0x361d2b3d, 0x1939260f, 0x19c27960,
+        0x5223a708, 0xf71312b6, 0xebadfe6e, 0xeac31f66, 0xe3bc4595, 0xa67bc883, 0xb17f37d1, 0x018cff28,
+        0xc332ddef, 0xbe6c5aa5, 0x65582185, 0x68ab9802, 0xeecea50f, 0xdb2f953b, 0x2aef7dad, 0x5b6e2f84,
+        0x1521b628, 0x29076170, 0xecdd4775, 0x619f1510, 0x13cca830, 0xeb61bd96, 0x0334fe1e, 0xaa0363cf,
+        0xb5735c90, 0x4c70a239, 0xd59e9e0b, 0xcbaade14, 0xeecc86bc, 0x60622ca7, 0x9cab5cab, 0xb2f3846e,
+        0x648b1eaf, 0x19bdf0ca, 0xa02369b9, 0x655abb50, 0x40685a32, 0x3c2ab4b3, 0x319ee9d5, 0xc021b8f7,
+        0x9b540b19, 0x875fa099, 0x95f7997e, 0x623d7da8, 0xf837889a, 0x97e32d77, 0x11ed935f, 0x16681281,
+        0x0e358829, 0xc7e61fd6, 0x96dedfa1, 0x7858ba99, 0x57f584a5, 0x1b227263, 0x9b83c3ff, 0x1ac24696,
+        0xcdb30aeb, 0x532e3054, 0x8fd948e4, 0x6dbc3128, 0x58ebf2ef, 0x34c6ffea, 0xfe28ed61, 0xee7c3c73,
+        0x5d4a14d9, 0xe864b7e3, 0x42105d14, 0x203e13e0, 0x45eee2b6, 0xa3aaabea, 0xdb6c4f15, 0xfacb4fd0,
+        0xc742f442, 0xef6abbb5, 0x654f3b1d, 0x41cd2105, 0xd81e799e, 0x86854dc7, 0xe44b476a, 0x3d816250,
+        0xcf62a1f2, 0x5b8d2646, 0xfc8883a0, 0xc1c7b6a3, 0x7f1524c3, 0x69cb7492, 0x47848a0b, 0x5692b285,
+        0x095bbf00, 0xad19489d, 0x1462b174, 0x23820e00, 0x58428d2a, 0x0c55f5ea, 0x1dadf43e, 0x233f7061,
+        0x3372f092, 0x8d937e41, 0xd65fecf1, 0x6c223bdb, 0x7cde3759, 0xcbee7460, 0x4085f2a7, 0xce77326e,
+        0xa6078084, 0x19f8509e, 0xe8efd855, 0x61d99735, 0xa969a7aa, 0xc50c06c2, 0x5a04abfc, 0x800bcadc,
+        0x9e447a2e, 0xc3453484, 0xfdd56705, 0x0e1e9ec9, 0xdb73dbd3, 0x105588cd, 0x675fda79, 0xe3674340,
+        0xc5c43465, 0x713e38d8, 0x3d28f89e, 0xf16dff20, 0x153e21e7, 0x8fb03d4a, 0xe6e39f2b, 0xdb83adf7
+    ],
+    s2: [
+        0xe93d5a68, 0x948140f7, 0xf64c261c, 0x94692934, 0x411520f7, 0x7602d4f7, 0xbcf46b2e, 0xd4a20068,
+        0xd4082471, 0x3320f46a, 0x43b7d4b7, 0x500061af, 0x1e39f62e, 0x97244546, 0x14214f74, 0xbf8b8840,
+        0x4d95fc1d, 0x96b591af, 0x70f4ddd3, 0x66a02f45, 0xbfbc09ec, 0x03bd9785, 0x7fac6dd0, 0x31cb8504,
+        0x96eb27b3, 0x55fd3941, 0xda2547e6, 0xabca0a9a, 0x28507825, 0x530429f4, 0x0a2c86da, 0xe9b66dfb,
+        0x68dc1462, 0xd7486900, 0x680ec0a4, 0x27a18dee, 0x4f3ffea2, 0xe887ad8c, 0xb58ce006, 0x7af4d6b6,
+        0xaace1e7c, 0xd3375fec, 0xce78a399, 0x406b2a42, 0x20fe9e35, 0xd9f385b9, 0xee39d7ab, 0x3b124e8b,
+        0x1dc9faf7, 0x4b6d1856, 0x26a36631, 0xeae397b2, 0x3a6efa74, 0xdd5b4332, 0x6841e7f7, 0xca7820fb,
+        0xfb0af54e, 0xd8feb397, 0x454056ac, 0xba489527, 0x55533a3a, 0x20838d87, 0xfe6ba9b7, 0xd096954b,
+        0x55a867bc, 0xa1159a58, 0xcca92963, 0x99e1db33, 0xa62a4a56, 0x3f3125f9, 0x5ef47e1c, 0x9029317c,
+        0xfdf8e802, 0x04272f70, 0x80bb155c, 0x05282ce3, 0x95c11548, 0xe4c66d22, 0x48c1133f, 0xc70f86dc,
+        0x07f9c9ee, 0x41041f0f, 0x404779a4, 0x5d886e17, 0x325f51eb, 0xd59bc0d1, 0xf2bcc18f, 0x41113564,
+        0x257b7834, 0x602a9c60, 0xdff8e8a3, 0x1f636c1b, 0x0e12b4c2, 0x02e1329e, 0xaf664fd1, 0xcad18115,
+        0x6b2395e0, 0x333e92e1, 0x3b240b62, 0xeebeb922, 0x85b2a20e, 0xe6ba0d99, 0xde720c8c, 0x2da2f728,
+        0xd0127845, 0x95b794fd, 0x647d0862, 0xe7ccf5f0, 0x5449a36f, 0x877d48fa, 0xc39dfd27, 0xf33e8d1e,
+        0x0a476341, 0x992eff74, 0x3a6f6eab, 0xf4f8fd37, 0xa812dc60, 0xa1ebddf8, 0x991be14c, 0xdb6e6b0d,
+        0xc67b5510, 0x6d672c37, 0x2765d43b, 0xdcd0e804, 0xf1290dc7, 0xcc00ffa3, 0xb5390f92, 0x690fed0b,
+        0x667b9ffb, 0xcedb7d9c, 0xa091cf0b, 0xd9155ea3, 0xbb132f88, 0x515bad24, 0x7b9479bf, 0x763bd6eb,
+        0x37392eb3, 0xcc115979, 0x8026e297, 0xf42e312d, 0x6842ada7, 0xc66a2b3b, 0x12754ccc, 0x782ef11c,
+        0x6a124237, 0xb79251e7, 0x06a1bbe6, 0x4bfb6350, 0x1a6b1018, 0x11caedfa, 0x3d25bdd8, 0xe2e1c3c9,
+        0x44421659, 0x0a121386, 0xd90cec6e, 0xd5abea2a, 0x64af674e, 0xda86a85f, 0xbebfe988, 0x64e4c3fe,
+        0x9dbc8057, 0xf0f7c086, 0x60787bf8, 0x6003604d, 0xd1fd8346, 0xf6381fb0, 0x7745ae04, 0xd736fccc,
+        0x83426b33, 0xf01eab71, 0xb0804187, 0x3c005e5f, 0x77a057be, 0xbde8ae24, 0x55464299, 0xbf582e61,
+        0x4e58f48f, 0xf2ddfda2, 0xf474ef38, 0x8789bdc2, 0x5366f9c3, 0xc8b38e74, 0xb475f255, 0x46fcd9b9,
+        0x7aeb2661, 0x8b1ddf84, 0x846a0e79, 0x915f95e2, 0x466e598e, 0x20b45770, 0x8cd55591, 0xc902de4c,
+        0xb90bace1, 0xbb8205d0, 0x11a86248, 0x7574a99e, 0xb77f19b6, 0xe0a9dc09, 0x662d09a1, 0xc4324633,
+        0xe85a1f02, 0x09f0be8c, 0x4a99a025, 0x1d6efe10, 0x1ab93d1d, 0x0ba5a4df, 0xa186f20f, 0x2868f169,
+        0xdcb7da83, 0x573906fe, 0xa1e2ce9b, 0x4fcd7f52, 0x50115e01, 0xa70683fa, 0xa002b5c4, 0x0de6d027,
+        0x9af88c27, 0x773f8641, 0xc3604c06, 0x61a806b5, 0xf0177a28, 0xc0f586e0, 0x006058aa, 0x30dc7d62,
+        0x11e69ed7, 0x2338ea63, 0x53c2dd94, 0xc2c21634, 0xbbcbee56, 0x90bcb6de, 0xebfc7da1, 0xce591d76,
+        0x6f05e409, 0x4b7c0188, 0x39720a3d, 0x7c927c24, 0x86e3725f, 0x724d9db9, 0x1ac15bb4, 0xd39eb8fc,
+        0xed545578, 0x08fca5b5, 0xd83d7cd3, 0x4dad0fc4, 0x1e50ef5e, 0xb161e6f8, 0xa28514d9, 0x6c51133c,
+        0x6fd5c7e7, 0x56e14ec4, 0x362abfce, 0xddc6c837, 0xd79a3234, 0x92638212, 0x670efa8e, 0x406000e0
+    ],
+    s3: [
+        0x3a39ce37, 0xd3faf5cf, 0xabc27737, 0x5ac52d1b, 0x5cb0679e, 0x4fa33742, 0xd3822740, 0x99bc9bbe,
+        0xd5118e9d, 0xbf0f7315, 0xd62d1c7e, 0xc700c47b, 0xb78c1b6b, 0x21a19045, 0xb26eb1be, 0x6a366eb4,
+        0x5748ab2f, 0xbc946e79, 0xc6a376d2, 0x6549c2c8, 0x530ff8ee, 0x468dde7d, 0xd5730a1d, 0x4cd04dc6,
+        0x2939bbdb, 0xa9ba4650, 0xac9526e8, 0xbe5ee304, 0xa1fad5f0, 0x6a2d519a, 0x63ef8ce2, 0x9a86ee22,
+        0xc089c2b8, 0x43242ef6, 0xa51e03aa, 0x9cf2d0a4, 0x83c061ba, 0x9be96a4d, 0x8fe51550, 0xba645bd6,
+        0x2826a2f9, 0xa73a3ae1, 0x4ba99586, 0xef5562e9, 0xc72fefd3, 0xf752f7da, 0x3f046f69, 0x77fa0a59,
+        0x80e4a915, 0x87b08601, 0x9b09e6ad, 0x3b3ee593, 0xe990fd5a, 0x9e34d797, 0x2cf0b7d9, 0x022b8b51,
+        0x96d5ac3a, 0x017da67d, 0xd1cf3ed6, 0x7c7d2d28, 0x1f9f25cf, 0xadf2b89b, 0x5ad6b472, 0x5a88f54c,
+        0xe029ac71, 0xe019a5e6, 0x47b0acfd, 0xed93fa9b, 0xe8d3c48d, 0x283b57cc, 0xf8d56629, 0x79132e28,
+        0x785f0191, 0xed756055, 0xf7960e44, 0xe3d35e8c, 0x15056dd4, 0x88f46dba, 0x03a16125, 0x0564f0bd,
+        0xc3eb9e15, 0x3c9057a2, 0x97271aec, 0xa93a072a, 0x1b3f6d9b, 0x1e6321f5, 0xf59c66fb, 0x26dcf319,
+        0x7533d928, 0xb155fdf5, 0x03563482, 0x8aba3cbb, 0x28517711, 0xc20ad9f8, 0xabcc5167, 0xccad925f,
+        0x4de81751, 0x3830dc8e, 0x379d5862, 0x9320f991, 0xea7a90c2, 0xfb3e7bce, 0x5121ce64, 0x774fbe32,
+        0xa8b6e37e, 0xc3293d46, 0x48de5369, 0x6413e680, 0xa2ae0810, 0xdd6db224, 0x69852dfd, 0x09072166,
+        0xb39a460a, 0x6445c0dd, 0x586cdecf, 0x1c20c8ae, 0x5bbef7dd, 0x1b588d40, 0xccd2017f, 0x6bb4e3bb,
+        0xdda26a7e, 0x3a59ff45, 0x3e350a44, 0xbcb4cdd5, 0x72eacea8, 0xfa6484bb, 0x8d6612ae, 0xbf3c6f47,
+        0xd29be463, 0x542f5d9e, 0xaec2771b, 0xf64e6370, 0x740e0d8d, 0xe75b1357, 0xf8721671, 0xaf537d5d,
+        0x4040cb08, 0x4eb4e2cc, 0x34d2466a, 0x0115af84, 0xe1b00428, 0x95983a1d, 0x06b89fb4, 0xce6ea048,
+        0x6f3f3b82, 0x3520ab82, 0x011a1d4b, 0x277227f8, 0x611560b1, 0xe7933fdc, 0xbb3a792b, 0x344525bd,
+        0xa08839e1, 0x51ce794b, 0x2f32c9b7, 0xa01fbac9, 0xe01cc87e, 0xbcc7d1f6, 0xcf0111c3, 0xa1e8aac7,
+        0x1a908749, 0xd44fbd9a, 0xd0dadecb, 0xd50ada38, 0x0339c32a, 0xc6913667, 0x8df9317c, 0xe0b12b4f,
+        0xf79e59b7, 0x43f5bb3a, 0xf2d519ff, 0x27d9459c, 0xbf97222c, 0x15e6fc2a, 0x0f91fc71, 0x9b941525,
+        0xfae59361, 0xceb69ceb, 0xc2a86459, 0x12baa8d1, 0xb6c1075e, 0xe3056a0c, 0x10d25065, 0xcb03a442,
+        0xe0ec6e0e, 0x1698db3b, 0x4c98a0be, 0x3278e964, 0x9f1f9532, 0xe0d392df, 0xd3a0342b, 0x8971f21e,
+        0x1b0a7441, 0x4ba3348c, 0xc5be7120, 0xc37632d8, 0xdf359f8d, 0x9b992f2e, 0xe60b6f47, 0x0fe3f11d,
+        0xe54cda54, 0x1edad891, 0xce6279cf, 0xcd3e7e6f, 0x1618b166, 0xfd2c1d05, 0x848fd2c5, 0xf6fb2299,
+        0xf523f357, 0xa6327623, 0x93a83531, 0x56cccd02, 0xacf08162, 0x5a75ebb5, 0x6e163697, 0x88d273cc,
+        0xde966292, 0x81b949d0, 0x4c50901b, 0x71c65614, 0xe6c6c7bd, 0x327a140a, 0x45e1d006, 0xc3f27b9a,
+        0xc9aa53fd, 0x62a80f00, 0xbb25bfe2, 0x35bdd2f6, 0x71126905, 0xb2040222, 0xb6cbcf7c, 0xcd769c2b,
+        0x53113ec0, 0x1640e3d3, 0x38abbd60, 0x2547adf0, 0xba38209c, 0xf746ce76, 0x77afa1c5, 0x20756060,
+        0x85cbfe4e, 0x8ae88dd8, 0x7aaaf9b0, 0x4cf9aa7e, 0x1948c25c, 0x02fb8a8c, 0x01c36ae4, 0xd6ebe1f9,
+        0x90d4f869, 0xa65cdea0, 0x3f09252d, 0xc208e69f, 0xb74e6132, 0xce77e25b, 0x578fdfe3, 0x3ac372e6
+    ]
+};
+
+
+////////////////////////////////////////////////////////////////////////////
+//	fixes based on patch submitted by Peter Wood (#5791)
+const xor = function(x, y) {
+    return (((x>>0x10)^(y>>0x10))<<0x10)|(((x&0xffff)^(y&0xffff))&0xffff);
+};
+
+
+const f = function(v, box) {
+    const d=box.s3[v&0xff]; v>>=8;
+    const c=box.s2[v&0xff]; v>>=8;
+    const b=box.s1[v&0xff]; v>>=8;
+    const a=box.s0[v&0xff];
+
+    let r = (((a>>0x10)+(b>>0x10)+(((a&0xffff)+(b&0xffff))>>0x10))<<0x10)|(((a&0xffff)+(b&0xffff))&0xffff);
+    r = (((r>>0x10)^(c>>0x10))<<0x10)|(((r&0xffff)^(c&0xffff))&0xffff);
+    return (((r>>0x10)+(d>>0x10)+(((r&0xffff)+(d&0xffff))>>0x10))<<0x10)|(((r&0xffff)+(d&0xffff))&0xffff);
+};
+
+
+const eb = function(o, box) {
+	//	TODO: see if this can't be made more efficient
+    let l=o.left;
+    let r=o.right;
+    l=xor(l, box.p[0]);
+    r=xor(r, xor(f(l, box), box.p[1]));
+    l=xor(l, xor(f(r, box), box.p[2]));
+    r=xor(r, xor(f(l, box), box.p[3]));
+    l=xor(l, xor(f(r, box), box.p[4]));
+    r=xor(r, xor(f(l, box), box.p[5]));
+    l=xor(l, xor(f(r, box), box.p[6]));
+    r=xor(r, xor(f(l, box), box.p[7]));
+    l=xor(l, xor(f(r, box), box.p[8]));
+    r=xor(r, xor(f(l, box), box.p[9]));
+    l=xor(l, xor(f(r, box), box.p[10]));
+    r=xor(r, xor(f(l, box), box.p[11]));
+    l=xor(l, xor(f(r, box), box.p[12]));
+    r=xor(r, xor(f(l, box), box.p[13]));
+    l=xor(l, xor(f(r, box), box.p[14]));
+    r=xor(r, xor(f(l, box), box.p[15]));
+    l=xor(l, xor(f(r, box), box.p[16]));
+    o.right=l;
+    o.left=xor(r, box.p[17]);
+};
+
+const db = function(o, box) {
+    let l=o.left;
+    let r=o.right;
+    l=xor(l, box.p[17]);
+    r=xor(r, xor(f(l, box), box.p[16]));
+    l=xor(l, xor(f(r, box), box.p[15]));
+    r=xor(r, xor(f(l, box), box.p[14]));
+    l=xor(l, xor(f(r, box), box.p[13]));
+    r=xor(r, xor(f(l, box), box.p[12]));
+    l=xor(l, xor(f(r, box), box.p[11]));
+    r=xor(r, xor(f(l, box), box.p[10]));
+    l=xor(l, xor(f(r, box), box.p[9]));
+    r=xor(r, xor(f(l, box), box.p[8]));
+    l=xor(l, xor(f(r, box), box.p[7]));
+    r=xor(r, xor(f(l, box), box.p[6]));
+    l=xor(l, xor(f(r, box), box.p[5]));
+    r=xor(r, xor(f(l, box), box.p[4]));
+    l=xor(l, xor(f(r, box), box.p[3]));
+    r=xor(r, xor(f(l, box), box.p[2]));
+    l=xor(l, xor(f(r, box), box.p[1]));
+    o.right=l;
+    o.left=xor(r, box.p[0]);
+};
+
+const encryptBlock=function(inblock, outblock, box) {
+    const o = {};
+    o.left=inblock[0];
+    o.right=inblock[1];
+    eb(o, box);
+    outblock[0] = o.left;
+    outblock[1] = o.right;
+};
+
+const decryptBlock=function(inblock, outblock, box) {
+    const o= {};
+    o.left=inblock[0];
+    o.right=inblock[1];
+    db(o, box);
+    outblock[0] = o.left;
+    outblock[1] = o.right;
+};
+
+crypto.Blowfish = new function() {
+    this.createCipher=function(key, modeName) {
+        return new forge.cipher.BlockCipher({
+            algorithm: new Blowfish.Algorithm(key, modeName),
+            key: key,
+            decrypt: false
+        });
+    };
+
+    this.createDecipher=function(key, modeName) {
+        return new forge.cipher.BlockCipher({
+            algorithm: new Blowfish.Algorithm(key, modeName),
+            key: key,
+            decrypt: true
+        });
+    };
+}();
+
+
+crypto.Blowfish.Algorithm=function(key, modeName) {
+    this.initialize({key: key});
+    const _box = this.box;
+    const modeOption = {
+        blockSize: 8,
+        cipher: {
+            encrypt: function(inblock, outblock) {
+                encryptBlock(inblock, outblock, _box);
+            },
+            decrypt: function(inblock, outblock) {
+                decryptBlock(inblock, outblock, _box);
+            },
+        }
+    };
+
+    switch (modeName.toLowerCase()) {
+        case "ecb":
+            this.mode=new forge.cipher.modes.ecb(modeOption);
+            break;
+        case "cbc":
+            this.mode=new forge.cipher.modes.cbc(modeOption);
+            break;
+        case "cfb":
+            this.mode=new forge.cipher.modes.cfb(modeOption);
+            break;
+        case "ofb":
+            this.mode=new forge.cipher.modes.ofb(modeOption);
+            break;
+        case "ctr":
+            this.mode=new forge.cipher.modes.ctr(modeOption);
+            break;
+        default:
+            this.mode=new forge.cipher.modes.ecb(modeOption);
+            break;
+    }
+
+};
+
+crypto.Blowfish.Algorithm.prototype.initialize=function(options) {
+    const POW8=Math.pow(2, 8);
+
+    let k=options.key;
+    if (lang.isString(k)) {
+        k = arrayUtil.map(k.split(""), function(item) {
+            return item.charCodeAt(0) & 0xff;
+        });
+    }
+
+	//	init the boxes
+    let pos=0, data=0;
+    const res={ left: 0, right: 0 };
+    const box = {
+        p: arrayUtil.map(boxes.p.slice(0), function(item) {
+            const l=k.length;
+            for (let j=0; j<4; j++) {
+                data=(data*POW8)|k[pos++ % l];
+            }
+            return (((item>>0x10)^(data>>0x10))<<0x10)|(((item&0xffff)^(data&0xffff))&0xffff);
+        }),
+        s0: boxes.s0.slice(0),
+        s1: boxes.s1.slice(0),
+        s2: boxes.s2.slice(0),
+        s3: boxes.s3.slice(0)
+    };
+
+	//	encrypt p and the s boxes
+    for (let i=0, l=box.p.length; i<l;) {
+        eb(res, box);
+        box.p[i++]=res.left;
+        box.p[i++]=res.right;
+    }
+    for (let i=0; i<4; i++) {
+        for (let j=0, l=box["s"+i].length; j<l;) {
+            eb(res, box);
+            box["s"+i][j++]=res.left;
+            box["s"+i][j++]=res.right;
+        }
+    }
+    this.box = box;
+};
+
+export const Blowfish = crypto.Blowfish;

src/core/lib/Bombe.mjs

@@ -0,0 +1,756 @@
+/**
+ * Emulation of the Bombe machine.
+ *
+ * @author s2224834
+ * @author The National Museum of Computing - Bombe Rebuild Project
+ * @copyright Crown Copyright 2019
+ * @license Apache-2.0
+ */
+
+import OperationError from "../errors/OperationError.mjs";
+import Utils from "../Utils.mjs";
+import {Rotor, Plugboard, a2i, i2a} from "./Enigma.mjs";
+
+/**
+ * Convenience/optimisation subclass of Rotor
+ *
+ * This allows creating multiple Rotors which share backing maps, to avoid repeatedly parsing the
+ * rotor spec strings and duplicating the maps in memory.
+ */
+class CopyRotor extends Rotor {
+    /**
+     * Return a copy of this Rotor.
+     * @returns {Object}
+     */
+    copy() {
+        const clone = {
+            map: this.map,
+            revMap: this.revMap,
+            pos: this.pos,
+            step: this.step,
+            transform: this.transform,
+            revTransform: this.revTransform,
+        };
+        return clone;
+    }
+}
+
+/**
+ * Node in the menu graph
+ *
+ * A node represents a cipher/plaintext letter.
+ */
+class Node {
+    /**
+     * Node constructor.
+     * @param {number} letter - The plain/ciphertext letter this node represents (as a number).
+     */
+    constructor(letter) {
+        this.letter = letter;
+        this.edges = new Set();
+        this.visited = false;
+    }
+}
+
+/**
+ * Edge in the menu graph
+ *
+ * An edge represents an Enigma machine transformation between two letters.
+ */
+class Edge {
+    /**
+     * Edge constructor - an Enigma machine mapping between letters
+     * @param {number} pos - The rotor position, relative to the beginning of the crib, at this edge
+     * @param {number} node1 - Letter at one end (as a number)
+     * @param {number} node2 - Letter at the other end
+     */
+    constructor(pos, node1, node2) {
+        this.pos = pos;
+        this.node1 = node1;
+        this.node2 = node2;
+        node1.edges.add(this);
+        node2.edges.add(this);
+        this.visited = false;
+    }
+
+    /**
+     * Given the node at one end of this edge, return the other end.
+     * @param node {number} - The node we have
+     * @returns {number}
+     */
+    getOther(node) {
+        return this.node1 === node ? this.node2 : this.node1;
+    }
+}
+
+/**
+ * As all the Bombe's rotors move in step, at any given point the vast majority of the scramblers
+ * in the machine share the majority of their state, which is hosted in this class.
+ */
+class SharedScrambler {
+    /**
+     * SharedScrambler constructor.
+     * @param {Object[]} rotors - List of rotors in the shared state _only_.
+     * @param {Object} reflector - The reflector in use.
+     */
+    constructor(rotors, reflector) {
+        this.lowerCache = new Array(26);
+        this.higherCache = new Array(26);
+        for (let i=0; i<26; i++) {
+            this.higherCache[i] = new Array(26);
+        }
+        this.changeRotors(rotors, reflector);
+    }
+
+    /**
+     * Replace the rotors and reflector in this SharedScrambler.
+     * This takes care of flushing caches as well.
+     * @param {Object[]} rotors - List of rotors in the shared state _only_.
+     * @param {Object} reflector - The reflector in use.
+     */
+    changeRotors(rotors, reflector) {
+        this.reflector = reflector;
+        this.rotors = rotors;
+        this.rotorsRev = [].concat(rotors).reverse();
+        this.cacheGen();
+    }
+
+    /**
+     * Step the rotors forward.
+     * @param {number} n - How many rotors to step. This includes the rotors which are not part of
+     * the shared state, so should be 2 or more.
+     */
+    step(n) {
+        for (let i=0; i<n-1; i++) {
+            this.rotors[i].step();
+        }
+        this.cacheGen();
+    }
+
+    /**
+     * Optimisation: We pregenerate all routes through the machine with the top rotor removed,
+     * as these rarely change. This saves a lot of lookups. This function generates this route
+     * table.
+     * We also just-in-time cache the full routes through the scramblers, because after stepping
+     * the fast rotor some scramblers will be in states occupied by other scrambles on previous
+     * iterations.
+     */
+    cacheGen() {
+        for (let i=0; i<26; i++) {
+            this.lowerCache[i] = undefined;
+            for (let j=0; j<26; j++) {
+                this.higherCache[i][j] = undefined;
+            }
+        }
+        for (let i=0; i<26; i++) {
+            if (this.lowerCache[i] !== undefined) {
+                continue;
+            }
+            let letter = i;
+            for (const rotor of this.rotors) {
+                letter = rotor.transform(letter);
+            }
+            letter = this.reflector.transform(letter);
+            for (const rotor of this.rotorsRev) {
+                letter = rotor.revTransform(letter);
+            }
+            // By symmetry
+            this.lowerCache[i] = letter;
+            this.lowerCache[letter] = i;
+        }
+    }
+
+    /**
+     * Map a letter through this (partial) scrambler.
+     * @param {number} i - The letter
+     * @returns {number}
+     */
+    transform(i) {
+        return this.lowerCache[i];
+    }
+}
+
+/**
+ * Scrambler.
+ *
+ * This is effectively just an Enigma machine, but it only operates on one character at a time and
+ * the stepping mechanism is different.
+ */
+class Scrambler {
+    /** Scrambler constructor.
+     * @param {Object} base - The SharedScrambler whose state this scrambler uses
+     * @param {Object} rotor - The non-shared fast rotor in this scrambler
+     * @param {number} pos - Position offset from start of crib
+     * @param {number} end1 - Letter in menu this scrambler is attached to
+     * @param {number} end2 - Other letter in menu this scrambler is attached to
+     */
+    constructor(base, rotor, pos, end1, end2) {
+        this.baseScrambler = base;
+        this.initialPos = pos;
+        this.changeRotor(rotor);
+        this.end1 = end1;
+        this.end2 = end2;
+        // For efficiency reasons, we pull the relevant shared cache from the baseScrambler into
+        // this object - this saves us a few pointer dereferences
+        this.cache = this.baseScrambler.higherCache[pos];
+    }
+
+    /**
+     * Replace the rotor in this scrambler.
+     * The position is reset automatically.
+     * @param {Object} rotor - New rotor
+     */
+    changeRotor(rotor) {
+        this.rotor = rotor;
+        this.rotor.pos += this.initialPos;
+    }
+
+    /**
+     * Step the rotor forward.
+     *
+     * The base SharedScrambler needs to be instructed to step separately.
+     */
+    step() {
+        // The Bombe steps the slowest rotor on an actual Enigma fastest, for reasons.
+        // ...but for optimisation reasons I'm going to cheat and not do that, as this vastly
+        // simplifies caching the state of the majority of the scramblers. The results are the
+        // same, just in a slightly different order.
+        this.rotor.step();
+        this.cache = this.baseScrambler.higherCache[this.rotor.pos];
+    }
+
+
+    /**
+     * Run a letter through the scrambler.
+     * @param {number} i - The letter to transform (as a number)
+     * @returns {number}
+     */
+    transform(i) {
+        let letter = i;
+        const cached = this.cache[i];
+        if (cached !== undefined) {
+            return cached;
+        }
+        letter = this.rotor.transform(letter);
+        letter = this.baseScrambler.transform(letter);
+        letter = this.rotor.revTransform(letter);
+        this.cache[i] = letter;
+        this.cache[letter] = i;
+        return letter;
+    }
+
+    /**
+     * Given one letter in the menu this scrambler maps to, return the other.
+     * @param end {number} - The node we have
+     * @returns {number}
+     */
+    getOtherEnd(end) {
+        return this.end1 === end ? this.end2 : this.end1;
+    }
+
+    /**
+     * Read the position this scrambler is set to.
+     * Note that because of Enigma's stepping, you need to set an actual Enigma to the previous
+     * position in order to get it to make a certain set of electrical connections when a button
+     * is pressed - this function *does* take this into account.
+     * However, as with the rest of the Bombe, it does not take stepping into account - the middle
+     * and slow rotors are treated as static.
+     * @return {string}
+     */
+    getPos() {
+        let result = "";
+        // Roll back the fast rotor by one step
+        let pos = Utils.mod(this.rotor.pos - 1, 26);
+        result += i2a(pos);
+        for (let i=0; i<this.baseScrambler.rotors.length; i++) {
+            pos = this.baseScrambler.rotors[i].pos;
+            result += i2a(pos);
+        }
+        return result.split("").reverse().join("");
+    }
+}
+
+/**
+ * Bombe simulator class.
+ */
+export class BombeMachine {
+    /**
+     * Construct a Bombe.
+     *
+     * Note that there is no handling of offsets here: the crib specified must exactly match the
+     * ciphertext. It will check that the crib is sane (length is vaguely sensible and there's no
+     * matching characters between crib and ciphertext) but cannot check further - if it's wrong
+     * your results will be wrong!
+     *
+     * There is also no handling of rotor stepping - if the target Enigma stepped in the middle of
+     * your crib, you're out of luck. TODO: Allow specifying a step point - this is fairly easy to
+     * configure on a real Bombe, but we're not clear on whether it was ever actually done for
+     * real (there would almost certainly have been better ways of attacking in most situations
+     * than attempting to exhaust options for the stepping point, but in some circumstances, e.g.
+     * via Banburismus, the stepping point might have been known).
+     *
+     * @param {string[]} rotors - list of rotor spec strings (without step points!)
+     * @param {Object} reflector - Reflector object
+     * @param {string} ciphertext - The ciphertext to attack
+     * @param {string} crib - Known plaintext for this ciphertext
+     * @param {boolean} check - Whether to use the checking machine
+     * @param {function} update - Function to call to send status updates (optional)
+     */
+    constructor(rotors, reflector, ciphertext, crib, check, update=undefined) {
+        if (ciphertext.length < crib.length) {
+            throw new OperationError("Crib overruns supplied ciphertext");
+        }
+        if (crib.length < 2) {
+            // This is the absolute bare minimum to be sane, and even then it's likely too short to
+            // be useful
+            throw new OperationError("Crib is too short");
+        }
+        if (crib.length > 25) {
+            // A crib longer than this will definitely cause the middle rotor to step somewhere
+            // A shorter crib is preferable to reduce this chance, of course
+            throw new OperationError("Crib is too long");
+        }
+        for (let i=0; i<crib.length; i++) {
+            if (ciphertext[i] === crib[i]) {
+                throw new OperationError(`Invalid crib: character ${ciphertext[i]} at pos ${i} in both ciphertext and crib`);
+            }
+        }
+        this.ciphertext = ciphertext;
+        this.crib = crib;
+        this.initRotors(rotors);
+        this.check = check;
+        this.updateFn = update;
+
+        const [mostConnected, edges] = this.makeMenu();
+
+        // This is the bundle of wires corresponding to the 26 letters within each of the 26
+        // possible nodes in the menu
+        this.wires = new Array(26*26);
+
+        // These are the pseudo-Engima devices corresponding to each edge in the menu, and the
+        // nodes in the menu they each connect to
+        this.scramblers = new Array();
+        for (let i=0; i<26; i++) {
+            this.scramblers.push(new Array());
+        }
+        this.sharedScrambler = new SharedScrambler(this.baseRotors.slice(1), reflector);
+        this.allScramblers = new Array();
+        this.indicator = undefined;
+        for (const edge of edges) {
+            const cRotor = this.baseRotors[0].copy();
+            const end1 = a2i(edge.node1.letter);
+            const end2 = a2i(edge.node2.letter);
+            const scrambler = new Scrambler(this.sharedScrambler, cRotor, edge.pos, end1, end2);
+            if (edge.pos === 0) {
+                this.indicator = scrambler;
+            }
+            this.scramblers[end1].push(scrambler);
+            this.scramblers[end2].push(scrambler);
+            this.allScramblers.push(scrambler);
+        }
+        // The Bombe uses a set of rotors to keep track of what settings it's testing. We cheat and
+        // use one of the actual scramblers if there's one in the right position, but if not we'll
+        // just create one.
+        if (this.indicator === undefined) {
+            this.indicator = new Scrambler(this.sharedScrambler, this.baseRotors[0].copy(), 0, undefined, undefined);
+            this.allScramblers.push(this.indicator);
+        }
+
+        this.testRegister = a2i(mostConnected.letter);
+        // This is an arbitrary letter other than the most connected letter
+        for (const edge of mostConnected.edges) {
+            this.testInput = [this.testRegister, a2i(edge.getOther(mostConnected).letter)];
+            break;
+        }
+    }
+
+    /**
+     * Build Rotor objects from list of rotor wiring strings.
+     * @param {string[]} rotors - List of rotor wiring strings
+     */
+    initRotors(rotors) {
+        // This is ordered from the Enigma fast rotor to the slow, so bottom to top for the Bombe
+        this.baseRotors = [];
+        for (const rstr of rotors) {
+            const rotor = new CopyRotor(rstr, "", "A", "A");
+            this.baseRotors.push(rotor);
+        }
+    }
+
+    /**
+     * Replace the rotors and reflector in all components of this Bombe.
+     * @param {string[]} rotors - List of rotor wiring strings
+     * @param {Object} reflector - Reflector object
+     */
+    changeRotors(rotors, reflector) {
+        // At the end of the run, the rotors are all back in the same position they started
+        this.initRotors(rotors);
+        this.sharedScrambler.changeRotors(this.baseRotors.slice(1), reflector);
+        for (const scrambler of this.allScramblers) {
+            scrambler.changeRotor(this.baseRotors[0].copy());
+        }
+    }
+
+    /**
+     * If we have a way of sending status messages, do so.
+     * @param {...*} msg - Message to send.
+     */
+    update(...msg) {
+        if (this.updateFn !== undefined) {
+            this.updateFn(...msg);
+        }
+    }
+
+    /**
+     * Recursive depth-first search on the menu graph.
+     * This is used to a) isolate unconnected sub-graphs, and b) count the number of loops in each
+     * of those graphs.
+     * @param {Object} node - Node object to start the search from
+     * @returns {[number, number, Object, number, Object[]} - loop count, node count, most connected
+     *      node, order of most connected node, list of edges in this sub-graph
+     */
+    dfs(node) {
+        let loops = 0;
+        let nNodes = 1;
+        let mostConnected = node;
+        let nConnections = mostConnected.edges.size;
+        let edges = new Set();
+        node.visited = true;
+        for (const edge of node.edges) {
+            if (edge.visited) {
+                // Already been here from the other end.
+                continue;
+            }
+            edge.visited = true;
+            edges.add(edge);
+            const other = edge.getOther(node);
+            if (other.visited) {
+                // We have a loop, record that and continue
+                loops += 1;
+                continue;
+            }
+            // This is a newly visited node
+            const [oLoops, oNNodes, oMostConnected, oNConnections, oEdges] = this.dfs(other);
+            loops += oLoops;
+            nNodes += oNNodes;
+            edges = new Set([...edges, ...oEdges]);
+            if (oNConnections > nConnections) {
+                mostConnected = oMostConnected;
+                nConnections = oNConnections;
+            }
+        }
+        return [loops, nNodes, mostConnected, nConnections, edges];
+    }
+
+    /**
+     * Build a menu from the ciphertext and crib.
+     * A menu is just a graph where letters in either the ciphertext or crib (Enigma is symmetric,
+     * so there's no difference mathematically) are nodes and states of the Enigma machine itself
+     * are the edges.
+     * Additionally, we want a single connected graph, and of the subgraphs available, we want the
+     * one with the most loops (since these generate feedback cycles which efficiently close off
+     * disallowed states).
+     * Finally, we want to identify the most connected node in that graph (as it's the best choice
+     * of measurement point).
+     * @returns [Object, Object[]] - the most connected node, and the list of edges in the subgraph
+     */
+    makeMenu() {
+        // First, we make a graph of all of the mappings given by the crib
+        // Make all nodes first
+        const nodes = new Map();
+        for (const c of this.ciphertext + this.crib) {
+            if (!nodes.has(c)) {
+                const node = new Node(c);
+                nodes.set(c, node);
+            }
+        }
+        // Then all edges
+        for (let i=0; i<this.crib.length; i++) {
+            const a = this.crib[i];
+            const b = this.ciphertext[i];
+            new Edge(i, nodes.get(a), nodes.get(b));
+        }
+        // list of [loop_count, node_count, most_connected_node, connections_on_most_connected, edges]
+        const graphs = [];
+        // Then, for each unconnected subgraph, we count the number of loops and nodes
+        for (const start of nodes.keys()) {
+            if (nodes.get(start).visited) {
+                continue;
+            }
+            const subgraph = this.dfs(nodes.get(start));
+            graphs.push(subgraph);
+        }
+        // Return the subgraph with the most loops (ties broken by node count)
+        graphs.sort((a, b) => {
+            let result = b[0] - a[0];
+            if (result === 0) {
+                result = b[1] - a[1];
+            }
+            return result;
+        });
+        this.nLoops = graphs[0][0];
+        return [graphs[0][2], graphs[0][4]];
+    }
+
+    /**
+     * Bombe electrical simulation. Energise a wire. For all connected wires (both via the diagonal
+     * board and via the scramblers), energise them too, recursively.
+     * @param {number} i - Bombe wire bundle
+     * @param {number} j - Bombe stecker hypothesis wire within bundle
+     */
+    energise(i, j) {
+        const idx = 26*i + j;
+        if (this.wires[idx]) {
+            return;
+        }
+        this.wires[idx] = true;
+        // Welchman's diagonal board: if A steckers to B, that implies B steckers to A. Handle
+        // both.
+        const idxPair = 26*j + i;
+        this.wires[idxPair] = true;
+        if (i === this.testRegister || j === this.testRegister) {
+            this.energiseCount++;
+            if (this.energiseCount === 26) {
+                // no point continuing, bail out
+                return;
+            }
+        }
+
+        for (let k=0; k<this.scramblers[i].length; k++) {
+            const scrambler = this.scramblers[i][k];
+            const out = scrambler.transform(j);
+            const other = scrambler.getOtherEnd(i);
+            // Lift the pre-check before the call, to save some function call overhead
+            const otherIdx = 26*other + out;
+            if (!this.wires[otherIdx]) {
+                this.energise(other, out);
+                if (this.energiseCount === 26) {
+                    return;
+                }
+            }
+        }
+        if (i === j) {
+            return;
+        }
+        for (let k=0; k<this.scramblers[j].length; k++) {
+            const scrambler = this.scramblers[j][k];
+            const out = scrambler.transform(i);
+            const other = scrambler.getOtherEnd(j);
+            const otherIdx = 26*other + out;
+            if (!this.wires[otherIdx]) {
+                this.energise(other, out);
+                if (this.energiseCount === 26) {
+                    return;
+                }
+            }
+        }
+    }
+
+    /**
+     * Trial decryption at the current setting.
+     * Used after we get a stop.
+     * This applies the detected stecker pair if we have one. It does not handle the other
+     * steckering or stepping (which is why we limit it to 26 characters, since it's guaranteed to
+     * be wrong after that anyway).
+     * @param {string} stecker - Known stecker spec string.
+     * @returns {string}
+     */
+    tryDecrypt(stecker) {
+        const fastRotor = this.indicator.rotor;
+        const initialPos = fastRotor.pos;
+        const res = [];
+        const plugboard = new Plugboard(stecker);
+        // The indicator scrambler starts in the right place for the beginning of the ciphertext.
+        for (let i=0; i<Math.min(26, this.ciphertext.length); i++) {
+            const t = this.indicator.transform(plugboard.transform(a2i(this.ciphertext[i])));
+            res.push(i2a(plugboard.transform(t)));
+            this.indicator.step(1);
+        }
+        fastRotor.pos = initialPos;
+        return res.join("");
+    }
+
+    /**
+     * Format a steckered pair, in sorted order to allow uniquing.
+     * @param {number} a - A letter
+     * @param {number} b - Its stecker pair
+     * @returns {string}
+     */
+    formatPair(a, b) {
+        if (a < b) {
+            return `${i2a(a)}${i2a(b)}`;
+        }
+        return `${i2a(b)}${i2a(a)}`;
+    }
+
+    /**
+     * The checking machine was used to manually verify Bombe stops. Using a device which was
+     * effectively a non-stepping Enigma, the user would walk through each of the links in the
+     * menu at the rotor positions determined by the Bombe. By starting with the stecker pair the
+     * Bombe gives us, we find the stecker pair of each connected letter in the graph, and so on.
+     * If a contradiction is reached, the stop is invalid. If not, we have most (but not
+     * necessarily all) of the plugboard connections.
+     * You will notice that this procedure is exactly the same as what the Bombe itself does, only
+     * we start with an assumed good hypothesis and read out the stecker pair for every letter.
+     * On the real hardware that wasn't practical, but fortunately we're not the real hardware, so
+     * we don't need to implement the manual checking machine procedure.
+     * @param {number} pair - The stecker pair of the test register.
+     * @returns {string} - The empty string for invalid stops, or a plugboard configuration string
+     *      containing all known pairs.
+     */
+    checkingMachine(pair) {
+        if (pair !== this.testInput[1]) {
+            // We have a new hypothesis for this stop - apply the new one.
+            // De-energise the board
+            for (let i=0; i<this.wires.length; i++) {
+                this.wires[i] = false;
+            }
+            this.energiseCount = 0;
+            // Re-energise with the corrected hypothesis
+            this.energise(this.testRegister, pair);
+        }
+
+        const results = new Set();
+        results.add(this.formatPair(this.testRegister, pair));
+        for (let i=0; i<26; i++) {
+            let count = 0;
+            let other;
+            for (let j=0; j<26; j++) {
+                if (this.wires[i*26 + j]) {
+                    count++;
+                    other = j;
+                }
+            }
+            if (count > 1) {
+                // This is an invalid stop.
+                return "";
+            } else if (count === 0) {
+                // No information about steckering from this wire
+                continue;
+            }
+            results.add(this.formatPair(i, other));
+        }
+        return [...results].join(" ");
+    }
+
+    /**
+     * Check to see if the Bombe has stopped. If so, process the stop.
+     * @returns {(undefined|string[3])} - Undefined for no stop, or [rotor settings, plugboard settings, decryption preview]
+     */
+    checkStop() {
+        // Count the energised outputs
+        const count = this.energiseCount;
+        if (count === 26) {
+            return undefined;
+        }
+        // If it's not all of them, we have a stop
+        let steckerPair;
+        // The Bombe tells us one stecker pair as well. The input wire and test register we
+        // started with are hypothesised to be a stecker pair.
+        if (count === 25) {
+            // Our steckering hypothesis is wrong. Correct value is the un-energised wire.
+            for (let j=0; j<26; j++) {
+                if (!this.wires[26*this.testRegister + j]) {
+                    steckerPair = j;
+                    break;
+                }
+            }
+        } else if (count === 1) {
+            // This means our hypothesis for the steckering is correct.
+            steckerPair = this.testInput[1];
+        } else {
+            // This was known as a "boxing stop" - we have a stop but not a single hypothesis.
+            // If this happens a lot it implies the menu isn't good enough.
+            // If we have the checking machine enabled, we're going to just check each wire in
+            // turn. If we get 0 or 1 hit, great.
+            // If we get multiple hits, or the checking machine is off, the user will just have to
+            // deal with it.
+            if (!this.check) {
+                // We can't draw any conclusions about the steckering (one could maybe suggest
+                // options in some cases, but too hard to present clearly).
+                return [this.indicator.getPos(), "??", this.tryDecrypt("")];
+            }
+            let stecker = undefined;
+            for (let i = 0; i < 26; i++) {
+                const newStecker = this.checkingMachine(i);
+                if (newStecker !== "") {
+                    if (stecker !== undefined) {
+                        // Multiple hypotheses can't be ruled out.
+                        return [this.indicator.getPos(), "??", this.tryDecrypt("")];
+                    }
+                    stecker = newStecker;
+                }
+            }
+            if (stecker === undefined) {
+                // Checking machine ruled all possibilities out.
+                return undefined;
+            }
+            // If we got here, there was just one possibility allowed by the checking machine. Success.
+            return [this.indicator.getPos(), stecker, this.tryDecrypt(stecker)];
+        }
+        let stecker;
+        if (this.check) {
+            stecker = this.checkingMachine(steckerPair);
+            if (stecker === "") {
+                // Invalid stop - don't count it, don't return it
+                return undefined;
+            }
+        } else {
+            stecker = `${i2a(this.testRegister)}${i2a(steckerPair)}`;
+        }
+        const testDecrypt = this.tryDecrypt(stecker);
+        return [this.indicator.getPos(), stecker, testDecrypt];
+    }
+
+    /**
+     * Having set up the Bombe, do the actual attack run. This tries every possible rotor setting
+     * and attempts to logically invalidate them. If it can't, it's added to the list of candidate
+     * solutions.
+     * @returns {string[][3]} - list of 3-tuples of candidate rotor setting, plugboard settings, and decryption preview
+     */
+    run() {
+        let stops = 0;
+        const result = [];
+        // For each possible rotor setting
+        const nChecks = Math.pow(26, this.baseRotors.length);
+        for (let i=1; i<=nChecks; i++) {
+            // Benchmarking suggests this is faster than using .fill()
+            for (let i=0; i<this.wires.length; i++) {
+                this.wires[i] = false;
+            }
+            this.energiseCount = 0;
+            // Energise the test input, follow the current through each scrambler
+            // (and the diagonal board)
+            this.energise(...this.testInput);
+
+            const stop = this.checkStop();
+            if (stop !== undefined) {
+                stops++;
+                result.push(stop);
+            }
+            // Step all the scramblers
+            // This loop counts how many rotors have reached their starting position (meaning the
+            // next one needs to step as well)
+            let n = 1;
+            for (let j=1; j<this.baseRotors.length; j++) {
+                if ((i % Math.pow(26, j)) === 0) {
+                    n++;
+                } else {
+                    break;
+                }
+            }
+            if (n > 1) {
+                this.sharedScrambler.step(n);
+            }
+            for (const scrambler of this.allScramblers) {
+                scrambler.step();
+            }
+            // Send status messages at what seems to be a reasonably sensible frequency
+            // (note this won't be triggered on 3-rotor runs - they run fast enough it doesn't seem necessary)
+            if (n > 3) {
+                this.update(this.nLoops, stops, i/nChecks);
+            }
+        }
+        return result;
+    }
+}

src/core/lib/CanvasComponents.mjs

@@ -0,0 +1,179 @@
+/**
+ * @author tlwr [toby@toby.codes]
+ * @author Matt C [me@mitt.dev]
+ * @copyright Crown Copyright 2019
+ * @license Apache-2.0
+ */
+
+import OperationError from "../errors/OperationError.mjs";
+import Utils from "../Utils.mjs";
+
+/**
+ * @constant
+ * @default
+ */
+export const RECORD_DELIMITER_OPTIONS = ["Line feed", "CRLF"];
+
+
+/**
+ * @constant
+ * @default
+ */
+export const FIELD_DELIMITER_OPTIONS = ["Space", "Comma", "Semi-colon", "Colon", "Tab"];
+
+
+/**
+ * Default from colour
+ *
+ * @constant
+ * @default
+ */
+export const COLOURS = {
+    min: "white",
+    max: "black"
+};
+
+
+/**
+ * Gets values from input for a plot.
+ *
+ * @param {string} input
+ * @param {string} recordDelimiter
+ * @param {string} fieldDelimiter
+ * @param {boolean} columnHeadingsAreIncluded - whether we should skip the first record
+ * @param {number} length
+ * @returns {Object[]}
+ */
+export function getValues(input, recordDelimiter, fieldDelimiter, columnHeadingsAreIncluded, length) {
+    let headings;
+    const values = [];
+
+    input
+        .split(recordDelimiter)
+        .forEach((row, rowIndex) => {
+            const split = row.split(fieldDelimiter);
+            if (split.length !== length) throw new OperationError(`Each row must have length ${length}.`);
+
+            if (columnHeadingsAreIncluded && rowIndex === 0) {
+                headings = split;
+            } else {
+                values.push(split);
+            }
+        });
+    return { headings, values };
+}
+
+
+/**
+ * Gets values from input for a scatter plot.
+ *
+ * @param {string} input
+ * @param {string} recordDelimiter
+ * @param {string} fieldDelimiter
+ * @param {boolean} columnHeadingsAreIncluded - whether we should skip the first record
+ * @returns {Object[]}
+ */
+export function getScatterValues(input, recordDelimiter, fieldDelimiter, columnHeadingsAreIncluded) {
+    let { headings, values } = getValues(
+        input,
+        recordDelimiter,
+        fieldDelimiter,
+        columnHeadingsAreIncluded,
+        2
+    );
+
+    if (headings) {
+        headings = {x: headings[0], y: headings[1]};
+    }
+
+    values = values.map(row => {
+        const x = parseFloat(row[0]),
+            y = parseFloat(row[1]);
+
+        if (Number.isNaN(x)) throw new OperationError("Values must be numbers in base 10.");
+        if (Number.isNaN(y)) throw new OperationError("Values must be numbers in base 10.");
+
+        return [x, y];
+    });
+
+    return { headings, values };
+}
+
+
+/**
+ * Gets values from input for a scatter plot with colour from the third column.
+ *
+ * @param {string} input
+ * @param {string} recordDelimiter
+ * @param {string} fieldDelimiter
+ * @param {boolean} columnHeadingsAreIncluded - whether we should skip the first record
+ * @returns {Object[]}
+ */
+export function getScatterValuesWithColour(input, recordDelimiter, fieldDelimiter, columnHeadingsAreIncluded) {
+    let { headings, values } = getValues(
+        input,
+        recordDelimiter, fieldDelimiter,
+        columnHeadingsAreIncluded,
+        3
+    );
+
+    if (headings) {
+        headings = {x: headings[0], y: headings[1]};
+    }
+
+    values = values.map(row => {
+        const x = parseFloat(row[0]),
+            y = parseFloat(row[1]),
+            colour = row[2];
+
+        if (Number.isNaN(x)) throw new OperationError("Values must be numbers in base 10.");
+        if (Number.isNaN(y)) throw new OperationError("Values must be numbers in base 10.");
+
+        return [x, y, Utils.escapeHtml(colour)];
+    });
+
+    return { headings, values };
+}
+
+/**
+ * Gets values from input for a time series plot.
+ *
+ * @param {string} input
+ * @param {string} recordDelimiter
+ * @param {string} fieldDelimiter
+ * @param {boolean} columnHeadingsAreIncluded - whether we should skip the first record
+ * @returns {Object[]}
+ */
+export function getSeriesValues(input, recordDelimiter, fieldDelimiter, columnHeadingsAreIncluded) {
+    const { values } = getValues(
+        input,
+        recordDelimiter, fieldDelimiter,
+        false,
+        3
+    );
+
+    let xValues = new Set();
+    const series = {};
+
+    values.forEach(row => {
+        const serie = row[0],
+            xVal = row[1],
+            val = parseFloat(row[2]);
+
+        if (Number.isNaN(val)) throw new OperationError("Values must be numbers in base 10.");
+
+        xValues.add(xVal);
+        if (typeof series[serie] === "undefined") series[serie] = {};
+        series[serie][xVal] = val;
+    });
+
+    xValues = new Array(...xValues);
+
+    const seriesList = [];
+    for (const seriesName in series) {
+        const serie = series[seriesName];
+        seriesList.push({name: seriesName, data: serie});
+    }
+
+    return { xValues, series: seriesList };
+}

src/core/lib/ChrEnc.mjs

@@ -6,22 +6,73 @@
  * @license Apache-2.0
  */
 
+import cptable from "codepage";
+
 /**
  * Character encoding format mappings.
  */
-export const IO_FORMAT = {
+export const CHR_ENC_CODE_PAGES = {
     "UTF-8 (65001)": 65001,
     "UTF-7 (65000)": 65000,
-    "UTF16LE (1200)": 1200,
-    "UTF16BE (1201)": 1201,
-    "UTF16 (1201)": 1201,
+    "UTF-16LE (1200)": 1200,
+    "UTF-16BE (1201)": 1201,
+    "UTF-32LE (12000)": 12000,
+    "UTF-32BE (12001)": 12001,
     "IBM EBCDIC International (500)": 500,
     "IBM EBCDIC US-Canada (37)": 37,
+    "IBM EBCDIC Multilingual/ROECE (Latin 2) (870)": 870,
+    "IBM EBCDIC Greek Modern (875)": 875,
+    "IBM EBCDIC French (1010)": 1010,
+    "IBM EBCDIC Turkish (Latin 5) (1026)": 1026,
+    "IBM EBCDIC Latin 1/Open System (1047)": 1047,
+    "IBM EBCDIC Lao (1132/1133/1341)": 1132,
+    "IBM EBCDIC US-Canada (037 + Euro symbol) (1140)": 1140,
+    "IBM EBCDIC Germany (20273 + Euro symbol) (1141)": 1141,
+    "IBM EBCDIC Denmark-Norway (20277 + Euro symbol) (1142)": 1142,
+    "IBM EBCDIC Finland-Sweden (20278 + Euro symbol) (1143)": 1143,
+    "IBM EBCDIC Italy (20280 + Euro symbol) (1144)": 1144,
+    "IBM EBCDIC Latin America-Spain (20284 + Euro symbol) (1145)": 1145,
+    "IBM EBCDIC United Kingdom (20285 + Euro symbol) (1146)": 1146,
+    "IBM EBCDIC France (20297 + Euro symbol) (1147)": 1147,
+    "IBM EBCDIC International (500 + Euro symbol) (1148)": 1148,
+    "IBM EBCDIC Icelandic (20871 + Euro symbol) (1149)": 1149,
+    "IBM EBCDIC Germany (20273)": 20273,
+    "IBM EBCDIC Denmark-Norway (20277)": 20277,
+    "IBM EBCDIC Finland-Sweden (20278)": 20278,
+    "IBM EBCDIC Italy (20280)": 20280,
+    "IBM EBCDIC Latin America-Spain (20284)": 20284,
+    "IBM EBCDIC United Kingdom (20285)": 20285,
+    "IBM EBCDIC Japanese Katakana Extended (20290)": 20290,
+    "IBM EBCDIC France (20297)": 20297,
+    "IBM EBCDIC Arabic (20420)": 20420,
+    "IBM EBCDIC Greek (20423)": 20423,
+    "IBM EBCDIC Hebrew (20424)": 20424,
+    "IBM EBCDIC Korean Extended (20833)": 20833,
+    "IBM EBCDIC Thai (20838)": 20838,
+    "IBM EBCDIC Icelandic (20871)": 20871,
+    "IBM EBCDIC Cyrillic Russian (20880)": 20880,
+    "IBM EBCDIC Turkish (20905)": 20905,
+    "IBM EBCDIC Latin 1/Open System (1047 + Euro symbol) (20924)": 20924,
+    "IBM EBCDIC Cyrillic Serbian-Bulgarian (21025)": 21025,
+    "OEM United States (437)": 437,
+    "OEM Greek (formerly 437G); Greek (DOS) (737)": 737,
+    "OEM Baltic; Baltic (DOS) (775)": 775,
+    "OEM Russian; Cyrillic + Euro symbol (808)": 808,
+    "OEM Multilingual Latin 1; Western European (DOS) (850)": 850,
+    "OEM Latin 2; Central European (DOS) (852)": 852,
+    "OEM Cyrillic (primarily Russian) (855)": 855,
+    "OEM Turkish; Turkish (DOS) (857)": 857,
+    "OEM Multilingual Latin 1 + Euro symbol (858)": 858,
+    "OEM Portuguese; Portuguese (DOS) (860)": 860,
+    "OEM Icelandic; Icelandic (DOS) (861)": 861,
+    "OEM Hebrew; Hebrew (DOS) (862)": 862,
+    "OEM French Canadian; French Canadian (DOS) (863)": 863,
+    "OEM Arabic; Arabic (864) (864)": 864,
+    "OEM Nordic; Nordic (DOS) (865)": 865,
+    "OEM Russian; Cyrillic (DOS) (866)": 866,
+    "OEM Modern Greek; Greek, Modern (DOS) (869)": 869,
+    "OEM Cyrillic (primarily Russian) + Euro Symbol (872)": 872,
     "Windows-874 Thai (874)": 874,
-    "Japanese Shift-JIS (932)": 932,
-    "Simplified Chinese GBK (936)": 936,
-    "Korean (949)": 949,
-    "Traditional Chinese Big5 (950)": 950,
     "Windows-1250 Central European (1250)": 1250,
     "Windows-1251 Cyrillic (1251)": 1251,
     "Windows-1252 Latin (1252)": 1252,
@@ -31,10 +82,6 @@ export const IO_FORMAT = {
     "Windows-1256 Arabic (1256)": 1256,
     "Windows-1257 Baltic (1257)": 1257,
     "Windows-1258 Vietnam (1258)": 1258,
-    "US-ASCII (20127)": 20127,
-    "Simplified Chinese GB2312 (20936)": 20936,
-    "KOI8-R Russian Cyrillic (20866)": 20866,
-    "KOI8-U Ukrainian Cyrillic (21866)": 21866,
     "ISO-8859-1 Latin 1 Western European (28591)": 28591,
     "ISO-8859-2 Latin 2 Central European (28592)": 28592,
     "ISO-8859-3 Latin 3 South European (28593)": 28593,
@@ -43,6 +90,7 @@ export const IO_FORMAT = {
     "ISO-8859-6 Latin/Arabic (28596)": 28596,
     "ISO-8859-7 Latin/Greek (28597)": 28597,
     "ISO-8859-8 Latin/Hebrew (28598)": 28598,
+    "ISO 8859-8 Hebrew (ISO-Logical) (38598)": 38598,
     "ISO-8859-9 Latin 5 Turkish (28599)": 28599,
     "ISO-8859-10 Latin 6 Nordic (28600)": 28600,
     "ISO-8859-11 Latin/Thai (28601)": 28601,
@@ -50,9 +98,134 @@ export const IO_FORMAT = {
     "ISO-8859-14 Latin 8 Celtic (28604)": 28604,
     "ISO-8859-15 Latin 9 (28605)": 28605,
     "ISO-8859-16 Latin 10 (28606)": 28606,
-    "ISO-2022 JIS Japanese (50222)": 50222,
+    "ISO 2022 JIS Japanese with no halfwidth Katakana (50220)": 50220,
+    "ISO 2022 JIS Japanese with halfwidth Katakana (50221)": 50221,
+    "ISO 2022 Japanese JIS X 0201-1989 (1 byte Kana-SO/SI) (50222)": 50222,
+    "ISO 2022 Korean (50225)": 50225,
+    "ISO 2022 Simplified Chinese (50227)": 50227,
+    "ISO 6937 Non-Spacing Accent (20269)": 20269,
     "EUC Japanese (51932)": 51932,
+    "EUC Simplified Chinese (51936)": 51936,
     "EUC Korean (51949)": 51949,
+    "ISCII Devanagari (57002)": 57002,
+    "ISCII Bengali (57003)": 57003,
+    "ISCII Tamil (57004)": 57004,
+    "ISCII Telugu (57005)": 57005,
+    "ISCII Assamese (57006)": 57006,
+    "ISCII Oriya (57007)": 57007,
+    "ISCII Kannada (57008)": 57008,
+    "ISCII Malayalam (57009)": 57009,
+    "ISCII Gujarati (57010)": 57010,
+    "ISCII Punjabi (57011)": 57011,
+    "Japanese Shift-JIS (932)": 932,
+    "Simplified Chinese GBK (936)": 936,
+    "Korean (949)": 949,
+    "Traditional Chinese Big5 (950)": 950,
+    "US-ASCII (7-bit) (20127)": 20127,
+    "Simplified Chinese GB2312 (20936)": 20936,
+    "KOI8-R Russian Cyrillic (20866)": 20866,
+    "KOI8-U Ukrainian Cyrillic (21866)": 21866,
+    "Mazovia (Polish) MS-DOS (620)": 620,
+    "Arabic (ASMO 708) (708)": 708,
+    "Arabic (Transparent ASMO); Arabic (DOS) (720)": 720,
+    "Kamenický (Czech) MS-DOS (895)": 895,
+    "Korean (Johab) (1361)": 1361,
+    "MAC Roman (10000)": 10000,
+    "Japanese (Mac) (10001)": 10001,
+    "MAC Traditional Chinese (Big5) (10002)": 10002,
+    "Korean (Mac) (10003)": 10003,
+    "Arabic (Mac) (10004)": 10004,
+    "Hebrew (Mac) (10005)": 10005,
+    "Greek (Mac) (10006)": 10006,
+    "Cyrillic (Mac) (10007)": 10007,
+    "MAC Simplified Chinese (GB 2312) (10008)": 10008,
+    "Romanian (Mac) (10010)": 10010,
+    "Ukrainian (Mac) (10017)": 10017,
+    "Thai (Mac) (10021)": 10021,
+    "MAC Latin 2 (Central European) (10029)": 10029,
+    "Icelandic (Mac) (10079)": 10079,
+    "Turkish (Mac) (10081)": 10081,
+    "Croatian (Mac) (10082)": 10082,
+    "CNS Taiwan (Chinese Traditional) (20000)": 20000,
+    "TCA Taiwan (20001)": 20001,
+    "ETEN Taiwan (Chinese Traditional) (20002)": 20002,
+    "IBM5550 Taiwan (20003)": 20003,
+    "TeleText Taiwan (20004)": 20004,
+    "Wang Taiwan (20005)": 20005,
+    "Western European IA5 (IRV International Alphabet 5) (20105)": 20105,
+    "IA5 German (7-bit) (20106)": 20106,
+    "IA5 Swedish (7-bit) (20107)": 20107,
+    "IA5 Norwegian (7-bit) (20108)": 20108,
+    "T.61 (20261)": 20261,
+    "Japanese (JIS 0208-1990 and 0212-1990) (20932)": 20932,
+    "Korean Wansung (20949)": 20949,
+    "Extended/Ext Alpha Lowercase (21027)": 21027,
+    "Europa 3 (29001)": 29001,
+    "Atari ST/TT (47451)": 47451,
+    "HZ-GB2312 Simplified Chinese (52936)": 52936,
     "Simplified Chinese GB18030 (54936)": 54936,
 };
 
+
+export const CHR_ENC_SIMPLE_LOOKUP = {};
+export const CHR_ENC_SIMPLE_REVERSE_LOOKUP = {};
+
+for (const name in CHR_ENC_CODE_PAGES) {
+    const simpleName = name.match(/(^.+)\([\d/]+\)$/)[1];
+
+    CHR_ENC_SIMPLE_LOOKUP[simpleName] = CHR_ENC_CODE_PAGES[name];
+    CHR_ENC_SIMPLE_REVERSE_LOOKUP[CHR_ENC_CODE_PAGES[name]] = simpleName;
+}
+
+
+/**
+ * Returns the width of the character set for the given codepage.
+ * For example, UTF-8 is a Single Byte Character Set, whereas
+ * UTF-16 is a Double Byte Character Set.
+ *
+ * @param {number} page - The codepage number
+ * @returns {number}
+ */
+export function chrEncWidth(page) {
+    if (typeof page !== "number") return 0;
+
+    // Raw Bytes have a width of 1
+    if (page === 0) return 1;
+
+    const pageStr = page.toString();
+    // Confirm this page is legitimate
+    if (!Object.prototype.hasOwnProperty.call(CHR_ENC_SIMPLE_REVERSE_LOOKUP, pageStr))
+        return 0;
+
+    // Statically defined code pages
+    if (Object.prototype.hasOwnProperty.call(cptable, pageStr))
+        return cptable[pageStr].dec.length > 256 ? 2 : 1;
+
+    // Cached code pages
+    if (cptable.utils.cache.sbcs.includes(pageStr))
+        return 1;
+    if (cptable.utils.cache.dbcs.includes(pageStr))
+        return 2;
+
+    // Dynamically generated code pages
+    if (Object.prototype.hasOwnProperty.call(cptable.utils.magic, pageStr)) {
+        // Generate a single character and measure it
+        const a = cptable.utils.encode(page, "a");
+        return a.length;
+    }
+
+    return 0;
+}
+
+/**
+ * Unicode Normalisation Forms
+ *
+ * @author Matthieu [m@tthieu.xyz]
+ * @copyright Crown Copyright 2019
+ * @license Apache-2.0
+ */
+
+/**
+ * Character encoding format mappings.
+ */
+export const UNICODE_NORMALISATION_FORMS = ["NFD", "NFC", "NFKD", "NFKC"];

src/core/lib/CipherSaber2.mjs

@@ -0,0 +1,34 @@
+/**
+ * @author n1073645 [n1073645@gmail.com]
+ * @copyright Crown Copyright 2020
+ * @license Apache-2.0
+ */
+export function encode(tempIVP, key, rounds, input) {
+    const ivp = new Uint8Array(key.concat(tempIVP));
+    const state = new Array(256).fill(0);
+    let j = 0, i = 0;
+    const result = [];
+
+    // Mixing states based off of IV.
+    for (let i = 0; i < 256; i++)
+        state[i] = i;
+    const ivpLength = ivp.length;
+    for (let r = 0; r < rounds; r ++) {
+        for (let k = 0; k < 256; k++) {
+            j = (j + state[k] + ivp[k % ivpLength]) % 256;
+            [state[k], state[j]] = [state[j], state[k]];
+        }
+    }
+    j = 0;
+    i = 0;
+
+    // XOR cipher with key.
+    for (let x = 0; x < input.length; x++) {
+        i = (++i) % 256;
+        j = (j + state[i]) % 256;
+        [state[i], state[j]] = [state[j], state[i]];
+        const n = (state[i] + state[j]) % 256;
+        result.push(state[n] ^ input[x]);
+    }
+    return result;
+}

src/core/lib/Ciphers.mjs

@@ -9,7 +9,7 @@
  *
  */
 
-import OperationError from "../errors/OperationError";
+import OperationError from "../errors/OperationError.mjs";
 import CryptoJS from "crypto-js";
 
 /**

src/core/lib/Colossus.mjs

@@ -0,0 +1,417 @@
+/**
+ * Colossus - an emulation of the world's first electronic computer
+ *
+ * @author VirtualColossus [martin@virtualcolossus.co.uk]
+ * @copyright Crown Copyright 2019
+ * @license Apache-2.0
+ */
+import {INIT_PATTERNS, ITA2_TABLE, ROTOR_SIZES} from "../lib/Lorenz.mjs";
+
+/**
+ * Colossus simulator class.
+ */
+export class ColossusComputer {
+    /**
+     * Construct a Colossus.
+     *
+     * @param {string} ciphertext
+     * @param {string} pattern - named pattern of Chi, Mu and Psi wheels
+     * @param {Object} qbusin - which data inputs are being sent to q bus - each can be null, plain or delta
+     * @param {Object[]} qbusswitches - Q bus calculation switches, multiple rows
+     * @param {Object} control - control switches which specify stepping modes
+     * @param {Object} starts - rotor start positions
+     */
+    constructor(ciphertext, pattern, qbusin, qbusswitches, control, starts, settotal, limit) {
+        this.ITAlookup = ITA2_TABLE;
+        this.ReverseITAlookup = {};
+        for (const letter in this.ITAlookup) {
+            const code = this.ITAlookup[letter];
+            this.ReverseITAlookup[code] = letter;
+        }
+
+        this.initThyratrons(pattern);
+
+        this.ciphertext = ciphertext;
+        this.qbusin = qbusin;
+        this.qbusswitches = qbusswitches;
+        this.control = control;
+        this.starts = starts;
+        this.settotal = settotal;
+        this.limitations = limit;
+
+
+        this.allCounters = [0, 0, 0, 0, 0];
+
+        this.Zbits = [0, 0, 0, 0, 0]; // Z input is the cipher tape
+        this.ZbitsOneBack = [0, 0, 0, 0, 0]; // for delta
+        this.Qbits = [0, 0, 0, 0, 0]; // input generated for placing onto the Q-bus (the logic processor)
+
+        this.Xbits = [0, 0, 0, 0, 0]; // X is the Chi wheel bits
+        this.Xptr = [0, 0, 0, 0, 0]; // pointers to the current X bits (Chi wheels)
+        this.XbitsOneBack = [0, 0, 0, 0, 0]; // the X bits one back (for delta)
+
+        this.Sbits = [0, 0, 0, 0, 0]; // S is the Psi wheel bits
+        this.Sptr = [0, 0, 0, 0, 0]; // pointers to the current S bits (Psi wheels)
+        this.SbitsOneBack = [0, 0, 0, 0, 0]; // the S bits one back (for delta)
+
+        this.Mptr = [0, 0];
+
+        this.rotorPtrs = {};
+
+        this.totalmotor = 0;
+        this.P5Zbit = [0, 0];
+    }
+
+    /**
+     * Begin a run
+     *
+     * @returns {object}
+     */
+    run() {
+        const result = {
+            printout: ""
+        };
+
+        // loop until our start positions are back to the beginning
+        this.rotorPtrs = {X1: this.starts.X1, X2: this.starts.X2, X3: this.starts.X3, X4: this.starts.X4, X5: this.starts.X5, M61: this.starts.M61, M37: this.starts.M37, S1: this.starts.S1, S2: this.starts.S2, S3: this.starts.S3, S4: this.starts.S4, S5: this.starts.S5};
+        // this.rotorPtrs = this.starts;
+        let runcount = 1;
+
+        const fast = this.control.fast;
+        const slow = this.control.slow;
+
+        // Print Headers
+        result.printout += fast + " " + slow + "\n";
+
+        do {
+            this.allCounters = [0, 0, 0, 0, 0];
+            this.ZbitsOneBack = [0, 0, 0, 0, 0];
+            this.XbitsOneBack = [0, 0, 0, 0, 0];
+
+            // Run full tape loop and process counters
+            this.runTape();
+
+            // Only print result if larger than set total
+            let fastRef = "00";
+            let slowRef = "00";
+            if (fast !== "") fastRef = this.rotorPtrs[fast].toString().padStart(2, "0");
+            if (slow !== "") slowRef = this.rotorPtrs[slow].toString().padStart(2, "0");
+            let printline = "";
+            for (let c=0;c<5;c++) {
+                if (this.allCounters[c] > this.settotal) {
+                    printline += String.fromCharCode(c+97) + this.allCounters[c]+" ";
+                }
+            }
+            if (printline !== "") {
+                result.printout += fastRef + " " + slowRef + " : ";
+                result.printout += printline;
+                result.printout += "\n";
+            }
+
+            // Step fast rotor if required
+            if (fast !== "") {
+                this.rotorPtrs[fast]++;
+                if (this.rotorPtrs[fast] > ROTOR_SIZES[fast]) this.rotorPtrs[fast] = 1;
+            }
+
+            // Step slow rotor if fast rotor has returned to initial start position
+            if (slow !== "" && this.rotorPtrs[fast] === this.starts[fast]) {
+                this.rotorPtrs[slow]++;
+                if (this.rotorPtrs[slow] > ROTOR_SIZES[slow]) this.rotorPtrs[slow] = 1;
+            }
+
+            runcount++;
+        } while (JSON.stringify(this.rotorPtrs) !== JSON.stringify(this.starts));
+
+        result.counters = this.allCounters;
+        result.runcount = runcount;
+
+        return result;
+    }
+
+    /**
+     * Run tape loop
+     */
+    runTape() {
+        let charZin = "";
+
+        this.Xptr = [this.rotorPtrs.X1, this.rotorPtrs.X2, this.rotorPtrs.X3, this.rotorPtrs.X4, this.rotorPtrs.X5];
+        this.Mptr = [this.rotorPtrs.M37, this.rotorPtrs.M61];
+        this.Sptr = [this.rotorPtrs.S1, this.rotorPtrs.S2, this.rotorPtrs.S3, this.rotorPtrs.S4, this.rotorPtrs.S5];
+
+        // Run full loop of all character on the input tape (Z)
+        for (let i=0; i<this.ciphertext.length; i++) {
+            charZin = this.ciphertext.charAt(i);
+
+            // Firstly, we check what inputs are specified on the Q-bus input switches
+            this.getQbusInputs(charZin);
+
+            /*
+             * Pattern conditions on individual impulses. Matching patterns of bits on the Q bus.
+             * This is the top section on Colussus K rack - the Q bus programming switches
+             */
+            const tmpcnt = this.runQbusProcessingConditional();
+
+            /*
+             * Addition of impulses.
+             * This is the bottom section of Colossus K rack.
+             */
+            this.runQbusProcessingAddition(tmpcnt);
+
+            // Store Z bit impulse 5 two back required for P5 limitation
+            this.P5Zbit[1] = this.P5Zbit[0];
+            this.P5Zbit[0] = this.ITAlookup[charZin].split("")[4];
+
+            // Step rotors
+            this.stepThyratrons();
+
+        }
+    }
+
+    /**
+     * Step thyratron rings to simulate movement of Lorenz rotors
+     * Chi rotors all step one per character
+     * Motor M61 rotor steps one per character, M37 steps dependant on M61 setting
+     * Psi rotors only step dependant on M37 setting + limitation
+     */
+    stepThyratrons() {
+        let X2bPtr = this.Xptr[1]-1;
+        if (X2bPtr===0) X2bPtr = ROTOR_SIZES.X2;
+        let S1bPtr = this.Sptr[0]-1;
+        if (S1bPtr===0) S1bPtr = ROTOR_SIZES.S1;
+
+        // Get Chi rotor 5 two back to calculate plaintext (Z+Chi+Psi=Plain)
+        let X5bPtr=this.Xptr[4]-1;
+        if (X5bPtr===0) X5bPtr=ROTOR_SIZES.X5;
+        X5bPtr=X5bPtr-1;
+        if (X5bPtr===0) X5bPtr=ROTOR_SIZES.X5;
+        // Get Psi rotor 5 two back to calculate plaintext (Z+Chi+Psi=Plain)
+        let S5bPtr=this.Sptr[4]-1;
+        if (S5bPtr===0) S5bPtr=ROTOR_SIZES.S5;
+        S5bPtr=S5bPtr-1;
+        if (S5bPtr===0) S5bPtr=ROTOR_SIZES.S5;
+
+        const x2sw = this.limitations.X2;
+        const s1sw = this.limitations.S1;
+        const p5sw = this.limitations.P5;
+
+        // Limitation calculations
+        let lim=1;
+        if (x2sw) lim = this.rings.X[2][X2bPtr-1];
+        if (s1sw) lim = lim ^ this.rings.S[1][S1bPtr-1];
+
+        // P5
+        if (p5sw) {
+            let p5lim = this.P5Zbit[1];
+            p5lim = p5lim ^ this.rings.X[5][X5bPtr-1];
+            p5lim = p5lim ^ this.rings.S[5][S5bPtr-1];
+            lim = lim ^ p5lim;
+        }
+
+        const basicmotor = this.rings.M[2][this.Mptr[0]-1];
+        this.totalmotor = basicmotor;
+
+        if (x2sw || s1sw) {
+            if (basicmotor===0 && lim===1) {
+                this.totalmotor = 0;
+            } else {
+                this.totalmotor = 1;
+            }
+        }
+
+        // Step Chi rotors
+        for (let r=0; r<5; r++) {
+            this.Xptr[r]++;
+            if (this.Xptr[r] > ROTOR_SIZES["X"+(r+1)]) this.Xptr[r] = 1;
+        }
+
+        if (this.totalmotor) {
+            // Step Psi rotors
+            for (let r=0; r<5; r++) {
+                this.Sptr[r]++;
+                if (this.Sptr[r] > ROTOR_SIZES["S"+(r+1)]) this.Sptr[r] = 1;
+            }
+        }
+
+        // Move M37 rotor if M61 set
+        if (this.rings.M[1][this.Mptr[1]-1]===1) this.Mptr[0]++;
+        if (this.Mptr[0] > ROTOR_SIZES.M37) this.Mptr[0]=1;
+
+        // Always move M61 rotor
+        this.Mptr[1]++;
+        if (this.Mptr[1] > ROTOR_SIZES.M61) this.Mptr[1]=1;
+    }
+
+    /**
+     * Get Q bus inputs
+     */
+    getQbusInputs(charZin) {
+        // Zbits - the bits from the current character from the cipher tape.
+        this.Zbits = this.ITAlookup[charZin].split("");
+        if (this.qbusin.Z === "Z") {
+            // direct Z
+            this.Qbits = this.Zbits;
+        } else if (this.qbusin.Z === "ΔZ") {
+            // delta Z, the Bitwise XOR of this character Zbits + last character Zbits
+            for (let b=0;b<5;b++) {
+                this.Qbits[b] = this.Zbits[b] ^ this.ZbitsOneBack[b];
+            }
+        }
+        this.ZbitsOneBack = this.Zbits.slice(); // copy value of object, not reference
+
+        // Xbits - the current Chi wheel bits
+        for (let b=0;b<5;b++) {
+            this.Xbits[b] = this.rings.X[b+1][this.Xptr[b]-1];
+        }
+        if (this.qbusin.Chi !== "") {
+            if (this.qbusin.Chi === "Χ") {
+                // direct X added to Qbits
+                for (let b=0;b<5;b++) {
+                    this.Qbits[b] = this.Qbits[b] ^ this.Xbits[b];
+                }
+            } else if (this.qbusin.Chi === "ΔΧ") {
+                // delta X
+                for (let b=0;b<5;b++) {
+                    this.Qbits[b] = this.Qbits[b] ^ this.Xbits[b];
+                    this.Qbits[b] = this.Qbits[b] ^ this.XbitsOneBack[b];
+                }
+            }
+        }
+        this.XbitsOneBack = this.Xbits.slice();
+
+        // Sbits - the current Psi wheel bits
+        for (let b=0;b<5;b++) {
+            this.Sbits[b] = this.rings.S[b+1][this.Sptr[b]-1];
+        }
+        if (this.qbusin.Psi !== "") {
+            if (this.qbusin.Psi === "Ψ") {
+                // direct S added to Qbits
+                for (let b=0;b<5;b++) {
+                    this.Qbits[b] = this.Qbits[b] ^ this.Sbits[b];
+                }
+            } else if (this.qbusin.Psi === "ΔΨ") {
+                // delta S
+                for (let b=0;b<5;b++) {
+                    this.Qbits[b] = this.Qbits[b] ^ this.Sbits[b];
+                    this.Qbits[b] = this.Qbits[b] ^ this.SbitsOneBack[b];
+                }
+            }
+        }
+        this.SbitsOneBack = this.Sbits.slice();
+    }
+
+    /**
+     * Conditional impulse Q bus section
+     */
+    runQbusProcessingConditional() {
+        const cnt = [-1, -1, -1, -1, -1];
+        const numrows = this.qbusswitches.condition.length;
+
+        for (let r=0;r<numrows;r++) {
+            const row = this.qbusswitches.condition[r];
+            if (row.Counter !== "") {
+                let result = true;
+                const cPnt = row.Counter-1;
+                const Qswitch = this.readBusSwitches(row.Qswitches);
+                // Match switches to bit pattern
+                for (let s=0;s<5;s++) {
+                    if (Qswitch[s] >= 0 && Qswitch[s] !== parseInt(this.Qbits[s], 10)) result = false;
+                }
+                // Check for NOT switch
+                if (row.Negate) result = !result;
+
+                // AND each row to get final result
+                if (cnt[cPnt] === -1) {
+                    cnt[cPnt] = result;
+                } else if (!result) {
+                    cnt[cPnt] = false;
+                }
+            }
+        }
+
+        // Negate the whole column, this allows A OR B by doing NOT(NOT A AND NOT B)
+        for (let c=0;c<5;c++) {
+            if (this.qbusswitches.condNegateAll && cnt[c] !== -1) cnt[c] = !cnt[c];
+        }
+
+        return cnt;
+    }
+
+    /**
+     * Addition of impulses Q bus section
+     */
+    runQbusProcessingAddition(cnt) {
+        const row = this.qbusswitches.addition[0];
+        const Qswitch = row.Qswitches.slice();
+
+        // To save making the arguments of this operation any larger, limiting addition counter to first one only
+        // Colossus could actually add into any of the five counters.
+        if (row.C1) {
+            let addition = 0;
+            for (let s=0;s<5;s++) {
+                // XOR addition
+                if (Qswitch[s]) {
+                    addition = addition ^ this.Qbits[s];
+                }
+            }
+            const equals = (row.Equals===""?-1:(row.Equals==="."?0:1));
+            if (addition === equals) {
+                // AND with conditional rows to get final result
+                if (cnt[0] === -1) cnt[0] = true;
+            } else {
+                cnt[0] = false;
+            }
+        }
+
+        // Final check, check for addition section negate
+        // then, if any column set, from top to bottom of rack, add to counter.
+        for (let c=0;c<5;c++) {
+            if (this.qbusswitches.addNegateAll && cnt[c] !== -1) cnt[c] = !cnt[c];
+
+            if (this.qbusswitches.totalMotor === "" || (this.qbusswitches.totalMotor === "x" && this.totalmotor === 0) || (this.qbusswitches.totalMotor === "." && this.totalmotor === 1)) {
+                if (cnt[c] === true) this.allCounters[c]++;
+            }
+
+        }
+    }
+
+    /**
+     * Initialise thyratron rings
+     * These hold the pattern of 1s & 0s for each rotor on banks of thyraton GT1C valves which act as a one-bit store.
+     */
+    initThyratrons(pattern) {
+        this.rings = {
+            X: {
+                1: INIT_PATTERNS[pattern].X[1].slice().reverse(),
+                2: INIT_PATTERNS[pattern].X[2].slice().reverse(),
+                3: INIT_PATTERNS[pattern].X[3].slice().reverse(),
+                4: INIT_PATTERNS[pattern].X[4].slice().reverse(),
+                5: INIT_PATTERNS[pattern].X[5].slice().reverse()
+            },
+            M: {
+                1: INIT_PATTERNS[pattern].M[1].slice().reverse(),
+                2: INIT_PATTERNS[pattern].M[2].slice().reverse(),
+            },
+            S: {
+                1: INIT_PATTERNS[pattern].S[1].slice().reverse(),
+                2: INIT_PATTERNS[pattern].S[2].slice().reverse(),
+                3: INIT_PATTERNS[pattern].S[3].slice().reverse(),
+                4: INIT_PATTERNS[pattern].S[4].slice().reverse(),
+                5: INIT_PATTERNS[pattern].S[5].slice().reverse()
+            }
+        };
+    }
+
+    /**
+     * Read argument bus switches X & . and convert to 1 & 0
+     */
+    readBusSwitches(row) {
+        const output = [-1, -1, -1, -1, -1];
+        for (let c=0;c<5;c++) {
+            if (row[c]===".") output[c] = 0;
+            if (row[c]==="x") output[c] = 1;
+        }
+        return output;
+    }
+
+}

src/core/lib/ConvertCoordinates.mjs

@@ -6,9 +6,22 @@
  * @license Apache-2.0
  */
 
+import OperationError from "../errors/OperationError.mjs";
 import geohash from "ngeohash";
+/*
+Currently unable to update to geodesy v2 as we cannot load .js modules into a .mjs file.
+When we do update, imports will look like this:
+
+import LatLonEllipsoidal from "geodesy/latlon-ellipsoidal.js";
+import Mgrs from "geodesy/mgrs.js";
+import OsGridRef from "geodesy/osgridref.js";
+import Utm from "geodesy/utm.js";
+*/
 import geodesy from "geodesy";
-import OperationError from "../errors/OperationError";
+const LatLonEllipsoidal = geodesy.LatLonEllipsoidal,
+    Mgrs = geodesy.Mgrs,
+    OsGridRef = geodesy.OsGridRef,
+    Utm = geodesy.Utm;
 
 /**
  * Co-ordinate formats
@@ -116,22 +129,22 @@ export function convertCoordinates (input, inFormat, inDelim, outFormat, outDeli
     switch (inFormat) {
         case "Geohash":
             hash = geohash.decode(input.replace(/[^A-Za-z0-9]/g, ""));
-            latlon = new geodesy.LatLonEllipsoidal(hash.latitude, hash.longitude);
+            latlon = new LatLonEllipsoidal(hash.latitude, hash.longitude);
             break;
         case "Military Grid Reference System":
-            utm = geodesy.Mgrs.parse(input.replace(/[^A-Za-z0-9]/g, "")).toUtm();
+            utm = Mgrs.parse(input.replace(/[^A-Za-z0-9]/g, "")).toUtm();
             latlon = utm.toLatLonE();
             break;
         case "Ordnance Survey National Grid":
-            osng = geodesy.OsGridRef.parse(input.replace(/[^A-Za-z0-9]/g, ""));
-            latlon = geodesy.OsGridRef.osGridToLatLon(osng);
+            osng = OsGridRef.parse(input.replace(/[^A-Za-z0-9]/g, ""));
+            latlon = OsGridRef.osGridToLatLon(osng);
             break;
         case "Universal Transverse Mercator":
             // Geodesy needs a space between the first 2 digits and the next letter
             if (/^[\d]{2}[A-Za-z]/.test(input)) {
                 input = input.slice(0, 2) + " " + input.slice(2);
             }
-            utm = geodesy.Utm.parse(input);
+            utm = Utm.parse(input);
             latlon = utm.toLatLonE();
             break;
         case "Degrees Minutes Seconds":
@@ -143,7 +156,7 @@ export function convertCoordinates (input, inFormat, inDelim, outFormat, outDeli
                 if (splitLat.length >= 3 && splitLong.length >= 3) {
                     lat = convDMSToDD(splitLat[0], splitLat[1], splitLat[2], 10);
                     lon = convDMSToDD(splitLong[0], splitLong[1], splitLong[2], 10);
-                    latlon = new geodesy.LatLonEllipsoidal(lat.degrees, lon.degrees);
+                    latlon = new LatLonEllipsoidal(lat.degrees, lon.degrees);
                 } else {
                     throw new OperationError("Invalid co-ordinate format for Degrees Minutes Seconds");
                 }
@@ -152,7 +165,7 @@ export function convertCoordinates (input, inFormat, inDelim, outFormat, outDeli
                 splitLat = splitInput(split[0]);
                 if (splitLat.length >= 3) {
                     lat = convDMSToDD(splitLat[0], splitLat[1], splitLat[2]);
-                    latlon = new geodesy.LatLonEllipsoidal(lat.degrees, lat.degrees);
+                    latlon = new LatLonEllipsoidal(lat.degrees, lat.degrees);
                 } else {
                     throw new OperationError("Invalid co-ordinate format for Degrees Minutes Seconds");
                 }
@@ -168,7 +181,7 @@ export function convertCoordinates (input, inFormat, inDelim, outFormat, outDeli
                 // Convert to decimal degrees, and then convert to a geodesy object
                 lat = convDDMToDD(splitLat[0], splitLat[1], 10);
                 lon = convDDMToDD(splitLong[0], splitLong[1], 10);
-                latlon = new geodesy.LatLonEllipsoidal(lat.degrees, lon.degrees);
+                latlon = new LatLonEllipsoidal(lat.degrees, lon.degrees);
             } else {
                 // Not a pair, so only try to convert one set of co-ordinates
                 splitLat = splitInput(input);
@@ -176,7 +189,7 @@ export function convertCoordinates (input, inFormat, inDelim, outFormat, outDeli
                     throw new OperationError("Invalid co-ordinate format for Degrees Decimal Minutes.");
                 }
                 lat = convDDMToDD(splitLat[0], splitLat[1], 10);
-                latlon = new geodesy.LatLonEllipsoidal(lat.degrees, lat.degrees);
+                latlon = new LatLonEllipsoidal(lat.degrees, lat.degrees);
             }
             break;
         case "Decimal Degrees":
@@ -186,14 +199,14 @@ export function convertCoordinates (input, inFormat, inDelim, outFormat, outDeli
                 if (splitLat.length !== 1 || splitLong.length !== 1) {
                     throw new OperationError("Invalid co-ordinate format for Decimal Degrees.");
                 }
-                latlon = new geodesy.LatLonEllipsoidal(splitLat[0], splitLong[0]);
+                latlon = new LatLonEllipsoidal(splitLat[0], splitLong[0]);
             } else {
                 // Not a pair, so only try to convert one set of co-ordinates
                 splitLat = splitInput(split[0]);
                 if (splitLat.length !== 1) {
                     throw new OperationError("Invalid co-ordinate format for Decimal Degrees.");
                 }
-                latlon = new geodesy.LatLonEllipsoidal(splitLat[0], splitLat[0]);
+                latlon = new LatLonEllipsoidal(splitLat[0], splitLat[0]);
             }
             break;
         default:
@@ -260,7 +273,7 @@ export function convertCoordinates (input, inFormat, inDelim, outFormat, outDeli
             convLat = mgrs.toString(precision);
             break;
         case "Ordnance Survey National Grid":
-            osng = geodesy.OsGridRef.latLonToOsGrid(latlon);
+            osng = OsGridRef.latLonToOsGrid(latlon);
             if (osng.toString() === "") {
                 throw new OperationError("Could not convert co-ordinates to OS National Grid. Are the co-ordinates in range?");
             }
@@ -327,13 +340,13 @@ export function convertCoordinates (input, inFormat, inDelim, outFormat, outDeli
  * @param {string} input - The input data to be split
  * @returns {number[]} An array of the different items in the string, stored as floats
  */
-function splitInput (input){
+function splitInput (input) {
     const split = [];
 
     input.split(/\s+/).forEach(item => {
         // Remove any character that isn't a digit, decimal point or negative sign
         item = item.replace(/[^0-9.-]/g, "");
-        if (item.length > 0){
+        if (item.length > 0) {
             // Turn the item into a float
             split.push(parseFloat(item));
         }
@@ -350,7 +363,7 @@ function splitInput (input){
  * @param {number} precision - The precision the result should be rounded to
  * @returns {{string: string, degrees: number}} An object containing the raw converted value (obj.degrees), and a formatted string version (obj.string)
  */
-function convDMSToDD (degrees, minutes, seconds, precision){
+function convDMSToDD (degrees, minutes, seconds, precision) {
     const absDegrees = Math.abs(degrees);
     let conv = absDegrees + (minutes / 60) + (seconds / 3600);
     let outString = round(conv, precision) + "°";
@@ -566,7 +579,7 @@ export function findFormat (input, delim) {
     // Test DMS/DDM/DD formats
     if (testData !== undefined) {
         const split = splitInput(testData);
-        switch (split.length){
+        switch (split.length) {
             case 3:
                 return "Degrees Minutes Seconds";
             case 2:

src/core/lib/Crypt.mjs

@@ -0,0 +1,9 @@
+/**
+ * Crypt resources.
+ *
+ * @author n1474335 [n1474335@gmail.com]
+ * @copyright Crown Copyright 2021
+ * @license Apache-2.0
+ */
+
+export const cryptNotice = "WARNING: Cryptographic operations in CyberChef should not be relied upon to provide security in any situation. No guarantee is offered for their correctness. We advise you not to use keys generated from CyberChef in operational contexts.";

src/core/lib/Decimal.mjs

@@ -6,7 +6,7 @@
  * @license Apache-2.0
  */
 
-import Utils from "../Utils";
+import Utils from "../Utils.mjs";
 
 
 /**

src/core/lib/Delim.mjs

@@ -32,7 +32,7 @@ export const WORD_DELIM_OPTIONS = ["Line feed", "CRLF", "Forward slash", "Backsl
 export const INPUT_DELIM_OPTIONS = ["Line feed", "CRLF", "Space", "Comma", "Semi-colon", "Colon", "Nothing (separate chars)"];
 
 /**
- * Armithmetic sequence delimiters
+ * Arithmetic sequence delimiters
  */
 export const ARITHMETIC_DELIM_OPTIONS = ["Line feed", "Space", "Comma", "Semi-colon", "Colon", "CRLF"];
 
@@ -72,3 +72,12 @@ export const JOIN_DELIM_OPTIONS = [
     {name: "Nothing (join chars)", value: ""}
 ];
 
+/**
+ * RGBA list delimiters.
+ */
+export const RGBA_DELIM_OPTIONS = [
+    {name: "Comma", value: ","},
+    {name: "Space", value: " "},
+    {name: "CRLF", value: "\\r\\n"},
+    {name: "Line Feed", value: "\n"}
+];

src/core/lib/Enigma.mjs

@@ -0,0 +1,369 @@
+/**
+ * Emulation of the Enigma machine.
+ *
+ * @author s2224834
+ * @copyright Crown Copyright 2019
+ * @license Apache-2.0
+ */
+import OperationError from "../errors/OperationError.mjs";
+import Utils from "../Utils.mjs";
+
+/**
+ * Provided default Enigma rotor set.
+ * These are specified as a list of mappings from the letters A through Z in order, optionally
+ * followed by < and a list of letters at which the rotor steps.
+ */
+export const ROTORS = [
+    {name: "I", value: "EKMFLGDQVZNTOWYHXUSPAIBRCJ<R"},
+    {name: "II", value: "AJDKSIRUXBLHWTMCQGZNPYFVOE<F"},
+    {name: "III", value: "BDFHJLCPRTXVZNYEIWGAKMUSQO<W"},
+    {name: "IV", value: "ESOVPZJAYQUIRHXLNFTGKDCMWB<K"},
+    {name: "V", value: "VZBRGITYUPSDNHLXAWMJQOFECK<A"},
+    {name: "VI", value: "JPGVOUMFYQBENHZRDKASXLICTW<AN"},
+    {name: "VII", value: "NZJHGRCXMYSWBOUFAIVLPEKQDT<AN"},
+    {name: "VIII", value: "FKQHTLXOCBJSPDZRAMEWNIUYGV<AN"},
+];
+
+export const ROTORS_FOURTH = [
+    {name: "Beta", value: "LEYJVCNIXWPBQMDRTAKZGFUHOS"},
+    {name: "Gamma", value: "FSOKANUERHMBTIYCWLQPZXVGJD"},
+];
+
+/**
+ * Provided default Enigma reflector set.
+ * These are specified as 13 space-separated transposed pairs covering every letter.
+ */
+export const REFLECTORS = [
+    {name: "B", value: "AY BR CU DH EQ FS GL IP JX KN MO TZ VW"},
+    {name: "C", value: "AF BV CP DJ EI GO HY KR LZ MX NW TQ SU"},
+    {name: "B Thin", value: "AE BN CK DQ FU GY HW IJ LO MP RX SZ TV"},
+    {name: "C Thin", value: "AR BD CO EJ FN GT HK IV LM PW QZ SX UY"},
+];
+
+export const LETTERS = "ABCDEFGHIJKLMNOPQRSTUVWXYZ".split("");
+
+/**
+ * Map a letter to a number in 0..25.
+ *
+ * @param {char} c
+ * @param {boolean} permissive - Case insensitive; don't throw errors on other chars.
+ * @returns {number}
+ */
+export function a2i(c, permissive=false) {
+    const i = Utils.ord(c);
+    if (i >= 65 && i <= 90) {
+        return i - 65;
+    }
+    if (permissive) {
+        // Allow case insensitivity
+        if (i >= 97 && i <= 122) {
+            return i - 97;
+        }
+        return -1;
+    }
+    throw new OperationError("a2i called on non-uppercase ASCII character");
+}
+
+/**
+ * Map a number in 0..25 to a letter.
+ *
+ * @param {number} i
+ * @returns {char}
+ */
+export function i2a(i) {
+    if (i >= 0 && i < 26) {
+        return Utils.chr(i+65);
+    }
+    throw new OperationError("i2a called on value outside 0..25");
+}
+
+/**
+ * A rotor in the Enigma machine.
+ */
+export class Rotor {
+    /**
+     * Rotor constructor.
+     *
+     * @param {string} wiring - A 26 character string of the wiring order.
+     * @param {string} steps - A 0..26 character string of stepping points.
+     * @param {char} ringSetting - The ring setting.
+     * @param {char} initialPosition - The initial position of the rotor.
+     */
+    constructor(wiring, steps, ringSetting, initialPosition) {
+        if (!/^[A-Z]{26}$/.test(wiring)) {
+            throw new OperationError("Rotor wiring must be 26 unique uppercase letters");
+        }
+        if (!/^[A-Z]{0,26}$/.test(steps)) {
+            throw new OperationError("Rotor steps must be 0-26 unique uppercase letters");
+        }
+        if (!/^[A-Z]$/.test(ringSetting)) {
+            throw new OperationError("Rotor ring setting must be exactly one uppercase letter");
+        }
+        if (!/^[A-Z]$/.test(initialPosition)) {
+            throw new OperationError("Rotor initial position must be exactly one uppercase letter");
+        }
+        this.map = new Array(26);
+        this.revMap = new Array(26);
+        const uniq = {};
+        for (let i=0; i<LETTERS.length; i++) {
+            const a = a2i(LETTERS[i]);
+            const b = a2i(wiring[i]);
+            this.map[a] = b;
+            this.revMap[b] = a;
+            uniq[b] = true;
+        }
+        if (Object.keys(uniq).length !== LETTERS.length) {
+            throw new OperationError("Rotor wiring must have each letter exactly once");
+        }
+        const rs = a2i(ringSetting);
+        this.steps = new Set();
+        for (const x of steps) {
+            this.steps.add(Utils.mod(a2i(x) - rs, 26));
+        }
+        if (this.steps.size !== steps.length) {
+            // This isn't strictly fatal, but it's probably a mistake
+            throw new OperationError("Rotor steps must be unique");
+        }
+        this.pos = Utils.mod(a2i(initialPosition) - rs, 26);
+    }
+
+    /**
+     * Step the rotor forward by one.
+     */
+    step() {
+        this.pos = Utils.mod(this.pos + 1, 26);
+        return this.pos;
+    }
+
+    /**
+     * Transform a character through this rotor forwards.
+     *
+     * @param {number} c - The character.
+     * @returns {number}
+     */
+    transform(c) {
+        return Utils.mod(this.map[Utils.mod(c + this.pos, 26)] - this.pos, 26);
+    }
+
+    /**
+     * Transform a character through this rotor backwards.
+     *
+     * @param {number} c - The character.
+     * @returns {number}
+     */
+    revTransform(c) {
+        return Utils.mod(this.revMap[Utils.mod(c + this.pos, 26)] - this.pos, 26);
+    }
+}
+
+/**
+ * Base class for plugboard and reflector (since these do effectively the same
+ * thing).
+ */
+class PairMapBase {
+    /**
+     * PairMapBase constructor.
+     *
+     * @param {string} pairs - A whitespace separated string of letter pairs to swap.
+     * @param {string} [name='PairMapBase'] - For errors, the name of this object.
+     */
+    constructor(pairs, name="PairMapBase") {
+        // I've chosen to make whitespace significant here to make a) code and
+        // b) inputs easier to read
+        this.pairs = pairs;
+        this.map = {};
+        if (pairs === "") {
+            return;
+        }
+        pairs.split(/\s+/).forEach(pair => {
+            if (!/^[A-Z]{2}$/.test(pair)) {
+                throw new OperationError(name + " must be a whitespace-separated list of uppercase letter pairs");
+            }
+            const a = a2i(pair[0]), b = a2i(pair[1]);
+            if (a === b) {
+                // self-stecker
+                return;
+            }
+            if (Object.prototype.hasOwnProperty.call(this.map, a)) {
+                throw new OperationError(`${name} connects ${pair[0]} more than once`);
+            }
+            if (Object.prototype.hasOwnProperty.call(this.map, b)) {
+                throw new OperationError(`${name} connects ${pair[1]} more than once`);
+            }
+            this.map[a] = b;
+            this.map[b] = a;
+        });
+    }
+
+    /**
+     * Transform a character through this object.
+     * Returns other characters unchanged.
+     *
+     * @param {number} c - The character.
+     * @returns {number}
+     */
+    transform(c) {
+        if (!Object.prototype.hasOwnProperty.call(this.map, c)) {
+            return c;
+        }
+        return this.map[c];
+    }
+
+    /**
+     * Alias for transform, to allow interchangeable use with rotors.
+     *
+     * @param {number} c - The character.
+     * @returns {number}
+     */
+    revTransform(c) {
+        return this.transform(c);
+    }
+}
+
+/**
+ * Reflector. PairMapBase but requires that all characters are accounted for.
+ *
+ * Includes a couple of optimisations on that basis.
+ */
+export class Reflector extends PairMapBase {
+    /**
+     * Reflector constructor. See PairMapBase.
+     * Additional restriction: every character must be accounted for.
+     */
+    constructor(pairs) {
+        super(pairs, "Reflector");
+        const s = Object.keys(this.map).length;
+        if (s !== 26) {
+            throw new OperationError("Reflector must have exactly 13 pairs covering every letter");
+        }
+        const optMap = new Array(26);
+        for (const x of Object.keys(this.map)) {
+            optMap[x] = this.map[x];
+        }
+        this.map = optMap;
+    }
+
+    /**
+     * Transform a character through this object.
+     *
+     * @param {number} c - The character.
+     * @returns {number}
+     */
+    transform(c) {
+        return this.map[c];
+    }
+}
+
+/**
+ * Plugboard. Unmodified PairMapBase.
+ */
+export class Plugboard extends PairMapBase {
+    /**
+     * Plugboard constructor. See PairMapbase.
+     */
+    constructor(pairs) {
+        super(pairs, "Plugboard");
+    }
+}
+
+/**
+ * Base class for the Enigma machine itself. Holds rotors, a reflector, and a plugboard.
+ */
+export class EnigmaBase {
+    /**
+     * EnigmaBase constructor.
+     *
+     * @param {Object[]} rotors - List of Rotors.
+     * @param {Object} reflector - A Reflector.
+     * @param {Plugboard} plugboard - A Plugboard.
+     */
+    constructor(rotors, reflector, plugboard) {
+        this.rotors = rotors;
+        this.rotorsRev = [].concat(rotors).reverse();
+        this.reflector = reflector;
+        this.plugboard = plugboard;
+    }
+
+    /**
+     * Step the rotors forward by one.
+     *
+     * This happens before the output character is generated.
+     *
+     * Note that rotor 4, if it's there, never steps.
+     *
+     * Why is all the logic in EnigmaBase and not a nice neat method on
+     * Rotor that knows when it should advance the next item?
+     * Because the double stepping anomaly is a thing. tl;dr if the left rotor
+     * should step the next time the middle rotor steps, the middle rotor will
+     * immediately step.
+     */
+    step() {
+        const r0 = this.rotors[0];
+        const r1 = this.rotors[1];
+        r0.step();
+        // The second test here is the double-stepping anomaly
+        if (r0.steps.has(r0.pos) || r1.steps.has(Utils.mod(r1.pos + 1, 26))) {
+            r1.step();
+            if (r1.steps.has(r1.pos)) {
+                const r2 = this.rotors[2];
+                r2.step();
+            }
+        }
+    }
+
+    /**
+     * Encrypt (or decrypt) some data.
+     * Takes an arbitrary string and runs the Engima machine on that data from
+     * *its current state*, and outputs the result. Non-alphabetic characters
+     * are returned unchanged.
+     *
+     * @param {string} input - Data to encrypt.
+     * @returns {string}
+     */
+    crypt(input) {
+        let result = "";
+        for (const c of input) {
+            let letter = a2i(c, true);
+            if (letter === -1) {
+                result += c;
+                continue;
+            }
+            // First, step the rotors forward.
+            this.step();
+            // Now, run through the plugboard.
+            letter = this.plugboard.transform(letter);
+            // Then through each wheel in sequence, through the reflector, and
+            // backwards through the wheels again.
+            for (const rotor of this.rotors) {
+                letter = rotor.transform(letter);
+            }
+            letter = this.reflector.transform(letter);
+            for (const rotor of this.rotorsRev) {
+                letter = rotor.revTransform(letter);
+            }
+            // Finally, back through the plugboard.
+            letter = this.plugboard.revTransform(letter);
+            result += i2a(letter);
+        }
+        return result;
+    }
+}
+
+/**
+ * The Enigma machine itself. Holds 3-4 rotors, a reflector, and a plugboard.
+ */
+export class EnigmaMachine extends EnigmaBase {
+    /**
+     * EnigmaMachine constructor.
+     *
+     * @param {Object[]} rotors - List of Rotors.
+     * @param {Object} reflector - A Reflector.
+     * @param {Plugboard} plugboard - A Plugboard.
+     */
+    constructor(rotors, reflector, plugboard) {
+        super(rotors, reflector, plugboard);
+        if (rotors.length !== 3 && rotors.length !== 4) {
+            throw new OperationError("Enigma must have 3 or 4 rotors");
+        }
+    }
+}

src/core/lib/Extract.mjs

@@ -12,15 +12,15 @@
  *
  * @param {string} input
  * @param {RegExp} searchRegex
- * @param {RegExp} removeRegex - A regular expression defining results to remove from the
+ * @param {RegExp} [removeRegex=null] - A regular expression defining results to remove from the
  *      final list
- * @param {boolean} includeTotal - Whether or not to include the total number of results
+ * @param {Function} [sortBy=null] - The sorting comparison function to apply
+ * @param {boolean} [unique=false] - Whether to unique the results
  * @returns {string}
  */
-export function search (input, searchRegex, removeRegex, includeTotal) {
-    let output = "",
-        total = 0,
-        match;
+export function search(input, searchRegex, removeRegex=null, sortBy=null, unique=false) {
+    let results = [];
+    let match;
 
     while ((match = searchRegex.exec(input))) {
         // Moves pointer when an empty string is matched (prevents infinite loop)
@@ -30,14 +30,19 @@ export function search (input, searchRegex, removeRegex, includeTotal) {
 
         if (removeRegex && removeRegex.test(match[0]))
             continue;
-        total++;
-        output += match[0] + "\n";
+
+        results.push(match[0]);
+    }
+
+    if (sortBy) {
+        results = results.sort(sortBy);
     }
 
-    if (includeTotal)
-        output = "Total found: " + total + "\n\n" + output;
+    if (unique) {
+        results = results.unique();
+    }
 
-    return output;
+    return results;
 }
 
 

src/core/lib/FileSignatures.mjs

@@ -0,0 +1,4191 @@
+/**
+ * File signatures and extractor functions
+ *
+ * @author n1474335 [n1474335@gmail.com]
+ * @copyright Crown Copyright 2018
+ * @license Apache-2.0
+ *
+ */
+import Stream from "./Stream.mjs";
+
+/**
+ * A categorised table of file types, including signatures to identify them and functions
+ * to extract them where possible.
+ */
+export const FILE_SIGNATURES = {
+    "Images": [
+        {
+            name: "Joint Photographic Experts Group image",
+            extension: "jpg,jpeg,jpe,thm,mpo",
+            mime: "image/jpeg",
+            description: "",
+            signature: {
+                0: 0xff,
+                1: 0xd8,
+                2: 0xff,
+                3: [0xc0, 0xc4, 0xdb, 0xdd, 0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5, 0xe7, 0xe8, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xfe]
+            },
+            extractor: extractJPEG
+        },
+        {
+            name: "Graphics Interchange Format image",
+            extension: "gif",
+            mime: "image/gif",
+            description: "",
+            signature: {
+                0: 0x47, // GIF
+                1: 0x49,
+                2: 0x46,
+                3: 0x38, // 8
+                4: [0x37, 0x39], // 7|9
+                5: 0x61  // a
+            },
+            extractor: extractGIF
+        },
+        {
+            name: "Portable Network Graphics image",
+            extension: "png",
+            mime: "image/png",
+            description: "",
+            signature: {
+                0: 0x89,
+                1: 0x50, // PNG
+                2: 0x4e,
+                3: 0x47,
+                4: 0x0d,
+                5: 0x0a,
+                6: 0x1a,
+                7: 0x0a
+            },
+            extractor: extractPNG
+        },
+        {
+            name: "WEBP Image",
+            extension: "webp",
+            mime: "image/webp",
+            description: "",
+            signature: {
+                8: 0x57,
+                9: 0x45,
+                10: 0x42,
+                11: 0x50
+            },
+            extractor: extractWEBP
+        },
+        {
+            name: "Camera Image File Format",
+            extension: "crw",
+            mime: "image/x-canon-crw",
+            description: "",
+            signature: {
+                6: 0x48, // HEAPCCDR
+                7: 0x45,
+                8: 0x41,
+                9: 0x50,
+                10: 0x43,
+                11: 0x43,
+                12: 0x44,
+                13: 0x52
+            },
+            extractor: null
+        },
+        { // Place before tiff check
+            name: "Canon CR2 raw image",
+            extension: "cr2",
+            mime: "image/x-canon-cr2",
+            description: "",
+            signature: [
+                {
+                    0: 0x49,
+                    1: 0x49,
+                    2: 0x2a,
+                    3: 0x0,
+                    8: 0x43,
+                    9: 0x52
+                },
+                {
+                    0: 0x4d,
+                    1: 0x4d,
+                    2: 0x0,
+                    3: 0x2a,
+                    8: 0x43,
+                    9: 0x52
+                }
+            ],
+            extractor: null
+        },
+        {
+            name: "Tagged Image File Format image",
+            extension: "tif",
+            mime: "image/tiff",
+            description: "",
+            signature: [
+                {
+                    0: 0x49,
+                    1: 0x49,
+                    2: 0x2a,
+                    3: 0x0
+                },
+                {
+                    0: 0x4d,
+                    1: 0x4d,
+                    2: 0x0,
+                    3: 0x2a
+                }
+            ],
+            extractor: null
+        },
+        {
+            name: "Bitmap image",
+            extension: "bmp",
+            mime: "image/bmp",
+            description: "",
+            signature: {
+                0: 0x42,
+                1: 0x4d,
+                7: 0x0,
+                9: 0x0,
+                14: [0x0c, 0x28, 0x38, 0x40, 0x6c, 0x7c],
+                15: 0x0,
+                16: 0x0,
+                17: 0x0
+            },
+            extractor: extractBMP
+        },
+        {
+            name: "JPEG Extended Range image",
+            extension: "jxr",
+            mime: "image/vnd.ms-photo",
+            description: "",
+            signature: {
+                0: 0x49,
+                1: 0x49,
+                2: 0xbc
+            },
+            extractor: null
+        },
+        {
+            name: "Photoshop image",
+            extension: "psd",
+            mime: "image/vnd.adobe.photoshop",
+            description: "",
+            signature: {
+                0: 0x38, // 8BPS
+                1: 0x42,
+                2: 0x50,
+                3: 0x53,
+                4: 0x0,
+                5: 0x1,
+                6: 0x0,
+                7: 0x0,
+                8: 0x0,
+                9: 0x0,
+                10: 0x0,
+                11: 0x0
+            },
+            extractor: null
+        },
+        {
+            name: "Photoshop Large Document",
+            extension: "psb",
+            mime: "application/x-photoshop",
+            description: "",
+            signature: {
+                0: 0x38, // 8BPS
+                1: 0x42,
+                2: 0x50,
+                3: 0x53,
+                4: 0x0,
+                5: 0x2,
+                6: 0x0,
+                7: 0x0,
+                8: 0x0,
+                9: 0x0,
+                10: 0x0,
+                11: 0x0,
+                12: 0x0
+            },
+            extractor: null
+        },
+        {
+            name: "Paint Shop Pro image",
+            extension: "psp",
+            mime: "image/psp",
+            description: "",
+            signature: [
+                {
+                    0: 0x50, // Paint Shop Pro Im
+                    1: 0x61,
+                    2: 0x69,
+                    3: 0x6e,
+                    4: 0x74,
+                    5: 0x20,
+                    6: 0x53,
+                    7: 0x68,
+                    8: 0x6f,
+                    9: 0x70,
+                    10: 0x20,
+                    11: 0x50,
+                    12: 0x72,
+                    13: 0x6f,
+                    14: 0x20,
+                    15: 0x49,
+                    16: 0x6d
+                },
+                {
+                    0: 0x7e,
+                    1: 0x42,
+                    2: 0x4b,
+                    3: 0x0
+                }
+            ],
+            extractor: null
+        },
+        {
+            name: "The GIMP image",
+            extension: "xcf",
+            mime: "image/x-xcf",
+            description: "",
+            signature: {
+                0: 0x67, // gimp xcf
+                1: 0x69,
+                2: 0x6d,
+                3: 0x70,
+                4: 0x20,
+                5: 0x78,
+                6: 0x63,
+                7: 0x66,
+                8: 0x20,
+                9: [0x66, 0x76],
+                10: [0x69, 0x30],
+                11: [0x6c, 0x30],
+                12: [0x65, 0x31, 0x32, 0x33]
+            },
+            extractor: null
+        },
+        {
+            name: "Icon image",
+            extension: "ico",
+            mime: "image/x-icon",
+            description: "",
+            signature: {
+                0: 0x0,
+                1: 0x0,
+                2: 0x1,
+                3: 0x0,
+                4: [0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8, 0x9, 0xa, 0xb, 0xc, 0xd, 0xe, 0xf, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15],
+                5: 0x0,
+                6: [0x10, 0x20, 0x30, 0x40, 0x80],
+                7: [0x10, 0x20, 0x30, 0x40, 0x80],
+                9: 0x0,
+                10: [0x0, 0x1]
+            },
+            extractor: extractICO
+        },
+        {
+            name: "Radiance High Dynamic Range image",
+            extension: "hdr",
+            mime: "image/vnd.radiance",
+            description: "",
+            signature: {
+                0: 0x23, // #?RADIANCE
+                1: 0x3f,
+                2: 0x52,
+                3: 0x41,
+                4: 0x44,
+                5: 0x49,
+                6: 0x41,
+                7: 0x4e,
+                8: 0x43,
+                9: 0x45,
+                10: 0x0a
+            },
+            extractor: null
+        },
+        {
+            name: "Sony ARW image",
+            extension: "arw",
+            mime: "image/x-raw",
+            description: "",
+            signature: {
+                0: 0x05,
+                1: 0x0,
+                2: 0x0,
+                3: 0x0,
+                4: 0x41,
+                5: 0x57,
+                6: 0x31,
+                7: 0x2e
+            },
+            extractor: null
+        },
+        {
+            name: "Fujifilm Raw Image",
+            extension: "raf",
+            mime: "image/x-raw",
+            description: "",
+            signature: {
+                0: 0x46, // FUJIFILMCCD-RAW
+                1: 0x55,
+                2: 0x4a,
+                3: 0x49,
+                4: 0x46,
+                5: 0x49,
+                6: 0x4c,
+                7: 0x4d,
+                8: 0x43,
+                9: 0x43,
+                10: 0x44,
+                11: 0x2d,
+                12: 0x52,
+                13: 0x41,
+                14: 0x57
+            },
+            extractor: null
+        },
+        {
+            name: "Minolta RAW image",
+            extension: "mrw",
+            mime: "image/x-raw",
+            description: "",
+            signature: {
+                0: 0x0,
+                1: 0x4d, // MRM
+                2: 0x52,
+                3: 0x4d
+            },
+            extractor: null
+        },
+        {
+            name: "Adobe Bridge Thumbnail Cache",
+            extension: "bct",
+            mime: "application/octet-stream",
+            description: "",
+            signature: {
+                0: 0x6c,
+                1: 0x6e,
+                2: 0x62,
+                3: 0x74,
+                4: 0x02,
+                5: 0x0,
+                6: 0x0,
+                7: 0x0
+            },
+            extractor: null
+        },
+        {
+            name: "Microsoft Document Imaging",
+            extension: "mdi",
+            mime: "image/vnd.ms-modi",
+            description: "",
+            signature: {
+                0: 0x45,
+                1: 0x50,
+                2: 0x2a,
+                3: 0x00
+            },
+            extractor: null
+        },
+        {
+            name: "Joint Photographic Experts Group image (under Base64)",
+            extension: "B64",
+            mime: "application/octet-stream",
+            description: "",
+            signature: {
+                0: 0x2f,
+                1: 0x39,
+                2: 0x6a,
+                3: 0x2f,
+                4: 0x34
+            },
+            extractor: null
+        },
+        {
+            name: "Portable Network Graphics image (under Base64)",
+            extension: "B64",
+            mime: "application/octet-stream",
+            description: "",
+            signature: {
+                0: 0x69,
+                1: 0x56,
+                2: 0x42,
+                3: 0x4f,
+                4: 0x52,
+                5: 0x77,
+                6: 0x30
+            },
+            extractor: null
+        },
+        {
+            name: "AutoCAD Drawing",
+            extension: "dwg,123d",
+            mime: "application/acad",
+            description: "",
+            signature: {
+                0: 0x41,
+                1: 0x43,
+                2: 0x31,
+                3: 0x30,
+                4: [0x30, 0x31],
+                5: [0x30, 0x31, 0x32, 0x33, 0x34, 0x35],
+                6: 0x00
+            },
+            extractor: null
+        },
+        {
+            name: "AutoCAD Drawing",
+            extension: "dwg,dwt",
+            mime: "application/acad",
+            description: "",
+            signature: [
+                {
+                    0: 0x41,
+                    1: 0x43,
+                    2: 0x31,
+                    3: 0x30,
+                    4: 0x31,
+                    5: 0x38,
+                    6: 0x00
+                },
+                {
+                    0: 0x41,
+                    1: 0x43,
+                    2: 0x31,
+                    3: 0x30,
+                    4: 0x32,
+                    5: 0x34,
+                    6: 0x00
+                },
+                {
+                    0: 0x41,
+                    1: 0x43,
+                    2: 0x31,
+                    3: 0x30,
+                    4: 0x32,
+                    5: 0x37,
+                    6: 0x00
+                }
+            ],
+            extractor: null
+        },
+        {
+            name: "Targa Image",
+            extension: "tga",
+            mime: "image/x-targa",
+            description: "",
+            signature: [
+                { // This signature is not at the beginning of the file. The extractor works backwards.
+                    0: 0x54,
+                    1: 0x52,
+                    2: 0x55,
+                    3: 0x45,
+                    4: 0x56,
+                    5: 0x49,
+                    6: 0x53,
+                    7: 0x49,
+                    8: 0x4f,
+                    9: 0x4e,
+                    10: 0x2d,
+                    11: 0x58,
+                    12: 0x46,
+                    13: 0x49,
+                    14: 0x4c,
+                    15: 0x45,
+                    16: 0x2e
+                }
+            ],
+            extractor: extractTARGA
+        }
+    ],
+    "Video": [
+        { // Place before webm
+            name: "Matroska Multimedia Container",
+            extension: "mkv",
+            mime: "video/x-matroska",
+            description: "",
+            signature: {
+                31: 0x6d,
+                32: 0x61,
+                33: 0x74,
+                34: 0x72,
+                35: 0x6f,
+                36: 0x73,
+                37: 0x6b,
+                38: 0x61
+            },
+            extractor: null
+        },
+        {
+            name: "WEBM video",
+            extension: "webm",
+            mime: "video/webm",
+            description: "",
+            signature: {
+                0: 0x1a,
+                1: 0x45,
+                2: 0xdf,
+                3: 0xa3
+            },
+            extractor: null
+        },
+        { // Place before MPEG-4
+            name: "Flash MP4 video",
+            extension: "f4v",
+            mime: "video/mp4",
+            description: "",
+            signature: {
+                4: 0x66,
+                5: 0x74,
+                6: 0x79,
+                7: 0x70,
+                8: [0x66, 0x46],
+                9: 0x34,
+                10: [0x76, 0x56],
+                11: 0x20
+            },
+            extractor: null
+        },
+        {
+            name: "MPEG-4 video",
+            extension: "mp4",
+            mime: "video/mp4",
+            description: "",
+            signature: [
+                {
+                    0: 0x0,
+                    1: 0x0,
+                    2: 0x0,
+                    3: [0x18, 0x20],
+                    4: 0x66,
+                    5: 0x74,
+                    6: 0x79,
+                    7: 0x70
+                },
+                {
+                    0: 0x33, // 3gp5
+                    1: 0x67,
+                    2: 0x70,
+                    3: 0x35
+                },
+                {
+                    0: 0x0,
+                    1: 0x0,
+                    2: 0x0,
+                    3: 0x1c,
+                    4: 0x66,
+                    5: 0x74,
+                    6: 0x79,
+                    7: 0x70,
+                    8: 0x6d,
+                    9: 0x70,
+                    10: 0x34,
+                    11: 0x32,
+                    16: 0x6d, // mp41mp42isom
+                    17: 0x70,
+                    18: 0x34,
+                    19: 0x31,
+                    20: 0x6d,
+                    21: 0x70,
+                    22: 0x34,
+                    23: 0x32,
+                    24: 0x69,
+                    25: 0x73,
+                    26: 0x6f,
+                    27: 0x6d
+                }
+            ],
+            extractor: null
+        },
+        {
+            name: "M4V video",
+            extension: "m4v",
+            mime: "video/x-m4v",
+            description: "",
+            signature: {
+                0: 0x0,
+                1: 0x0,
+                2: 0x0,
+                3: 0x1c,
+                4: 0x66,
+                5: 0x74,
+                6: 0x79,
+                7: 0x70,
+                8: 0x4d,
+                9: 0x34,
+                10: 0x56
+            },
+            extractor: null
+        },
+        {
+            name: "Quicktime video",
+            extension: "mov",
+            mime: "video/quicktime",
+            description: "",
+            signature: {
+                0: 0x0,
+                1: 0x0,
+                2: 0x0,
+                3: 0x14,
+                4: 0x66,
+                5: 0x74,
+                6: 0x79,
+                7: 0x70
+            },
+            extractor: null
+        },
+        {
+            name: "Audio Video Interleave",
+            extension: "avi",
+            mime: "video/x-msvideo",
+            description: "",
+            signature: {
+                0: 0x52,
+                1: 0x49,
+                2: 0x46,
+                3: 0x46,
+                8: 0x41,
+                9: 0x56,
+                10: 0x49
+            },
+            extractor: null
+        },
+        {
+            name: "Windows Media Video",
+            extension: "wmv",
+            mime: "video/x-ms-wmv",
+            description: "",
+            signature: {
+                0: 0x30,
+                1: 0x26,
+                2: 0xb2,
+                3: 0x75,
+                4: 0x8e,
+                5: 0x66,
+                6: 0xcf,
+                7: 0x11,
+                8: 0xa6,
+                9: 0xd9
+            },
+            extractor: null
+        },
+        {
+            name: "MPEG video",
+            extension: "mpg",
+            mime: "video/mpeg",
+            description: "",
+            signature: {
+                0: 0x0,
+                1: 0x0,
+                2: 0x1,
+                3: 0xba
+            },
+            extractor: null
+        },
+        {
+            name: "Flash Video",
+            extension: "flv",
+            mime: "video/x-flv",
+            description: "",
+            signature: {
+                0: 0x46,
+                1: 0x4c,
+                2: 0x56,
+                3: 0x1
+            },
+            extractor: extractFLV
+        },
+        {
+            name: "OGG Video",
+            extension: "ogv,ogm,opus,ogx",
+            mime: "video/ogg",
+            description: "",
+            signature: [
+                {
+                    0: 0x4f, // OggS
+                    1: 0x67,
+                    2: 0x67,
+                    3: 0x53,
+                    4: 0x00,
+                    5: 0x02,
+                    28: 0x01,
+                    29: 0x76, // video
+                    30: 0x69,
+                    31: 0x64,
+                    32: 0x65,
+                    33: 0x6f
+                },
+                {
+                    0: 0x4f, // OggS
+                    1: 0x67,
+                    2: 0x67,
+                    3: 0x53,
+                    4: 0x00,
+                    5: 0x02,
+                    28: 0x80,
+                    29: 0x74, // theora
+                    30: 0x68,
+                    31: 0x65,
+                    32: 0x6f,
+                    33: 0x72,
+                    34: 0x61
+                },
+                {
+                    0: 0x4f, // OggS
+                    1: 0x67,
+                    2: 0x67,
+                    3: 0x53,
+                    4: 0x00,
+                    5: 0x02,
+                    28: 0x66, // fishead
+                    29: 0x69,
+                    30: 0x73,
+                    31: 0x68,
+                    32: 0x65,
+                    33: 0x61,
+                    34: 0x64
+                }
+            ],
+            extractor: null
+        },
+    ],
+    "Audio": [
+        {
+            name: "Waveform Audio",
+            extension: "wav",
+            mime: "audio/x-wav",
+            description: "",
+            signature: {
+                0: 0x52,
+                1: 0x49,
+                2: 0x46,
+                3: 0x46,
+                8: 0x57,
+                9: 0x41,
+                10: 0x56,
+                11: 0x45
+            },
+            extractor: extractWAV
+        },
+        {
+            name: "OGG audio",
+            extension: "ogg",
+            mime: "audio/ogg",
+            description: "",
+            signature: {
+                0: 0x4f,
+                1: 0x67,
+                2: 0x67,
+                3: 0x53
+            },
+            extractor: null
+        },
+        {
+            name: "Musical Instrument Digital Interface audio",
+            extension: "midi",
+            mime: "audio/midi",
+            description: "",
+            signature: {
+                0: 0x4d,
+                1: 0x54,
+                2: 0x68,
+                3: 0x64
+            },
+            extractor: null
+        },
+        {
+            name: "MPEG-3 audio",
+            extension: "mp3",
+            mime: "audio/mpeg",
+            description: "",
+            signature: [
+                {
+                    0: 0x49,
+                    1: 0x44,
+                    2: 0x33
+                },
+                {
+                    0: 0xff,
+                    1: 0xfb
+                }
+            ],
+            extractor: extractMP3
+        },
+        {
+            name: "MPEG-4 Part 14 audio",
+            extension: "m4a",
+            mime: "audio/m4a",
+            description: "",
+            signature: [
+                {
+                    4: 0x66,
+                    5: 0x74,
+                    6: 0x79,
+                    7: 0x70,
+                    8: 0x4d,
+                    9: 0x34,
+                    10: 0x41
+                },
+                {
+                    0: 0x4d,
+                    1: 0x34,
+                    2: 0x41,
+                    3: 0x20
+                }
+            ],
+            extractor: null
+        },
+        {
+            name: "Free Lossless Audio Codec",
+            extension: "flac",
+            mime: "audio/x-flac",
+            description: "",
+            signature: {
+                0: 0x66,
+                1: 0x4c,
+                2: 0x61,
+                3: 0x43
+            },
+            extractor: null
+        },
+        {
+            name: "Adaptive Multi-Rate audio codec",
+            extension: "amr",
+            mime: "audio/amr",
+            description: "",
+            signature: {
+                0: 0x23,
+                1: 0x21,
+                2: 0x41,
+                3: 0x4d,
+                4: 0x52,
+                5: 0x0a
+            },
+            extractor: null
+        },
+        {
+            name: "Audacity",
+            extension: "au",
+            mime: "audio/x-au",
+            description: "",
+            signature: {
+                0: 0x64, // dns.
+                1: 0x6e,
+                2: 0x73,
+                3: 0x2e,
+
+                24: 0x41, // AudacityBlockFile
+                25: 0x75,
+                26: 0x64,
+                27: 0x61,
+                28: 0x63,
+                29: 0x69,
+                30: 0x74,
+                31: 0x79,
+                32: 0x42,
+                33: 0x6c,
+                34: 0x6f,
+                35: 0x63,
+                36: 0x6b,
+                37: 0x46,
+                38: 0x69,
+                39: 0x6c,
+                40: 0x65
+            },
+            extractor: null
+        },
+        {
+            name: "Audacity Block",
+            extension: "auf",
+            mime: "application/octet-stream",
+            description: "",
+            signature: {
+                0: 0x41, // AudacityBlockFile
+                1: 0x75,
+                2: 0x64,
+                3: 0x61,
+                4: 0x63,
+                5: 0x69,
+                6: 0x74,
+                7: 0x79,
+                8: 0x42,
+                9: 0x6c,
+                10: 0x6f,
+                11: 0x63,
+                12: 0x6b,
+                13: 0x46,
+                14: 0x69,
+                15: 0x6c,
+                16: 0x65
+            },
+            extractor: null
+        },
+        {
+            name: "Audio Interchange File",
+            extension: "aif",
+            mime: "audio/x-aiff",
+            description: "",
+            signature: {
+                0: 0x46, // FORM
+                1: 0x4f,
+                2: 0x52,
+                3: 0x4d,
+                8: 0x41, // AIFF
+                9: 0x49,
+                10: 0x46,
+                11: 0x46
+            },
+            extractor: null
+        },
+        {
+            name: "Audio Interchange File (compressed)",
+            extension: "aifc",
+            mime: "audio/x-aifc",
+            description: "",
+            signature: {
+                0: 0x46, // FORM
+                1: 0x4f,
+                2: 0x52,
+                3: 0x4d,
+                8: 0x41, // AIFC
+                9: 0x49,
+                10: 0x46,
+                11: 0x43
+            },
+            extractor: null
+        }
+    ],
+    "Documents": [
+        {
+            name: "Portable Document Format",
+            extension: "pdf",
+            mime: "application/pdf",
+            description: "",
+            signature: {
+                0: 0x25,
+                1: 0x50,
+                2: 0x44,
+                3: 0x46
+            },
+            extractor: extractPDF
+        },
+        {
+            name: "Portable Document Format (under Base64)",
+            extension: "B64",
+            mime: "application/octet-stream",
+            description: "",
+            signature: {
+                0: 0x41,
+                1: 0x4a,
+                2: 0x56,
+                3: 0x42,
+                4: 0x45,
+                5: 0x52,
+                6: 0x69
+            },
+            extractor: null
+        },
+        { // Place before PostScript
+            name: "Adobe PostScript",
+            extension: "ps,eps,ai,pfa",
+            mime: "application/postscript",
+            description: "",
+            signature: {
+                0: 0x25,
+                1: 0x21,
+                2: 0x50,
+                3: 0x53,
+                4: 0x2d,
+                5: 0x41,
+                6: 0x64,
+                7: 0x6f,
+                8: 0x62,
+                9: 0x65
+            },
+            extractor: null
+        },
+        {
+            name: "PostScript",
+            extension: "ps",
+            mime: "application/postscript",
+            description: "",
+            signature: {
+                0: 0x25,
+                1: 0x21
+            },
+            extractor: null
+        },
+        {
+            name: "Encapsulated PostScript",
+            extension: "eps,ai",
+            mime: "application/eps",
+            description: "",
+            signature: {
+                0: 0xc5,
+                1: 0xd0,
+                2: 0xd3,
+                3: 0xc6
+            },
+            extractor: null
+        },
+        {
+            name: "Rich Text Format",
+            extension: "rtf",
+            mime: "application/rtf",
+            description: "",
+            signature: {
+                0: 0x7b,
+                1: 0x5c,
+                2: 0x72,
+                3: 0x74
+            },
+            extractor: extractRTF
+        },
+        {
+            name: "Microsoft Office document/OLE2",
+            extension: "ole2,doc,xls,dot,ppt,xla,ppa,pps,pot,msi,sdw,db,vsd,msg",
+            mime: "application/msword,application/vnd.ms-excel,application/vnd.ms-powerpoint",
+            description: "Microsoft Office documents",
+            signature: {
+                0: 0xd0,
+                1: 0xcf,
+                2: 0x11,
+                3: 0xe0,
+                4: 0xa1,
+                5: 0xb1,
+                6: 0x1a,
+                7: 0xe1
+            },
+            extractor: null
+        },
+        {
+            name: "Microsoft Office document/OLE2 (under Base64)",
+            extension: "B64",
+            mime: "application/octet-stream",
+            description: "",
+            signature: {
+                0: 0x30,
+                1: 0x4d,
+                2: 0x38,
+                3: 0x52,
+                4: 0x34,
+                5: 0x4b,
+                6: 0x47,
+                7: 0x78
+            },
+            extractor: null
+        },
+        {
+            name: "Microsoft Office 2007+ document",
+            extension: "docx,xlsx,pptx",
+            mime: "application/vnd.openxmlformats-officedocument.wordprocessingml.document,application/vnd.openxmlformats-officedocument.spreadsheetml.sheet,application/vnd.openxmlformats-officedocument.presentationml.presentation",
+            description: "",
+            signature: {
+                38: 0x5f, // _Types].xml
+                39: 0x54,
+                40: 0x79,
+                41: 0x70,
+                42: 0x65,
+                43: 0x73,
+                44: 0x5d,
+                45: 0x2e,
+                46: 0x78,
+                47: 0x6d,
+                48: 0x6c
+            },
+            extractor: extractZIP
+        },
+        {
+            name: "Microsoft Access database",
+            extension: "mdb,mda,mde,mdt,fdb,psa",
+            mime: "application/msaccess",
+            description: "",
+            signature: {
+                0: 0x00,
+                1: 0x01,
+                2: 0x00,
+                3: 0x00,
+                4: 0x53, // Standard Jet
+                5: 0x74,
+                6: 0x61,
+                7: 0x6e,
+                8: 0x64,
+                9: 0x61,
+                10: 0x72,
+                11: 0x64,
+                12: 0x20,
+                13: 0x4a,
+                14: 0x65,
+                15: 0x74
+            },
+            extractor: null
+        },
+        {
+            name: "Microsoft Access 2007+ database",
+            extension: "accdb,accde,accda,accdu",
+            mime: "application/msaccess",
+            description: "",
+            signature: {
+                0: 0x00,
+                1: 0x01,
+                2: 0x00,
+                3: 0x00,
+                4: 0x53, // Standard ACE DB
+                5: 0x74,
+                6: 0x61,
+                7: 0x6e,
+                8: 0x64,
+                9: 0x61,
+                10: 0x72,
+                11: 0x64,
+                12: 0x20,
+                13: 0x41,
+                14: 0x43,
+                15: 0x45,
+                16: 0x20
+            },
+            extractor: null
+        },
+        {
+            name: "Microsoft OneNote document",
+            extension: "one",
+            mime: "application/onenote",
+            description: "",
+            signature: {
+                0: 0xe4,
+                1: 0x52,
+                2: 0x5c,
+                3: 0x7b,
+                4: 0x8c,
+                5: 0xd8,
+                6: 0xa7,
+                7: 0x4d,
+                8: 0xae,
+                9: 0xb1,
+                10: 0x53,
+                11: 0x78,
+                12: 0xd0,
+                13: 0x29,
+                14: 0x96,
+                15: 0xd3
+            },
+            extractor: null
+        },
+        {
+            name: "Outlook Express database",
+            extension: "dbx",
+            mime: "application/octet-stream",
+            description: "",
+            signature: {
+                0: 0xcf,
+                1: 0xad,
+                2: 0x12,
+                3: 0xfe,
+                4: [0x30, 0xc5, 0xc6, 0xc7],
+                11: 0x11
+            },
+            extractor: null
+        },
+        {
+            name: "Personal Storage Table (Outlook)",
+            extension: "pst,ost,fdb,pab",
+            mime: "application/octet-stream",
+            description: "",
+            signature: {
+                0: 0x21, // !BDN
+                1: 0x42,
+                2: 0x44,
+                3: 0x4e
+            },
+            extractor: null
+        },
+        {
+            name: "Microsoft Exchange Database",
+            extension: "edb",
+            mime: "application/octet-stream",
+            description: "",
+            signature: {
+                4: 0xef,
+                5: 0xcd,
+                6: 0xab,
+                7: 0x89,
+                8: [0x20, 0x23],
+                9: 0x06,
+                10: 0x00,
+                11: 0x00,
+                12: [0x00, 0x01],
+                13: 0x00,
+                14: 0x00,
+                15: 0x00
+            },
+            extractor: null
+        },
+        {
+            name: "WordPerfect document",
+            extension: "wpd,wp,wp5,wp6,wpp,bk!,wcm",
+            mime: "application/wordperfect",
+            description: "",
+            signature: {
+                0: 0xff,
+                1: 0x57,
+                2: 0x50,
+                3: 0x43,
+                7: [0x00, 0x01, 0x02],
+                8: 0x01,
+                9: 0x0a
+            },
+            extractor: null
+        },
+        {
+            name: "EPUB e-book",
+            extension: "epub",
+            mime: "application/epub+zip",
+            description: "",
+            signature: {
+                0: 0x50,
+                1: 0x4b,
+                2: 0x3,
+                3: 0x4,
+                30: 0x6d, // mimetypeapplication/epub_zip
+                31: 0x69,
+                32: 0x6d,
+                33: 0x65,
+                34: 0x74,
+                35: 0x79,
+                36: 0x70,
+                37: 0x65,
+                38: 0x61,
+                39: 0x70,
+                40: 0x70,
+                41: 0x6c,
+                42: 0x69,
+                43: 0x63,
+                44: 0x61,
+                45: 0x74,
+                46: 0x69,
+                47: 0x6f,
+                48: 0x6e,
+                49: 0x2f,
+                50: 0x65,
+                51: 0x70,
+                52: 0x75,
+                53: 0x62,
+                54: 0x2b,
+                55: 0x7a,
+                56: 0x69,
+                57: 0x70
+            },
+            extractor: extractZIP
+        },
+    ],
+    "Applications": [
+        {
+            name: "Windows Portable Executable",
+            extension: "exe,dll,drv,vxd,sys,ocx,vbx,com,fon,scr",
+            mime: "application/vnd.microsoft.portable-executable",
+            description: "",
+            signature: {
+                0: 0x4d,
+                1: 0x5a,
+                3: [0x0, 0x1, 0x2],
+                5: [0x0, 0x1, 0x2]
+            },
+            extractor: extractMZPE
+        },
+        {
+            name: "Executable and Linkable Format",
+            extension: "elf,bin,axf,o,prx,so",
+            mime: "application/x-executable",
+            description: "Executable and Linkable Format file. No standard file extension.",
+            signature: {
+                0: 0x7f,
+                1: 0x45,
+                2: 0x4c,
+                3: 0x46
+            },
+            extractor: extractELF
+        },
+        {
+            name: "MacOS Mach-O object",
+            extension: "dylib",
+            mime: "application/octet-stream",
+            description: "",
+            signature: [
+                {
+                    0: 0xca,
+                    1: 0xfe,
+                    2: 0xba,
+                    3: 0xbe,
+                    4: 0x00,
+                    5: 0x00,
+                    6: 0x00,
+                    7: [0x01, 0x02, 0x03]
+                },
+                {
+                    0: 0xce,
+                    1: 0xfa,
+                    2: 0xed,
+                    3: 0xfe,
+                    4: 0x07,
+                    5: 0x00,
+                    6: 0x00,
+                    7: 0x00,
+                    8: [0x01, 0x02, 0x03]
+                }
+            ],
+            extractor: extractMACHO
+        },
+        {
+            name: "MacOS Mach-O 64-bit object",
+            extension: "dylib",
+            mime: "application/octet-stream",
+            description: "",
+            signature: {
+                0: 0xcf,
+                1: 0xfa,
+                2: 0xed,
+                3: 0xfe
+            },
+            extractor: extractMACHO
+        },
+        {
+            name: "Adobe Flash",
+            extension: "swf",
+            mime: "application/x-shockwave-flash",
+            description: "",
+            signature: {
+                0: [0x43, 0x46],
+                1: 0x57,
+                2: 0x53,
+            },
+            extractor: null
+        },
+        {
+            name: "Java Class",
+            extension: "class",
+            mime: "application/java-vm",
+            description: "",
+            signature: {
+                0: 0xca,
+                1: 0xfe,
+                2: 0xba,
+                3: 0xbe
+            },
+            extractor: null
+        },
+        {
+            name: "Dalvik Executable",
+            extension: "dex",
+            mime: "application/octet-stream",
+            description: "Dalvik Executable as used by Android",
+            signature: {
+                0: 0x64,
+                1: 0x65,
+                2: 0x78,
+                3: 0x0a,
+                4: 0x30,
+                5: 0x33,
+                6: 0x35,
+                7: 0x0
+            },
+            extractor: null
+        },
+        {
+            name: "Google Chrome Extension",
+            extension: "crx",
+            mime: "application/crx",
+            description: "Google Chrome extension or packaged app",
+            signature: {
+                0: 0x43,
+                1: 0x72,
+                2: 0x32,
+                3: 0x34
+            },
+            extractor: null
+        },
+    ],
+    "Archives": [
+        {
+            name: "PKZIP archive",
+            extension: "zip",
+            mime: "application/zip",
+            description: "",
+            signature: {
+                0: 0x50,
+                1: 0x4b,
+                2: [0x3, 0x5, 0x7],
+                3: [0x4, 0x6, 0x8]
+            },
+            extractor: extractZIP
+        },
+        {
+            name: "PKZIP archive (under Base64)",
+            extension: "B64",
+            mime: "application/octet-stream",
+            description: "",
+            signature: {
+                0: 0x55,
+                1: 0x45,
+                2: 0x73,
+                3: 0x44,
+                4: 0x42,
+                5: 0x42
+            },
+            extractor: null
+        },
+        {
+            name: "TAR archive",
+            extension: "tar",
+            mime: "application/x-tar",
+            description: "",
+            signature: {
+                257: 0x75, // ustar
+                258: 0x73,
+                259: 0x74,
+                260: 0x61,
+                261: 0x72
+            },
+            extractor: extractTAR
+        },
+        {
+            name: "Roshal Archive",
+            extension: "rar",
+            mime: "application/x-rar-compressed",
+            description: "",
+            signature: {
+                0: 0x52,
+                1: 0x61,
+                2: 0x72,
+                3: 0x21,
+                4: 0x1a,
+                5: 0x7,
+                6: [0x0, 0x1]
+            },
+            extractor: null
+        },
+        {
+            name: "Gzip",
+            extension: "gz",
+            mime: "application/gzip",
+            description: "",
+            signature: {
+                0: 0x1f,
+                1: 0x8b,
+                2: 0x8
+            },
+            extractor: extractGZIP
+        },
+        {
+            name: "Bzip2",
+            extension: "bz2",
+            mime: "application/x-bzip2",
+            description: "",
+            signature: {
+                0: 0x42,
+                1: 0x5a,
+                2: 0x68
+            },
+            extractor: extractBZIP2
+        },
+        {
+            name: "7zip",
+            extension: "7z",
+            mime: "application/x-7z-compressed",
+            description: "",
+            signature: {
+                0: 0x37,
+                1: 0x7a,
+                2: 0xbc,
+                3: 0xaf,
+                4: 0x27,
+                5: 0x1c
+            },
+            extractor: null
+        },
+        {
+            name: "Zlib Deflate",
+            extension: "zlib",
+            mime: "application/x-deflate",
+            description: "",
+            signature: {
+                0: 0x78,
+                1: [0x1, 0x9c, 0xda, 0x5e]
+            },
+            extractor: extractZlib
+        },
+        {
+            name: "xz compression",
+            extension: "xz",
+            mime: "application/x-xz",
+            description: "",
+            signature: {
+                0: 0xfd,
+                1: 0x37,
+                2: 0x7a,
+                3: 0x58,
+                4: 0x5a,
+                5: 0x0
+            },
+            extractor: extractXZ
+        },
+        {
+            name: "Tarball",
+            extension: "tar.z",
+            mime: "application/x-gtar",
+            description: "",
+            signature: {
+                0: 0x1f,
+                1: [0x9d, 0xa0]
+            },
+            extractor: null
+        },
+        {
+            name: "ISO disk image",
+            extension: "iso",
+            mime: "application/octet-stream",
+            description: "ISO 9660 CD/DVD image file",
+            signature: [
+                {
+                    0x8001: 0x43,
+                    0x8002: 0x44,
+                    0x8003: 0x30,
+                    0x8004: 0x30,
+                    0x8005: 0x31
+                },
+                {
+                    0x8801: 0x43,
+                    0x8802: 0x44,
+                    0x8803: 0x30,
+                    0x8804: 0x30,
+                    0x8805: 0x31
+                },
+                {
+                    0x9001: 0x43,
+                    0x9002: 0x44,
+                    0x9003: 0x30,
+                    0x9004: 0x30,
+                    0x9005: 0x31
+                }
+            ],
+            extractor: null
+        },
+        {
+            name: "Virtual Machine Disk",
+            extension: "vmdk",
+            mime: "application/vmdk,application/x-virtualbox-vmdk",
+            description: "",
+            signature: {
+                0: 0x4b,
+                1: 0x44,
+                2: 0x4d,
+                3: 0x56,
+                5: 0x00,
+                6: 0x00,
+                7: 0x00
+            },
+            extractor: null
+        },
+        {
+            name: "Virtual Hard Drive",
+            extension: "vhd",
+            mime: "application/x-vhd",
+            description: "",
+            signature: {
+                0: 0x63, // conectix
+                1: 0x6f,
+                2: 0x6e,
+                3: 0x65,
+                4: 0x63,
+                5: 0x74,
+                6: 0x69,
+                7: 0x78
+            },
+            extractor: null
+        },
+        {
+            name: "Macintosh disk image",
+            extension: "dmf,dmg",
+            mime: "application/octet-stream",
+            description: "",
+            signature: {
+                0: 0x78,
+                1: 0x01,
+                2: 0x73,
+                3: 0x0d,
+                4: 0x62,
+                5: 0x62,
+                6: 0x60,
+                7: 0x60
+            },
+            extractor: null
+        },
+        {
+            name: "ARJ Archive",
+            extension: "arj",
+            mime: "application/x-arj-compressed",
+            description: "",
+            signature: {
+                0: 0x60,
+                1: 0xea,
+                8: [0x0, 0x10, 0x14],
+                9: 0x0,
+                10: 0x2
+            },
+            extractor: null
+        },
+        {
+            name: "WinAce Archive",
+            extension: "ace",
+            mime: "application/x-ace-compressed",
+            description: "",
+            signature: {
+                7: 0x2a, // **ACE**
+                8: 0x2a,
+                9: 0x41,
+                10: 0x43,
+                11: 0x45,
+                12: 0x2a,
+                13: 0x2a
+            },
+            extractor: null
+        },
+        {
+            name: "Macintosh BinHex Encoded File",
+            extension: "hqx",
+            mime: "application/mac-binhex",
+            description: "",
+            signature: {
+                11: 0x6d,  // must be converted with BinHex
+                12: 0x75,
+                13: 0x73,
+                14: 0x74,
+                15: 0x20,
+                16: 0x62,
+                17: 0x65,
+                18: 0x20,
+                19: 0x63,
+                20: 0x6f,
+                21: 0x6e,
+                22: 0x76,
+                23: 0x65,
+                24: 0x72,
+                25: 0x74,
+                26: 0x65,
+                27: 0x64,
+                28: 0x20,
+                29: 0x77,
+                30: 0x69,
+                31: 0x74,
+                32: 0x68,
+                33: 0x20,
+                34: 0x42,
+                35: 0x69,
+                36: 0x6e,
+                37: 0x48,
+                38: 0x65,
+                39: 0x78
+            },
+            extractor: null
+        },
+        {
+            name: "ALZip Archive",
+            extension: "alz",
+            mime: "application/octet-stream",
+            description: "",
+            signature: {
+                0: 0x41, // ALZ
+                1: 0x4c,
+                2: 0x5a,
+                3: 0x01,
+                4: 0x0a,
+                5: 0x0,
+                6: 0x0,
+                7: 0x0
+            },
+            extractor: null
+        },
+        {
+            name: "KGB Compressed Archive",
+            extension: "kgb",
+            mime: "application/x-kgb-compressed",
+            description: "",
+            signature: {
+                0: 0x4b, // KGB_arch -
+                1: 0x47,
+                2: 0x42,
+                3: 0x5f,
+                4: 0x61,
+                5: 0x72,
+                6: 0x63,
+                7: 0x68,
+                8: 0x20,
+                9: 0x2d
+            },
+            extractor: null
+        },
+        {
+            name: "Microsoft Cabinet",
+            extension: "cab",
+            mime: "vnd.ms-cab-compressed",
+            description: "",
+            signature: {
+                0: 0x4d,
+                1: 0x53,
+                2: 0x43,
+                3: 0x46,
+                4: 0x00,
+                5: 0x00,
+                6: 0x00,
+                7: 0x00
+            },
+            extractor: null
+        },
+        {
+            name: "Jar Archive",
+            extension: "jar",
+            mime: "application/java-archive",
+            description: "",
+            signature: {
+                0: 0x5f,
+                1: 0x27,
+                2: 0xa8,
+                3: 0x89
+            },
+            extractor: null
+        },
+        {
+            name: "Jar Archive",
+            extension: "jar",
+            mime: "application/java-archive",
+            description: "",
+            signature: {
+                0: 0x50,
+                1: 0x4B,
+                2: 0x03,
+                3: 0x04,
+                4: 0x14,
+                5: 0x00,
+                6: 0x08,
+                7: 0x00,
+                8: 0x08,
+                9: 0x00
+            },
+            extractor: extractZIP
+        },
+        {
+            name: "lzop compressed",
+            extension: "lzop,lzo",
+            mime: "application/x-lzop",
+            description: "",
+            signature: {
+                0: 0x89,
+                1: 0x4c, // LZO
+                2: 0x5a,
+                3: 0x4f,
+                4: 0x00,
+                5: 0x0d,
+                6: 0x0a,
+                7: 0x1a
+            },
+            extractor: extractLZOP
+        },
+        {
+            name: "Linux deb package",
+            extension: "deb",
+            mime: "application/vnd.debian.binary-package",
+            description: "",
+            signature: {
+                0: 0x21,
+                1: 0x3C,
+                2: 0x61,
+                3: 0x72,
+                4: 0x63,
+                5: 0x68,
+                6: 0x3e
+            },
+            extractor: extractDEB
+        },
+        {
+            name: "Apple Disk Image",
+            extension: "dmg",
+            mime: "application/x-apple-diskimage",
+            description: "",
+            signature: {
+                0: 0x78,
+                1: 0x01,
+                2: 0x73,
+                3: 0x0d,
+                4: 0x62,
+                5: 0x62,
+                6: 0x60
+            },
+            extractor: null
+        }
+    ],
+    "Miscellaneous": [
+        {
+            name: "UTF-8 text",
+            extension: "txt",
+            mime: "text/plain",
+            description: "UTF-8 encoded Unicode byte order mark, commonly but not exclusively seen in text files.",
+            signature: {
+                0: 0xef,
+                1: 0xbb,
+                2: 0xbf
+            },
+            extractor: null
+        },
+        { // Place before UTF-16 LE text
+            name: "UTF-32 LE text",
+            extension: "utf32le",
+            mime: "charset/utf32le",
+            description: "Little-endian UTF-32 encoded Unicode byte order mark.",
+            signature: {
+                0: 0xff,
+                1: 0xfe,
+                2: 0x00,
+                3: 0x00
+            },
+            extractor: null
+        },
+        {
+            name: "UTF-16 LE text",
+            extension: "utf16le",
+            mime: "charset/utf16le",
+            description: "Little-endian UTF-16 encoded Unicode byte order mark.",
+            signature: {
+                0: 0xff,
+                1: 0xfe
+            },
+            extractor: null
+        },
+        {
+            name: "Web Open Font Format",
+            extension: "woff",
+            mime: "application/font-woff",
+            description: "",
+            signature: {
+                0: 0x77,
+                1: 0x4f,
+                2: 0x46,
+                3: 0x46,
+                4: 0x0,
+                5: 0x1,
+                6: 0x0,
+                7: 0x0
+            },
+            extractor: null
+        },
+        {
+            name: "Web Open Font Format 2",
+            extension: "woff2",
+            mime: "application/font-woff",
+            description: "",
+            signature: {
+                0: 0x77,
+                1: 0x4f,
+                2: 0x46,
+                3: 0x32,
+                4: 0x0,
+                5: 0x1,
+                6: 0x0,
+                7: 0x0
+            },
+            extractor: null
+        },
+        {
+            name: "Embedded OpenType font",
+            extension: "eot",
+            mime: "application/octet-stream",
+            description: "",
+            signature: [
+                {
+                    8: 0x2,
+                    9: 0x0,
+                    10: 0x1,
+                    34: 0x4c,
+                    35: 0x50
+                },
+                {
+                    8: 0x1,
+                    9: 0x0,
+                    10: 0x0,
+                    34: 0x4c,
+                    35: 0x50
+                },
+                {
+                    8: 0x2,
+                    9: 0x0,
+                    10: 0x2,
+                    34: 0x4c,
+                    35: 0x50
+                },
+            ],
+            extractor: null
+        },
+        {
+            name: "TrueType Font",
+            extension: "ttf",
+            mime: "application/font-sfnt",
+            description: "",
+            signature: {
+                0: 0x0,
+                1: 0x1,
+                2: 0x0,
+                3: 0x0,
+                4: 0x0
+            },
+            extractor: null
+        },
+        {
+            name: "OpenType Font",
+            extension: "otf",
+            mime: "application/font-sfnt",
+            description: "",
+            signature: {
+                0: 0x4f,
+                1: 0x54,
+                2: 0x54,
+                3: 0x4f,
+                4: 0x0
+            },
+            extractor: null
+        },
+        {
+            name: "SQLite",
+            extension: "sqlite",
+            mime: "application/x-sqlite3",
+            description: "",
+            signature: {
+                0: 0x53,
+                1: 0x51,
+                2: 0x4c,
+                3: 0x69
+            },
+            extractor: extractSQLITE
+        },
+        {
+            name: "BitTorrent link",
+            extension: "torrent",
+            mime: "application/x-bittorrent",
+            description: "",
+            signature: [
+                {
+                    0: 0x64, // d8:announce##:
+                    1: 0x38,
+                    2: 0x3a,
+                    3: 0x61,
+                    4: 0x6e,
+                    5: 0x6e,
+                    6: 0x6f,
+                    7: 0x75,
+                    8: 0x6e,
+                    9: 0x63,
+                    10: 0x65,
+                    11: 0x23,
+                    12: 0x23,
+                    13: 0x3a
+                },
+                {
+                    0: 0x64, // d4:infod
+                    1: 0x34,
+                    2: 0x3a,
+                    3: 0x69,
+                    4: 0x6e,
+                    5: 0x66,
+                    6: 0x6f,
+                    7: 0x64,
+                    8: [0x34, 0x35, 0x36],
+                    9: 0x3a
+                }
+            ],
+            extractor: null
+        },
+        {
+            name: "Cryptocurrency wallet",
+            extension: "wallet",
+            mime: "application/octet-stream",
+            description: "",
+            signature: {
+                0: 0x00,
+                1: 0x00,
+                2: 0x00,
+                3: 0x00,
+                4: 0x01,
+                5: 0x00,
+                6: 0x00,
+                7: 0x00,
+                8: 0x00,
+                9: 0x00,
+                10: 0x00,
+                11: 0x00,
+                12: 0x62,
+                13: 0x31,
+                14: 0x05,
+                15: 0x00
+            },
+            extractor: null
+        },
+        {
+            name: "Registry fragment",
+            extension: "hbin",
+            mime: "application/octet-stream",
+            description: "",
+            signature: {
+                0: 0x68, // hbin
+                1: 0x62,
+                2: 0x69,
+                3: 0x6e,
+                4: 0x00
+            },
+            extractor: null
+        },
+        {
+            name: "Registry script",
+            extension: "rgs",
+            mime: "application/octet-stream",
+            description: "",
+            signature: {
+                0: 0x48, // HKCR
+                1: 0x4b,
+                2: 0x43,
+                3: 0x52,
+                4: 0x0d,
+                5: 0x0a,
+                6: 0x5c,
+                7: 0x7b
+            },
+            extractor: null
+        },
+        {
+            name: "WinNT Registry Hive",
+            extension: "registry",
+            mime: "application/octet-stream",
+            description: "",
+            signature: {
+                0: 0x72,
+                1: 0x65,
+                2: 0x67,
+                3: 0x66
+            },
+            extractor: null
+        },
+        {
+            name: "Windows Event Log",
+            extension: "evt",
+            mime: "application/octet-stream",
+            description: "",
+            signature: {
+                0: 0x30,
+                1: 0x00,
+                2: 0x00,
+                3: 0x00,
+                4: 0x4c,
+                5: 0x66,
+                6: 0x4c,
+                7: 0x65
+            },
+            extractor: extractEVT
+        },
+        {
+            name: "Windows Event Log",
+            extension: "evtx",
+            mime: "application/octet-stream",
+            description: "",
+            signature: {
+                0: 0x45, // ElfFile
+                1: 0x6c,
+                2: 0x66,
+                3: 0x46,
+                4: 0x69,
+                5: 0x6c,
+                6: 0x65
+            },
+            extractor: extractEVTX
+        },
+        {
+            name: "Windows Pagedump",
+            extension: "dmp",
+            mime: "application/octet-stream",
+            description: "",
+            signature: {
+                0: 0x50, // PAGEDU(MP|64)
+                1: 0x41,
+                2: 0x47,
+                3: 0x45,
+                4: 0x44,
+                5: 0x55,
+                6: [0x4d, 0x36],
+                7: [0x50, 0x34]
+            },
+            extractor: extractDMP
+        },
+        {
+            name: "Windows Prefetch",
+            extension: "pf",
+            mime: "application/x-pf",
+            description: "",
+            signature: {
+                0: [0x11, 0x17, 0x1a],
+                1: 0x0,
+                2: 0x0,
+                3: 0x0,
+                4: 0x53,
+                5: 0x43,
+                6: 0x43,
+                7: 0x41
+            },
+            extractor: extractPF
+        },
+        {
+            name: "Windows Prefetch (Win 10)",
+            extension: "pf",
+            mime: "application/x-pf",
+            description: "",
+            signature: {
+                0: 0x4d,
+                1: 0x41,
+                2: 0x4d,
+                3: 0x04,
+                7: 0x0
+            },
+            extractor: extractPFWin10
+        },
+        {
+            name: "PList (XML)",
+            extension: "plist",
+            mime: "application/xml",
+            description: "",
+            signature: {
+                39: 0x3c, // <!DOCTYPE plist
+                40: 0x21,
+                41: 0x44,
+                42: 0x4f,
+                43: 0x43,
+                44: 0x54,
+                45: 0x59,
+                46: 0x50,
+                47: 0x45,
+                48: 0x20,
+                49: 0x70,
+                50: 0x6c,
+                51: 0x69,
+                52: 0x73,
+                53: 0x74
+            },
+            extractor: extractPListXML
+        },
+        {
+            name: "PList (binary)",
+            extension: "bplist,plist,ipmeta,abcdp,mdbackup,mdinfo,strings,nib,ichat,qtz,webbookmark,webhistory",
+            mime: "application/x-plist",
+            description: "",
+            signature: {
+                0: 0x62, // bplist00
+                1: 0x70,
+                2: 0x6c,
+                3: 0x69,
+                4: 0x73,
+                5: 0x74,
+                6: 0x30,
+                7: 0x30
+            },
+            extractor: null
+        },
+        {
+            name: "MacOS X Keychain",
+            extension: "keychain",
+            mime: "application/octet-stream",
+            description: "",
+            signature: {
+                0: 0x6b, // kych
+                1: 0x79,
+                2: 0x63,
+                3: 0x68,
+                4: 0x00,
+                5: 0x01
+            },
+            extractor: extractMacOSXKeychain
+        },
+        {
+            name: "TCP Packet",
+            extension: "tcp",
+            mime: "application/tcp",
+            description: "",
+            signature: {
+                12: 0x08,
+                13: 0x00,
+                14: 0x45,
+                15: 0x00,
+                21: 0x00,
+                22: b => b >= 0x01 && b <= 0x80,
+                23: 0x06
+            },
+            extractor: null
+        },
+        {
+            name: "UDP Packet",
+            extension: "udp",
+            mime: "application/udp",
+            description: "",
+            signature: {
+                12: 0x08,
+                13: 0x00,
+                14: 0x45,
+                15: 0x00,
+                16: [0x00, 0x01, 0x02, 0x03, 0x04, 0x05],
+                22: b => b >= 0x01 && b <= 0x80,
+                23: 0x11
+            },
+            extractor: null
+        },
+        {
+            name: "Compiled HTML",
+            extension: "chm,chw,chi",
+            mime: "application/vnd.ms-htmlhelp",
+            description: "",
+            signature: {
+                0: 0x49, // ITSF
+                1: 0x54,
+                2: 0x53,
+                3: 0x46,
+                4: 0x03,
+                5: 0x00,
+                6: 0x00,
+                7: 0x00
+            },
+            extractor: null
+        },
+        {
+            name: "Windows Password",
+            extension: "pwl",
+            mime: "application/octet-stream",
+            description: "",
+            signature: {
+                0: 0xe3,
+                1: 0x82,
+                2: 0x85,
+                3: 0x96
+            },
+            extractor: null
+        },
+        {
+            name: "Bitlocker recovery key",
+            extension: "bitlocker",
+            mime: "application/octet-stream",
+            description: "",
+            signature: {
+                0: 0xff,
+                1: 0xfe,
+                2: 0x42,
+                3: 0x00,
+                4: 0x69,
+                5: 0x00,
+                6: 0x74,
+                7: 0x00,
+                8: 0x4c,
+                9: 0x00,
+                10: 0x6f,
+                11: 0x00,
+                12: 0x63,
+                13: 0x00,
+                14: 0x6b,
+                15: 0x00,
+                16: 0x65,
+                17: 0x00,
+                18: 0x72,
+                19: 0x00,
+                20: 0x20,
+                21: 0x00
+            },
+            extractor: null
+        },
+        {
+            name: "Certificate",
+            extension: "cer,cat,p7b,p7c,p7m,p7s,swz,rsa,crl,crt,der",
+            mime: "application/pkix-cert",
+            description: "",
+            signature: {
+                0: 0x30,
+                1: 0x82,
+                4: [0x06, 0x0a, 0x30]
+            },
+            extractor: null
+        },
+        {
+            name: "Certificate",
+            extension: "cat,swz,p7m",
+            mime: "application/vnd.ms-pki.seccat",
+            description: "",
+            signature: {
+                0: 0x30,
+                1: 0x83,
+                2: b => b !== 0x00,
+                5: 0x06,
+                6: 0x09
+            },
+            extractor: null
+        },
+        {
+            name: "PGP pubring",
+            extension: "pkr,gpg",
+            mime: "application/pgp-keys",
+            description: "",
+            signature: {
+                0: 0x99,
+                1: 0x01,
+                2: [0x0d, 0xa2],
+                3: 0x04
+            },
+            extractor: null
+        },
+        {
+            name: "PGP secring",
+            extension: "skr",
+            mime: "application/pgp-keys",
+            description: "",
+            signature: [
+                {
+                    0: 0x95,
+                    1: 0x01,
+                    2: 0xcf,
+                    3: 0x04
+                },
+                {
+                    0: 0x95,
+                    1: 0x03,
+                    2: 0xc6,
+                    3: 0x04
+                },
+                {
+                    0: 0x95,
+                    1: 0x05,
+                    2: 0x86,
+                    3: 0x04
+                }
+            ],
+            extractor: null
+        },
+        {
+            name: "PGP Safe",
+            extension: "pgd",
+            mime: "application/pgp-keys",
+            description: "",
+            signature: {
+                0: 0x50, // PGPdMAIN
+                1: 0x47,
+                2: 0x50,
+                3: 0x64,
+                4: 0x4d,
+                5: 0x41,
+                6: 0x49,
+                7: 0x4e,
+                8: 0x60,
+                9: 0x01,
+                10: 0x00
+            },
+            extractor: null
+        },
+        {
+            name: "Task Scheduler",
+            extension: "job",
+            mime: "application/octet-stream",
+            description: "",
+            signature: {
+                0: [0x00, 0x01, 0x02, 0x03],
+                1: [0x05, 0x06],
+                2: 0x01,
+                3: 0x00,
+                20: 0x46,
+                21: 0x00
+            },
+            extractor: null
+        },
+        {
+            name: "Windows Shortcut",
+            extension: "lnk",
+            mime: "application/x-ms-shortcut",
+            description: "",
+            signature: {
+                0: 0x4c,
+                1: 0x00,
+                2: 0x00,
+                3: 0x00,
+                4: 0x01,
+                5: 0x14,
+                6: 0x02,
+                7: 0x00,
+                8: 0x00,
+                9: 0x00,
+                10: 0x00,
+                11: 0x00,
+                12: 0xc0,
+                13: 0x00,
+                14: 0x00,
+                15: 0x00,
+                16: 0x00,
+                17: 0x00,
+                18: 0x00,
+                19: 0x46
+            },
+            extractor: extractLNK
+        },
+        {
+            name: "Bash",
+            extension: "bash",
+            mime: "application/bash",
+            description: "",
+            signature: {
+                0: 0x23, // #!/bin/bash
+                1: 0x21,
+                2: 0x2f,
+                3: 0x62,
+                4: 0x69,
+                5: 0x6e,
+                6: 0x2f,
+                7: 0x62,
+                8: 0x61,
+                9: 0x73,
+                10: 0x68,
+            },
+            extractor: null
+        },
+        {
+            name: "Shell",
+            extension: "sh",
+            mime: "application/sh",
+            description: "",
+            signature: {
+                0: 0x23, // #!/bin/sh
+                1: 0x21,
+                2: 0x2f,
+                3: 0x62,
+                4: 0x69,
+                5: 0x6e,
+                6: 0x2f,
+                7: 0x73,
+                8: 0x68,
+            },
+            extractor: null
+        },
+        {
+            name: "Python",
+            extension: "py,pyc,pyd,pyo,pyw,pyz",
+            mime: "application/python",
+            description: "",
+            signature: {
+                0: 0x23, // #!/usr/bin/python(2|3)
+                1: 0x21,
+                2: 0x2f,
+                3: 0x75,
+                4: 0x73,
+                5: 0x72,
+                6: 0x2f,
+                7: 0x62,
+                8: 0x69,
+                9: 0x6e,
+                10: 0x2f,
+                11: 0x70,
+                12: 0x79,
+                13: 0x74,
+                14: 0x68,
+                15: 0x6f,
+                16: 0x6e,
+                17: [0x32, 0x33, 0xa, 0xd],
+            },
+            extractor: null
+        },
+        {
+            name: "Ruby",
+            extension: "rb",
+            mime: "application/ruby",
+            description: "",
+            signature: {
+                0: 0x23, // #!/usr/bin/ruby
+                1: 0x21,
+                2: 0x2f,
+                3: 0x75,
+                4: 0x73,
+                5: 0x72,
+                6: 0x2f,
+                7: 0x62,
+                8: 0x69,
+                9: 0x6e,
+                10: 0x2f,
+                11: 0x72,
+                12: 0x75,
+                13: 0x62,
+                14: 0x79,
+            },
+            extractor: null
+        },
+        {
+            name: "perl",
+            extension: "pl,pm,t,pod",
+            mime: "application/perl",
+            description: "",
+            signature: {
+                0: 0x23, // #!/usr/bin/perl
+                1: 0x21,
+                2: 0x2f,
+                3: 0x75,
+                4: 0x73,
+                5: 0x72,
+                6: 0x2f,
+                7: 0x62,
+                8: 0x69,
+                9: 0x6e,
+                10: 0x2f,
+                11: 0x70,
+                12: 0x65,
+                13: 0x72,
+                14: 0x6c,
+            },
+            extractor: null
+        },
+        {
+            name: "php",
+            extension: "php,phtml,php3,php4,php5,php7,phps,php-s,pht,phar",
+            mime: "application/php",
+            description: "",
+            signature: {
+                0: 0x3c, // <?php
+                1: 0x3f,
+                2: 0x70,
+                3: 0x68,
+                4: 0x70,
+            },
+            extractor: null
+        },
+        {
+            name: "Smile",
+            extension: "sml",
+            mime: "	application/x-jackson-smile",
+            description: "",
+            signature: {
+                0: 0x3a,
+                1: 0x29,
+                2: 0xa
+            },
+            extractor: null
+        },
+        {
+            name: "Lua Bytecode",
+            extension: "luac",
+            mime: "application/x-lua",
+            description: "",
+            signature: {
+                0: 0x1b,
+                1: 0x4c,
+                2: 0x75,
+                3: 0x61
+            },
+            extractor: null
+        },
+        {
+            name: "WebAssembly binary",
+            extension: "wasm",
+            mime: "application/octet-stream",
+            description: "",
+            signature: {
+                0: 0x00,
+                1: 0x61,
+                2: 0x73,
+                3: 0x6d
+            },
+            extractor: null
+        }
+    ]
+};
+
+
+/**
+ * JPEG extractor.
+ *
+ * @param {Uint8Array} bytes
+ * @param {number} offset
+ * @returns {Uint8Array}
+ */
+export function extractJPEG(bytes, offset) {
+    const stream = new Stream(bytes.slice(offset));
+
+    while (stream.hasMore()) {
+        const marker = stream.getBytes(2);
+        if (marker[0] !== 0xff) throw new Error(`Invalid marker while parsing JPEG at pos ${stream.position}: ${marker}`);
+
+        let segmentSize = 0;
+        switch (marker[1]) {
+            // No length
+            case 0xd8: // Start of Image
+            case 0x01: // For temporary use in arithmetic coding
+                break;
+            case 0xd9: // End found
+                return stream.carve();
+
+            // Variable size segment
+            case 0xc0: // Start of frame (Baseline DCT)
+            case 0xc1: // Start of frame (Extended sequential DCT)
+            case 0xc2: // Start of frame (Progressive DCT)
+            case 0xc3: // Start of frame (Lossless sequential)
+            case 0xc4: // Define Huffman Table
+            case 0xc5: // Start of frame (Differential sequential DCT)
+            case 0xc6: // Start of frame (Differential progressive DCT)
+            case 0xc7: // Start of frame (Differential lossless)
+            case 0xc8: // Reserved for JPEG extensions
+            case 0xc9: // Start of frame (Extended sequential DCT)
+            case 0xca: // Start of frame (Progressive DCT)
+            case 0xcb: // Start of frame (Lossless sequential)
+            case 0xcc: // Define arithmetic conditioning table
+            case 0xcd: // Start of frame (Differential sequential DCT)
+            case 0xce: // Start of frame (Differential progressive DCT)
+            case 0xcf: // Start of frame (Differential lossless)
+            case 0xdb: // Define Quantization Table
+            case 0xde: // Define hierarchical progression
+            case 0xe0: // Application-specific
+            case 0xe1: // Application-specific
+            case 0xe2: // Application-specific
+            case 0xe3: // Application-specific
+            case 0xe4: // Application-specific
+            case 0xe5: // Application-specific
+            case 0xe6: // Application-specific
+            case 0xe7: // Application-specific
+            case 0xe8: // Application-specific
+            case 0xe9: // Application-specific
+            case 0xea: // Application-specific
+            case 0xeb: // Application-specific
+            case 0xec: // Application-specific
+            case 0xed: // Application-specific
+            case 0xee: // Application-specific
+            case 0xef: // Application-specific
+            case 0xfe: // Comment
+                segmentSize = stream.readInt(2, "be");
+                stream.position += segmentSize - 2;
+                break;
+
+            // 1 byte
+            case 0xdf: // Expand reference image
+                stream.position++;
+                break;
+
+            // 2 bytes
+            case 0xdc: // Define number of lines
+            case 0xdd: // Define restart interval
+                stream.position += 2;
+                break;
+
+            // Start scan
+            case 0xda: // Start of scan
+                segmentSize = stream.readInt(2, "be");
+                stream.position += segmentSize - 2;
+                stream.continueUntil(0xff);
+                break;
+
+            // Continue through encoded data
+            case 0x00: // Byte stuffing
+            case 0xd0: // Restart
+            case 0xd1: // Restart
+            case 0xd2: // Restart
+            case 0xd3: // Restart
+            case 0xd4: // Restart
+            case 0xd5: // Restart
+            case 0xd6: // Restart
+            case 0xd7: // Restart
+                stream.continueUntil(0xff);
+                break;
+
+            default:
+                stream.continueUntil(0xff);
+                break;
+        }
+    }
+
+    throw new Error("Unable to parse JPEG successfully");
+}
+
+
+/**
+ * GIF extractor.
+ *
+ * @param {Uint8Array} bytes
+ * @param {Number} offset
+ * @returns {Uint8Array}
+ */
+export function extractGIF(bytes, offset) {
+    const stream = new Stream(bytes.slice(offset));
+
+    // Move to application extension block.
+    stream.continueUntil([0x21, 0xff]);
+
+    // Move to Graphic Control Extension for frame #1.
+    stream.continueUntil([0x21, 0xf9]);
+    stream.moveForwardsBy(2);
+
+    while (stream.hasMore()) {
+        // Move to Image descriptor.
+        stream.moveForwardsBy(stream.readInt(1) + 1);
+
+        // Move past Image descriptor to the image data.
+        stream.moveForwardsBy(11);
+
+        // Loop until next Graphic Control Extension.
+        while (stream.getBytes(2) !== [0x21, 0xf9]) {
+            stream.moveBackwardsBy(2);
+            stream.moveForwardsBy(stream.readInt(1));
+            if (!stream.readInt(1))
+                break;
+            stream.moveBackwardsBy(1);
+        }
+
+        // When the end of the file is [0x00, 0x3b], end.
+        if (stream.readInt(1) === 0x3b)
+            break;
+
+        stream.moveForwardsBy(1);
+    }
+    return stream.carve();
+}
+
+
+/**
+ * Portable executable extractor.
+ * Assumes that the offset refers to an MZ header.
+ *
+ * @param {Uint8Array} bytes
+ * @param {number} offset
+ * @returns {Uint8Array}
+ */
+export function extractMZPE(bytes, offset) {
+    const stream = new Stream(bytes.slice(offset));
+
+    // Read pointer to PE header
+    stream.moveTo(0x3c);
+    const peAddress = stream.readInt(4, "le");
+
+    // Move to PE header
+    stream.moveTo(peAddress);
+
+    // Get number of sections
+    stream.moveForwardsBy(6);
+    const numSections = stream.readInt(2, "le");
+
+    // Read Optional Header Magic to determine the state of the image file
+    // 0x10b = normal executable, 0x107 = ROM image, 0x20b = PE32+ executable
+    stream.moveForwardsBy(16);
+    const optionalMagic = stream.readInt(2, "le");
+    const pe32Plus = optionalMagic === 0x20b;
+
+    // Move to Data Directory
+    const dataDirectoryOffset = pe32Plus ? 112 : 96;
+    stream.moveForwardsBy(dataDirectoryOffset - 2);
+
+    // Read Certificate Table address and size (IMAGE_DIRECTORY_ENTRY_SECURITY)
+    stream.moveForwardsBy(32);
+    const certTableAddress = stream.readInt(4, "le");
+    const certTableSize = stream.readInt(4, "le");
+
+    // PE files can contain extra data appended to the end of the file called an "overlay".
+    // This data is not covered by the PE header and could be any arbitrary format, so its
+    // length cannot be determined without contextual information.
+    // However, the Attribute Certificate Table is stored in the overlay - usually right at
+    // the end. Therefore, if this table is defined, we can use its offset and size to carve
+    // out the entire PE file, including the overlay.
+    // If the Certificate Table is not defined, we continue to parse the PE file as best we
+    // can up to the end of the final section, not including any appended data in the overlay.
+    if (certTableAddress > 0) {
+        stream.moveTo(certTableAddress + certTableSize);
+        return stream.carve();
+    }
+
+    // Move past Optional Header to Section Header
+    stream.moveForwardsBy(88);
+
+    // Move to final section header
+    stream.moveForwardsBy((numSections - 1) * 0x28);
+
+    // Get raw data info
+    stream.moveForwardsBy(16);
+    const rawDataSize = stream.readInt(4, "le");
+    const rawDataAddress = stream.readInt(4, "le");
+
+    // Move to end of final section
+    stream.moveTo(rawDataAddress + rawDataSize);
+
+    return stream.carve();
+}
+
+
+/**
+ * PDF extractor.
+ *
+ * @param {Uint8Array} bytes
+ * @param {number} offset
+ * @returns {Uint8Array}
+ */
+export function extractPDF(bytes, offset) {
+    const stream = new Stream(bytes.slice(offset));
+
+    // Find end-of-file marker (%%EOF)
+    stream.continueUntil([0x25, 0x25, 0x45, 0x4f, 0x46]);
+    stream.moveForwardsBy(5);
+    stream.consumeIf(0x0d);
+    stream.consumeIf(0x0a);
+
+    return stream.carve();
+}
+
+
+/**
+ * ZIP extractor.
+ *
+ * @param {Uint8Array} bytes
+ * @param {number} offset
+ * @returns {Uint8Array}
+ */
+export function extractZIP(bytes, offset) {
+    const stream = new Stream(bytes.slice(offset));
+
+    // Find End of central directory record
+    stream.continueUntil([0x50, 0x4b, 0x05, 0x06]);
+
+    // Get comment length and consume
+    stream.moveForwardsBy(20);
+    const commentLength = stream.readInt(2, "le");
+    stream.moveForwardsBy(commentLength);
+
+    return stream.carve();
+}
+
+
+/**
+ * MACHO extractor
+ *
+ * @param {Uint8Array} bytes
+ * @param {number} offset
+ * @returns {Uint8Array}
+ */
+export function extractMACHO(bytes, offset) {
+
+    // Magic bytes.
+    const MHCIGAM64 = "207250237254";
+    const MHMAGIC64 = "254237250207";
+    const MHCIGAM = "206250237254";
+
+
+    /**
+     * Checks to see if the file is 64-bit.
+     *
+     * @param {string} magic
+     * @returns {bool}
+     */
+    function isMagic64(magic) {
+        return magic === MHCIGAM64 || magic === MHMAGIC64;
+    }
+
+
+    /**
+     * Checks the endianness of the file.
+     *
+     * @param {string} magic
+     * @returns {bool}
+     */
+    function shouldSwapBytes(magic) {
+        return magic === MHCIGAM || magic === MHCIGAM64;
+    }
+
+
+    /**
+     * Jumps through segment information and calculates the sum of the segement sizes.
+     *
+     * @param {Stream} stream
+     * @param {number} offset
+     * @param {string} isSwap
+     * @param {number} ncmds
+     * @returns {number}
+     */
+    function dumpSegmentCommands(stream, offset, isSwap, ncmds) {
+        let total = 0;
+        const LCSEGEMENT64 = 0x19;
+        const LCSEGEMENT = 0x1;
+
+        for (let i = 0; i < ncmds; i++) {
+
+            // Move to start of segment.
+            stream.moveTo(offset);
+            const cmd = stream.readInt(4, isSwap);
+            if (cmd === LCSEGEMENT64) {
+
+                // Move to size of segment field.
+                stream.moveTo(offset + 48);
+
+                // Extract size of segement.
+                total += stream.readInt(8, isSwap);
+                stream.moveTo(offset + 4);
+
+                // Move to offset of next segment.
+                offset += stream.readInt(4, isSwap);
+            } else if (cmd === LCSEGEMENT) {
+                stream.moveTo(offset + 36);
+
+                // Extract size of segement.
+                total += stream.readInt(4, isSwap);
+                stream.moveTo(offset + 4);
+                offset += stream.readInt(4, isSwap);
+            }
+        }
+        return total;
+    }
+
+
+    /**
+     * Reads the number of command segments.
+     *
+     * @param {Stream} stream
+     * @param {bool} is64
+     * @param {string} isSwap
+     * @returns {number}
+     */
+    function dumpMachHeader(stream, is64, isSwap) {
+        let loadCommandsOffset = 28;
+        if (is64)
+            loadCommandsOffset += 4;
+
+        // Move to number of commands field.
+        stream.moveTo(16);
+        const ncmds = stream.readInt(4, isSwap);
+        return dumpSegmentCommands(stream, loadCommandsOffset, isSwap, ncmds);
+    }
+
+
+    const stream = new Stream(bytes.slice(offset));
+    const magic = stream.getBytes(4).join("");
+
+    // Move to the end of the final segment.
+    stream.moveTo(dumpMachHeader(stream, isMagic64(magic), shouldSwapBytes(magic) ? "le" : "be"));
+    return stream.carve();
+}
+
+
+/**
+ * TAR extractor.
+ *
+ * @param {Uint8Array} bytes
+ * @param {number} offset
+ * @returns {Uint8Array}
+ */
+export function extractTAR(bytes, offset) {
+    const stream = new Stream(bytes.slice(offset));
+    while (stream.hasMore()) {
+
+        // Move to ustar identifier.
+        stream.moveForwardsBy(0x101);
+        if (stream.getBytes(5).join("") !== [0x75, 0x73, 0x74, 0x61, 0x72].join("")) {
+            // Reverse back to the end of the last section.
+            stream.moveBackwardsBy(0x106);
+            break;
+        }
+
+        // Move back to file size field.
+        stream.moveBackwardsBy(0x8a);
+        let fsize = 0;
+
+        // Read file size field.
+        stream.getBytes(11).forEach((element, index) => {
+            fsize += (element - 48).toString();
+        });
+
+        // Round number up from octet to nearest 512.
+        fsize = (Math.ceil(parseInt(fsize, 8) / 512) * 512);
+
+        // Move forwards to the end of that file.
+        stream.moveForwardsBy(fsize + 0x179);
+    }
+    stream.consumeWhile(0x00);
+    return stream.carve();
+}
+
+
+/**
+ * PNG extractor.
+ *
+ * @param {Uint8Array} bytes
+ * @param {number} offset
+ * @returns {Uint8Array}
+ */
+export function extractPNG(bytes, offset) {
+    const stream = new Stream(bytes.slice(offset));
+
+    // Move past signature to first chunk
+    stream.moveForwardsBy(8);
+
+    let chunkSize = 0,
+        chunkType = "";
+
+    while (chunkType !== "IEND") {
+        chunkSize = stream.readInt(4, "be");
+        chunkType = stream.readString(4);
+
+        // Chunk data size + CRC checksum
+        stream.moveForwardsBy(chunkSize + 4);
+    }
+
+
+    return stream.carve();
+}
+
+
+/**
+ * WEBP extractor.
+ *
+ * @param {Uint8Array} bytes
+ * @param {number} offset
+ * @returns {Uint8Array}
+ */
+export function extractWEBP(bytes, offset) {
+    const stream = new Stream(bytes.slice(offset));
+
+    // Move to file size offset.
+    stream.moveForwardsBy(4);
+
+    // Read file size field.
+    const fileSize = stream.readInt(4, "le");
+
+    // Move to end of file.
+    // There is no need to minus 8 from the size as the size factors in the offset.
+    stream.moveForwardsBy(fileSize);
+
+    return stream.carve();
+}
+
+
+/**
+ * BMP extractor.
+ *
+ * @param {Uint8Array} bytes
+ * @param {number} offset
+ * @returns {Uint8Array}
+ */
+export function extractBMP(bytes, offset) {
+    const stream = new Stream(bytes.slice(offset));
+
+    // Move past header
+    stream.moveForwardsBy(2);
+
+    // Read full file size
+    const bmpSize = stream.readInt(4, "le");
+
+    // Move to end of file (file size minus header and size field)
+    stream.moveForwardsBy(bmpSize - 6);
+
+    return stream.carve();
+}
+
+
+/**
+ * ICO extractor.
+ *
+ * @param {Uint8Array} bytes
+ * @param {number} offset
+ */
+export function extractICO(bytes, offset) {
+    const stream = new Stream(bytes.slice(offset));
+
+    // Move to number of files there are.
+    stream.moveTo(4);
+
+    // Read the number of files stored in the ICO
+    const numberFiles = stream.readInt(2, "le");
+
+    // Move forward to the last file header.
+    stream.moveForwardsBy(8 + ((numberFiles-1) * 16));
+    const fileSize = stream.readInt(4, "le");
+    const fileOffset = stream.readInt(4, "le");
+
+    // Move to the end of the last file.
+    stream.moveTo(fileOffset + fileSize);
+    return stream.carve();
+}
+
+
+/**
+ * TARGA extractor.
+ *
+ * @param {Uint8Array} bytes
+ * @param {number} offset
+ */
+export function extractTARGA(bytes, offset) {
+    // Need all the bytes since we do not know how far up the image goes.
+    const stream = new Stream(bytes);
+    stream.moveTo(offset - 8);
+
+    // Read in the offsets of the possible areas.
+    const extensionOffset = stream.readInt(4, "le");
+    const developerOffset = stream.readInt(4, "le");
+
+    stream.moveBackwardsBy(8);
+
+    /**
+     * Moves backwards in the stream until it meet bytes that are the same as the amount of bytes moved.
+     *
+     * @param {number} sizeOfSize
+     * @param {number} maxSize
+     */
+    function moveBackwardsUntilSize(maxSize, sizeOfSize) {
+        for (let i = 0; i < maxSize; i++) {
+            stream.moveBackwardsBy(1);
+
+            // Read in sizeOfSize amount of bytes in.
+            const size = stream.readInt(sizeOfSize, "le") - 1;
+            stream.moveBackwardsBy(sizeOfSize);
+
+            // If the size matches.
+            if (size === i)
+                break;
+        }
+    }
+
+    /**
+     * Moves backwards in the stream until we meet bytes(when calculated) that are the same as the amount of bytes moved.
+     */
+    function moveBackwardsUntilImageSize() {
+        stream.moveBackwardsBy(5);
+
+        // The documentation said that 0x100000 was the largest the file could be.
+        for (let i = 0; i < 0x100000; i++) {
+
+            // (Height * Width * pixel depth in bits)/8
+            const total = (stream.readInt(2, "le") * stream.readInt(2, "le") * stream.readInt(1))/8;
+            if (total === i-1)
+                break;
+
+            stream.moveBackwardsBy(6);
+        }
+    }
+
+    if (extensionOffset || developerOffset) {
+        if (extensionOffset) {
+            // Size is stored in two bytes hence the maximum is 0xffff.
+            moveBackwardsUntilSize(0xffff, 2);
+
+            // Move to where we think the start of the file is.
+            stream.moveBackwardsBy(extensionOffset);
+        } else if (developerOffset) {
+            // Size is stored in 4 bytes hence the maxiumum is 0xffffffff.
+            moveBackwardsUntilSize(0xffffffff, 4);
+
+            // Size is stored in byte position 6 so have to move back.
+            stream.moveBackwardsBy(6);
+
+            // Move to where we think the start of the file is.
+            stream.moveBackwardsBy(developerOffset);
+        }
+    } else {
+        // Move backwards until size === number of bytes passed.
+        moveBackwardsUntilImageSize();
+
+        // Move backwards over the reaminder of the header + the 5 we borrowed in moveBackwardsUntilImageSize().
+        stream.moveBackwardsBy(0xc+5);
+    }
+
+    return stream.carve(stream.position, offset+0x12);
+}
+
+
+/**
+ * WAV extractor.
+ *
+ * @param {Uint8Array} bytes
+ * @param {Number} offset
+ * @returns {Uint8Array}
+ */
+export function extractWAV(bytes, offset) {
+    const stream = new Stream(bytes.slice(offset));
+
+    // Move to file size field.
+    stream.moveTo(4);
+
+    // Move to file size.
+    stream.moveTo(stream.readInt(4, "le") + 8);
+
+    return stream.carve();
+}
+
+
+/**
+ * MP3 extractor.
+ *
+ * @param {Uint8Array} bytes
+ * @param {Number} offset
+ * @returns {Uint8Array}
+ */
+export function extractMP3(bytes, offset) {
+    const stream = new Stream(bytes.slice(offset));
+
+    // Constants for flag byte.
+    const bitRateIndexes = ["free", 32000, 40000, 48000, 56000, 64000, 80000, 96000, 112000, 128000, 160000, 192000, 224000, 256000, 320000, "bad"];
+
+    const samplingRateFrequencyIndex = [44100, 48000, 32000, "reserved"];
+
+    // ID3 tag, move over it.
+    if ((stream.getBytes(3).toString() === [0x49, 0x44, 0x33].toString())) {
+        stream.moveTo(6);
+        const tagSize = (stream.readInt(1) << 21) | (stream.readInt(1) << 14) | (stream.readInt(1) << 7) | stream.readInt(1);
+        stream.moveForwardsBy(tagSize);
+    } else {
+        stream.moveTo(0);
+    }
+
+    // Loop over all the frame headers in the file.
+    while (stream.hasMore()) {
+
+        // If it has an old TAG frame at the end of it, fixed size, 128 bytes.
+        if (stream.getBytes(3) === [0x54, 0x41, 0x47].toString()) {
+            stream.moveForwardsBy(125);
+            break;
+        }
+
+        // If not start of frame.
+        if (stream.getBytes(2).toString() !== [0xff, 0xfb].toString()) {
+            stream.moveBackwardsBy(2);
+            break;
+        }
+
+        // Read flag byte.
+        const flags = stream.readInt(1);
+
+        // Extract frame bit rate from flag byte.
+        const bitRate = bitRateIndexes[flags >> 4];
+
+        // Extract frame sample rate from flag byte.
+        const sampleRate = samplingRateFrequencyIndex[(flags & 0x0f) >> 2];
+
+        // Padding if the frame size is not a multiple of the bitrate.
+        const padding = (flags & 0x02) >> 1;
+
+        // Things that are either not standard or undocumented.
+        if (bitRate === "free" || bitRate === "bad" || sampleRate === "reserved") {
+            stream.moveBackwardsBy(1);
+            break;
+        }
+
+        // Formula: FrameLength = (144 * BitRate / SampleRate ) + Padding
+        const frameSize = Math.floor(((144 * bitRate) / sampleRate) + padding);
+
+        // If the next move goes past the end of the bytestream then extract the entire bytestream.
+        // We assume complete frames in the above formula because there is no field that suggests otherwise.
+        if ((stream.position + frameSize) > stream.length) {
+            stream.moveTo(stream.length);
+            break;
+        } else {
+            stream.moveForwardsBy(frameSize - 3);
+        }
+    }
+    return stream.carve();
+}
+
+
+/**
+ * FLV extractor.
+ *
+ * @param {Uint8Array} bytes
+ * @param {number} offset
+ * @returns {Uint8Array}
+ */
+export function extractFLV(bytes, offset) {
+    const stream = new Stream(bytes.slice(offset));
+
+    // Move past signature, version and flags
+    stream.moveForwardsBy(5);
+
+    // Read header size
+    const headerSize = stream.readInt(4, "be");
+
+    // Skip through the rest of the header
+    stream.moveForwardsBy(headerSize - 9);
+
+    let tagSize = -11; // Fake size of previous tag header
+    while (stream.hasMore()) {
+        const prevTagSize = stream.readInt(4, "be");
+        const tagType = stream.readInt(1);
+
+        if ([8, 9, 18].indexOf(tagType) < 0) {
+            // This tag is not valid
+            stream.moveBackwardsBy(1);
+            break;
+        }
+
+        if (prevTagSize !== (tagSize + 11)) {
+            // Previous tag was not valid, reverse back over this header
+            // and the previous tag body and header
+            stream.moveBackwardsBy(tagSize + 11 + 5);
+            break;
+        }
+
+        tagSize = stream.readInt(3, "be");
+
+        // Move past the rest of the tag header and payload
+        stream.moveForwardsBy(7 + tagSize);
+    }
+
+    return stream.carve();
+}
+
+
+/**
+ * RTF extractor.
+ *
+ * @param {Uint8Array} bytes
+ * @param {number} offset
+ * @returns {Uint8Array}
+ */
+export function extractRTF(bytes, offset) {
+    const stream = new Stream(bytes.slice(offset));
+
+    let openTags = 0;
+
+    if (stream.readInt(1) !== 0x7b) { // {
+        throw new Error("Not a valid RTF file");
+    } else {
+        openTags++;
+    }
+
+    while (openTags > 0 && stream.hasMore()) {
+        switch (stream.readInt(1)) {
+            case 0x7b: // {
+                openTags++;
+                break;
+            case 0x7d: // }
+                openTags--;
+                break;
+            case 0x5c: // \
+                // Consume any more escapes and then skip over the next character
+                stream.consumeIf(0x5c);
+                stream.position++;
+                break;
+            default:
+                break;
+        }
+    }
+
+    return stream.carve();
+}
+
+
+/**
+ * SQLITE extractor.
+ *
+ * @param {Uint8Array} bytes
+ * @param {number} offset
+ * @returns {Uint8Array}
+ */
+export function extractSQLITE(bytes, offset) {
+    const stream = new Stream(bytes.slice(offset));
+
+    // Extract the size of the page.
+    stream.moveTo(16);
+    const pageSize = stream.readInt(2);
+
+    // Extract the number of pages.
+    stream.moveTo(28);
+    const numPages = stream.readInt(4);
+
+    // Move to the end of all the pages.
+    stream.moveTo(pageSize*numPages);
+
+    return stream.carve();
+}
+
+
+/**
+ * PList (XML) extractor.
+ *
+ * @param {Uint8Array} bytes
+ * @param {number} offset
+ * @returns {Uint8Array}
+ */
+export function extractPListXML(bytes, offset) {
+    const stream = new Stream(bytes.slice(offset));
+
+    let braceCount = 0;
+
+    // Continue to the first (<plist).
+    stream.continueUntil([0x3c, 0x70, 0x6c, 0x69, 0x73, 0x74]);
+    stream.moveForwardsBy(6);
+    braceCount++;
+
+    // While we have an unequal amount of braces.
+    while (braceCount > 0 && stream.hasMore()) {
+        if (stream.readInt(1) === 0x3c) {
+
+            // If we hit an <plist.
+            if (stream.getBytes(5).join("") === [0x70, 0x6c, 0x69, 0x73, 0x74].join("")) {
+                braceCount++;
+            } else {
+                stream.moveBackwardsBy(5);
+            }
+
+            // If we hit an </plist>.
+            if (stream.getBytes(7).join("") === [0x2f, 0x70, 0x6c, 0x69, 0x73, 0x74, 0x3e].join("")) {
+                braceCount--;
+            } else {
+                stream.moveBackwardsBy(7);
+            }
+        }
+    }
+    stream.consumeIf(0x0a);
+
+    return stream.carve();
+}
+
+
+/**
+ * MacOS X Keychain Extactor.
+ *
+ * @param {Uint8Array} bytes
+ * @param {number} offset
+ * @returns {Uint8Array}
+ */
+export function extractMacOSXKeychain(bytes, offset) {
+    const stream = new Stream(bytes.slice(offset));
+
+    // Move to size field.
+    stream.moveTo(0x14);
+
+    // Move forwards by size.
+    stream.moveForwardsBy(stream.readInt(4));
+
+    return stream.carve();
+}
+
+
+/**
+ * OLE2 extractor.
+ *
+ * @param {Uint8Array} bytes
+ * @param {number} offset
+ * @returns {Uint8Array}
+ */
+export function extractOLE2(bytes, offset) {
+    const stream = new Stream(bytes.slice(offset));
+    const entries = [
+        [[0x52, 0x00, 0x6f, 0x00, 0x6f, 0x00, 0x74, 0x00, 0x20, 0x00, 0x45, 0x00, 0x6e, 0x00, 0x74, 0x00, 0x72, 0x00, 0x79], 19, "Root Entry"],
+        [[0x57, 0x00, 0x6f, 0x00, 0x72, 0x00, 0x6b, 0x00, 0x62, 0x00, 0x6f, 0x00, 0x6f, 0x00, 0x6b], 15, "Workbook"],
+        [[0x43, 0x00, 0x75, 0x00, 0x72, 0x00, 0x72, 0x00, 0x65, 0x00, 0x6e, 0x00, 0x74, 0x00, 0x20, 0x00, 0x55, 0x00, 0x73, 0x00, 0x65, 0x00, 0x72],  23,  "Current User"],
+        [[0x50, 0x00, 0x6f, 0x00, 0x77, 0x00, 0x65, 0x00, 0x72, 0x00, 0x50, 0x00, 0x6f, 0x00, 0x69, 0x00, 0x6e, 0x00, 0x74, 0x00, 0x20, 0x00, 0x44, 0x00, 0x6f, 0x00, 0x63, 0x00, 0x75, 0x00, 0x6d, 0x00, 0x65, 0x00, 0x6e, 0x00, 0x74], 37, "PowerPoint Document"],
+        [[0x57, 0x00, 0x6f, 0x00, 0x72, 0x00, 0x64, 0x00, 0x44, 0x00, 0x6f, 0x00, 0x63, 0x00, 0x75, 0x00, 0x6d, 0x00, 0x65, 0x00, 0x6e, 0x00, 0x74], 23, "WordDocument"],
+        [[0x44, 0x00, 0x61, 0x00, 0x74, 0x00, 0x61], 7, "Data"],
+        [[0x50, 0x00, 0x69, 0x00, 0x63, 0x00, 0x74, 0x00, 0x75, 0x00, 0x72, 0x00, 0x65, 0x00, 0x73], 15, "Pictures"],
+        [[0x31, 0x00, 0x54, 0x00, 0x61, 0x00, 0x62, 0x00, 0x6c, 0x00, 0x65], 11, "1Table"],
+        [[0x05, 0x00, 0x53, 0x00, 0x75, 0x00, 0x6d, 0x00, 0x6d, 0x00, 0x61, 0x00, 0x72, 0x00, 0x79, 0x00, 0x49, 0x00, 0x6e, 0x00, 0x66, 0x00, 0x6f, 0x00, 0x72, 0x00, 0x6d, 0x00, 0x61, 0x00, 0x74, 0x00, 0x69, 0x00, 0x6f, 0x00, 0x6e], 37, "SummaryInformation"],
+        [[0x05, 0x00, 0x44, 0x00, 0x6f, 0x00, 0x63, 0x00, 0x75, 0x00, 0x6d, 0x00, 0x65, 0x00, 0x6e, 0x00, 0x74, 0x00, 0x53, 0x00, 0x75, 0x00, 0x6d, 0x00, 0x6d, 0x00, 0x61, 0x00, 0x72, 0x00, 0x79, 0x00, 0x49, 0x00, 0x6e, 0x00, 0x66, 0x00, 0x6f, 0x00, 0x72, 0x00, 0x6d, 0x00, 0x61, 0x00, 0x74, 0x00, 0x69, 0x00, 0x6f, 0x00, 0x6e], 53, "DocumentSummaryInformation"],
+        [[0x43, 0x00, 0x6f, 0x00, 0x6d, 0x00, 0x70, 0x00, 0x4f, 0x00, 0x62, 0x00, 0x6a], 13, "Comp Obj"],
+        [[0x01, 0x00], 2, "Entry"]
+    ];
+    let endianness = "le";
+
+    // Move to endianess field.
+    stream.moveForwardsBy(28);
+    if (stream.readInt(2, endianness) === 0xfffe)
+        endianness = "be";
+
+    // Calculate the size of the normal sectors.
+    const sizeOfSector = 2 ** stream.readInt(2, endianness);
+
+    // Move to root directory offset field.
+    stream.moveTo(48);
+
+    // Read root directory offset.
+    const rootStuff  = stream.readInt(4, endianness);
+
+    // Calculate root directory offset.
+    let total = 512 + (rootStuff * sizeOfSector);
+    stream.moveTo(total);
+
+    // While valid directory entries.
+    let found = true;
+    while (found) {
+        found = false;
+
+        // Attempt to determine what directory entry it is.
+        for (const element of entries) {
+
+            // If the byte pattern matches.
+            if (stream.getBytes(element[1]).join("") === element[0].join("")) {
+                stream.moveBackwardsBy(element[1]);
+                found = true;
+
+                // Move forwards by the size of the comp obj.
+                if (element[2] === "Comp Obj") {
+
+                    // The size of the Comp Obj entry - 128. Since we add 128 later.
+                    total += 128 * 6;
+                    stream.moveTo(total);
+                } else if (element[2] === "Entry") {
+
+                    // If there is an entry move backwards by 126 to then move forwards by 128. Hence a total displacement of 2.
+                    stream.moveBackwardsBy(126);
+                }
+                break;
+            }
+            stream.moveBackwardsBy(element[1]);
+        }
+
+        // If we have found a valid entry, move forwards by 128.
+        if (found) {
+
+            // Every entry is at least 128 in size, some are bigger which is dealt with by the above if statement.
+            total += 128;
+            stream.moveForwardsBy(128);
+        }
+    }
+
+    // Round up to a multiple of 512.
+    total = Math.ceil(total / 512) * 512;
+
+    stream.moveTo(total);
+    return stream.carve();
+}
+
+
+/**
+ * GZIP extractor.
+ *
+ * @param {Uint8Array} bytes
+ * @param {number} offset
+ * @returns {Uint8Array}
+ */
+export function extractGZIP(bytes, offset) {
+    const stream = new Stream(bytes.slice(offset));
+
+
+    /* HEADER */
+
+    // Skip over signature and compression method
+    stream.moveForwardsBy(3);
+
+    // Read flags
+    const flags = stream.readInt(1);
+
+    // Skip over last modification time
+    stream.moveForwardsBy(4);
+
+    // Read compression flags
+    stream.readInt(1);
+
+    // Skip over OS
+    stream.moveForwardsBy(1);
+
+
+    /* OPTIONAL HEADERS */
+
+    // Extra fields
+    if (flags & 0x4) {
+        const extraFieldsSize = stream.readInt(2, "le");
+        stream.moveForwardsby(extraFieldsSize);
+    }
+
+    // Original filename
+    if (flags & 0x8) {
+        stream.continueUntil(0x00);
+        stream.moveForwardsBy(1);
+    }
+
+    // Comment
+    if (flags & 0x10) {
+        stream.continueUntil(0x00);
+        stream.moveForwardsBy(1);
+    }
+
+    // Checksum
+    if (flags & 0x2) {
+        stream.moveForwardsBy(2);
+    }
+
+
+    /* DEFLATE DATA */
+
+    parseDEFLATE(stream);
+
+
+    /* FOOTER */
+
+    // Skip over checksum and size of original uncompressed input
+    stream.moveForwardsBy(8);
+
+    return stream.carve();
+}
+
+
+/**
+ * BZIP2 extractor.
+ *
+ * @param {Uint8Array} bytes
+ * @param {Number} offset
+ * @returns {Uint8Array}
+ */
+export function extractBZIP2(bytes, offset) {
+    const stream = new Stream(bytes.slice(offset));
+
+    // The EOFs shifted between all possible combinations.
+    const lookingfor = [
+        [0x77, 0x24, 0x53, 0x85, 0x09],
+        [0xee, 0x48, 0xa7, 0x0a, 0x12],
+        [0xdc, 0x91, 0x4e, 0x14, 0x24],
+        [0xb9, 0x22, 0x9c, 0x28, 0x48],
+        [0x72, 0x45, 0x38, 0x50, 0x90],
+        [0xbb, 0x92, 0x29, 0xc2, 0x84],
+        [0x5d, 0xc9, 0x14, 0xe1, 0x42],
+        [0x2e, 0xe4, 0x8a, 0x70, 0xa1],
+        [0x17, 0x72, 0x45, 0x38, 0x50]
+    ];
+
+    for (let i = 0; i < lookingfor.length; i++) {
+        // Continue until an EOF.
+        stream.continueUntil(lookingfor[i]);
+        if (stream.getBytes(5).join("") === lookingfor[i].join(""))
+            break;
+
+        // Jump back to the start if invalid EOF.
+        stream.moveTo(0);
+    }
+    stream.moveForwardsBy(4);
+    return stream.carve();
+}
+
+
+/**
+ * Zlib extractor.
+ *
+ * @param {Uint8Array} bytes
+ * @param {number} offset
+ * @returns {Uint8Array}
+ */
+export function extractZlib(bytes, offset) {
+    const stream = new Stream(bytes.slice(offset));
+
+    // Skip over CMF
+    stream.moveForwardsBy(1);
+
+    // Read flags
+    const flags = stream.readInt(1);
+
+    // Skip over preset dictionary checksum
+    if (flags & 0x20) {
+        stream.moveForwardsBy(4);
+    }
+
+    // Parse DEFLATE stream
+    parseDEFLATE(stream);
+
+    // Skip over final checksum
+    stream.moveForwardsBy(4);
+
+    return stream.carve();
+}
+
+
+/**
+ * XZ extractor.
+ *
+ * @param {Uint8Array} bytes
+ * @param {Number} offset
+ * @returns {string}
+ */
+export function extractXZ(bytes, offset) {
+    const stream = new Stream(bytes.slice(offset));
+
+    // Move forward to EOF marker
+    stream.continueUntil([0x00, 0x00, 0x00, 0x00, 0x04, 0x59, 0x5a]);
+
+    // Move over EOF marker
+    stream.moveForwardsBy(7);
+
+    return stream.carve();
+}
+
+
+/**
+ * DEB extractor.
+ *
+ * @param {Uint8Array} bytes
+ * @param {Number} offset
+ */
+export function extractDEB(bytes, offset) {
+    const stream = new Stream(bytes.slice(offset));
+
+    // Move past !<arch>
+    stream.moveForwardsBy(8);
+    while (stream.hasMore()) {
+
+        // Move to size field.
+        stream.moveForwardsBy(48);
+        let fsize= "";
+
+        // Convert size to a usable number.
+        for (const elem of stream.getBytes(10)) {
+            fsize += String.fromCharCode(elem);
+        }
+        fsize = parseInt(fsize.trim(), 10);
+
+        // Move past `\n
+        stream.moveForwardsBy(2);
+        stream.moveForwardsBy(fsize);
+    }
+    return stream.carve();
+}
+
+
+/**
+ * ELF extractor.
+ *
+ * @param {Uint8Array} bytes
+ * @param {number} offset
+ * @returns {Uint8Array}
+ */
+export function extractELF(bytes, offset) {
+    const stream = new Stream(bytes.slice(offset));
+
+    // Skip over magic number
+    stream.moveForwardsBy(4);
+
+    // Read architecture (x86 == 1, x64 == 2)
+    const x86 = stream.readInt(1) === 1;
+
+    // Read endianness (1 == little, 2 == big)
+    const endian = stream.readInt(1) === 1 ? "le" : "be";
+
+    // Skip over header values
+    stream.moveForwardsBy(x86 ? 26 : 34);
+
+    // Read section header table offset
+    const shoff = x86 ? stream.readInt(4, endian) : stream.readInt(8, endian);
+
+    // Skip over flags, header size and program header size and entries
+    stream.moveForwardsBy(10);
+
+    // Read section header table entry size
+    const shentsize = stream.readInt(2, endian);
+
+    // Read number of entries in the section header table
+    const shnum = stream.readInt(2, endian);
+
+    // Jump to section header table
+    stream.moveTo(shoff);
+
+    // Move past each section header
+    stream.moveForwardsBy(shentsize * shnum);
+
+    return stream.carve();
+}
+
+
+// Construct required Huffman Tables
+const fixedLiteralTableLengths = new Array(288);
+for (let i = 0; i < fixedLiteralTableLengths.length; i++) {
+    fixedLiteralTableLengths[i] =
+        (i <= 143) ? 8 :
+            (i <= 255) ? 9 :
+                (i <= 279) ? 7 :
+                    8;
+}
+const fixedLiteralTable = buildHuffmanTable(fixedLiteralTableLengths);
+const fixedDistanceTableLengths = new Array(30).fill(5);
+const fixedDistanceTable = buildHuffmanTable(fixedDistanceTableLengths);
+const huffmanOrder = [16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15];
+
+/**
+ * Steps through a DEFLATE stream
+ *
+ * @param {Stream} stream
+ */
+function parseDEFLATE(stream) {
+    // Parse DEFLATE data
+    let finalBlock = 0;
+
+    while (!finalBlock) {
+        // Read header
+        finalBlock = stream.readBits(1, "le");
+        const blockType = stream.readBits(2, "le");
+
+        if (blockType === 0) {
+            /* No compression */
+
+            // Consume the rest of the current byte
+            stream.moveForwardsBy(1);
+            // Read the block length value
+            const blockLength = stream.readInt(2, "le");
+            // Move to the end of this block
+            stream.moveForwardsBy(2 + blockLength);
+        } else if (blockType === 1) {
+            /* Fixed Huffman */
+
+            parseHuffmanBlock(stream, fixedLiteralTable, fixedDistanceTable);
+        } else if (blockType === 2) {
+            /* Dynamic Huffman */
+
+            // Read the number of liternal and length codes
+            const hlit = stream.readBits(5, "le") + 257;
+            // Read the number of distance codes
+            const hdist = stream.readBits(5, "le") + 1;
+            // Read the number of code lengths
+            const hclen = stream.readBits(4, "le") + 4;
+
+            // Parse code lengths
+            const codeLengths = new Uint8Array(huffmanOrder.length);
+            for (let i = 0; i < hclen; i++) {
+                codeLengths[huffmanOrder[i]] = stream.readBits(3, "le");
+            }
+
+            // Parse length table
+            const codeLengthsTable = buildHuffmanTable(codeLengths);
+            const lengthTable = new Uint8Array(hlit + hdist);
+
+            let code, repeat, prev;
+            for (let i = 0; i < hlit + hdist;) {
+                code = readHuffmanCode(stream, codeLengthsTable);
+                switch (code) {
+                    case 16:
+                        repeat = 3 + stream.readBits(2, "le");
+                        while (repeat--) lengthTable[i++] = prev;
+                        break;
+                    case 17:
+                        repeat = 3 + stream.readBits(3, "le");
+                        while (repeat--) lengthTable[i++] = 0;
+                        prev = 0;
+                        break;
+                    case 18:
+                        repeat = 11 + stream.readBits(7, "le");
+                        while (repeat--) lengthTable[i++] = 0;
+                        prev = 0;
+                        break;
+                    default:
+                        lengthTable[i++] = code;
+                        prev = code;
+                        break;
+                }
+            }
+
+            const dynamicLiteralTable = buildHuffmanTable(lengthTable.subarray(0, hlit));
+            const dynamicDistanceTable = buildHuffmanTable(lengthTable.subarray(hlit));
+
+            parseHuffmanBlock(stream, dynamicLiteralTable, dynamicDistanceTable);
+        } else {
+            throw new Error(`Invalid block type while parsing DEFLATE stream at pos ${stream.position}`);
+        }
+    }
+
+    // Consume final byte if it has not been fully consumed yet
+    if (stream.bitPos > 0)
+        stream.moveForwardsBy(1);
+}
+
+
+// Static length tables
+const lengthExtraTable = [
+    0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 0, 0, 0
+];
+const distanceExtraTable = [
+    0, 0, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13
+];
+
+/**
+ * Parses a Huffman Block given the literal and distance tables
+ *
+ * @param {Stream} stream
+ * @param {Uint32Array} litTab
+ * @param {Uint32Array} distTab
+ */
+function parseHuffmanBlock(stream, litTab, distTab) {
+    let code;
+    let loops = 0;
+    while ((code = readHuffmanCode(stream, litTab))) {
+        // console.log("Code: " + code + " (" + Utils.chr(code) + ") " + Utils.bin(code));
+
+        // End of block
+        if (code === 256) break;
+
+        // Detect probably infinite loops
+        if (++loops > 10000)
+            throw new Error("Caught in probable infinite loop while parsing Huffman Block");
+
+        // Literal
+        if (code < 256) continue;
+
+        // Length code
+        stream.readBits(lengthExtraTable[code - 257], "le");
+
+        // Dist code
+        code = readHuffmanCode(stream, distTab);
+        stream.readBits(distanceExtraTable[code], "le");
+    }
+}
+
+
+/**
+ * Builds a Huffman table given the relevant code lengths
+ *
+ * @param {Array} lengths
+ * @returns {Array} result
+ * @returns {Uint32Array} result.table
+ * @returns {number} result.maxCodeLength
+ * @returns {number} result.minCodeLength
+ */
+function buildHuffmanTable(lengths) {
+    const maxCodeLength = Math.max.apply(Math, lengths);
+    const minCodeLength = Math.min.apply(Math, lengths);
+    const size = 1 << maxCodeLength;
+    const table = new Uint32Array(size);
+
+    for (let bitLength = 1, code = 0, skip = 2; bitLength <= maxCodeLength;) {
+        for (let i = 0; i < lengths.length; i++) {
+            if (lengths[i] === bitLength) {
+                let reversed, rtemp, j;
+                for (reversed = 0, rtemp = code, j = 0; j < bitLength; j++) {
+                    reversed = (reversed << 1) | (rtemp & 1);
+                    rtemp >>= 1;
+                }
+
+                const value = (bitLength << 16) | i;
+                for (let j = reversed; j < size; j += skip) {
+                    table[j] = value;
+                }
+
+                code++;
+            }
+        }
+
+        bitLength++;
+        code <<= 1;
+        skip <<= 1;
+    }
+
+    return [table, maxCodeLength, minCodeLength];
+}
+
+
+/**
+ * Reads the next Huffman code from the stream, given the relevant code table
+ *
+ * @param {Stream} stream
+ * @param {Uint32Array} table
+ * @returns {number}
+ */
+function readHuffmanCode(stream, table) {
+    const [codeTable, maxCodeLength] = table;
+
+    // Read max length
+    const bitsBuf = stream.readBits(maxCodeLength, "le");
+    const codeWithLength = codeTable[bitsBuf & ((1 << maxCodeLength) - 1)];
+    const codeLength = codeWithLength >>> 16;
+
+    if (codeLength > maxCodeLength) {
+        throw new Error(`Invalid Huffman Code length while parsing DEFLATE block at pos ${stream.position}: ${codeLength}`);
+    }
+
+    stream.moveBackwardsByBits(maxCodeLength - codeLength);
+
+    return codeWithLength & 0xffff;
+}
+
+
+/**
+ * EVTX extractor.
+ *
+ * @param {Uint8Array} bytes
+ * @param {Number} offset
+ * @returns {Uint8Array}
+ */
+export function extractEVTX(bytes, offset) {
+    const stream = new Stream(bytes.slice(offset));
+
+    // Move to first ELFCHNK.
+    stream.moveTo(0x28);
+    const total = stream.readInt(4, "le") - 0x2c;
+    stream.moveForwardsBy(total);
+
+    while (stream.hasMore()) {
+        // Loop through ELFCHNKs.
+        if (stream.getBytes(7).join("") !== [0x45, 0x6c, 0x66, 0x43, 0x68, 0x6e, 0x6b].join(""))
+            break;
+        stream.moveForwardsBy(0xfff9);
+    }
+    stream.consumeWhile(0x00);
+    return stream.carve();
+}
+
+
+/**
+ * EVT extractor.
+ *
+ * @param {Uint8Array} bytes
+ * @param {Number} offset
+ * @returns {Uint8Array}
+ */
+export function extractEVT(bytes, offset) {
+    const stream = new Stream(bytes.slice(offset));
+
+    // Extract offset of EOF.
+    stream.moveTo(0x14);
+    const eofOffset = stream.readInt(4, "le");
+    stream.moveTo(eofOffset);
+
+    // Extract the size of the EOF.
+    const eofSize = stream.readInt(4, "le");
+
+    // Move past EOF.
+    stream.moveForwardsBy(eofSize-4);
+    return stream.carve();
+}
+
+
+/**
+ * DMP extractor.
+ *
+ * @param {Uint8Array} bytes
+ * @param {Number} offset
+ * @returns {Uint8Array}
+ */
+export function extractDMP(bytes, offset) {
+    const stream = new Stream(bytes.slice(offset));
+
+    // Move to fileSize field.
+    stream.moveTo(0x70);
+
+    // Multiply number of pages by page size. Plus 1 since the header is a page.
+    stream.moveTo((stream.readInt(4, "le") + 1) * 0x1000);
+
+    return stream.carve();
+}
+
+
+/**
+ * PF extractor.
+ *
+ * @param {Uint8Array} bytes
+ * @param {Number} offset
+ * @returns {Uint8Array}
+ */
+export function extractPF(bytes, offset) {
+    const stream = new Stream(bytes.slice(offset));
+
+    // Move to file size.
+    stream.moveTo(12);
+    stream.moveTo(stream.readInt(4, "be"));
+
+    return stream.carve();
+}
+
+
+/**
+ * PF (Win 10) extractor.
+ *
+ * @param {Uint8Array} bytes
+ * @param {Number} offset
+ * @returns {Uint8Array}
+ */
+export function extractPFWin10(bytes, offset) {
+    const stream = new Stream(bytes.slice(offset));
+
+    // Read in file size.
+    stream.moveTo(stream.readInt(4, "be"));
+
+    return stream.carve();
+}
+
+
+/**
+ * LNK extractor.
+ *
+ * @param {Uint8Array} bytes
+ * @param {Number} offset
+ * @returns {Uint8Array}
+ */
+export function extractLNK(bytes, offset) {
+    const stream = new Stream(bytes.slice(offset));
+
+    // Move to file size field.
+    stream.moveTo(0x34);
+    stream.moveTo(stream.readInt(4, "le"));
+
+    return stream.carve();
+}
+
+
+/**
+ * LZOP extractor.
+ *
+ * @param {Uint8Array} bytes
+ * @param {Number} offset
+ * @returns {Uint8Array}
+ */
+export function extractLZOP(bytes, offset) {
+    const stream = new Stream(bytes.slice(offset));
+
+    // Flag bits.
+    const F_ADLER32_D = 0x00000001;
+    const F_ADLER32_C = 0x00000002;
+    const F_CRC32_D = 0x00000100;
+    const F_CRC32_C = 0x00000200;
+    const F_H_FILTER = 0x00000800;
+    const F_H_EXTRA_FIELD = 0x00000040;
+
+    let numCheckSumC = 0, numCheckSumD = 0;
+
+    // Move over magic bytes.
+    stream.moveForwardsBy(9);
+
+    const version = stream.readInt(2, "be");
+
+    // Move to flag register offset.
+    stream.moveForwardsBy(6);
+    const flags = stream.readInt(4, "be");
+
+    if (version & F_H_FILTER)
+        stream.moveForwardsBy(4);
+
+    if (flags & F_ADLER32_C)
+        numCheckSumC++;
+
+    if (flags & F_CRC32_C)
+        numCheckSumC++;
+
+    if (flags & F_ADLER32_D)
+        numCheckSumD++;
+
+    if (flags & F_CRC32_D)
+        numCheckSumD++;
+
+    // Move over the mode, mtime_low
+    stream.moveForwardsBy(8);
+
+    if (version >= 0x0940)
+        stream.moveForwardsBy(4);
+
+    const fnameSize = stream.readInt(1, "be");
+
+    // Move forwards by size of file name and the following 4 byte checksum.
+    stream.moveForwardsBy(fnameSize);
+
+    if (flags & F_H_EXTRA_FIELD) {
+        const extraSize = stream.readInt(4, "be");
+        stream.moveForwardsBy(extraSize);
+    }
+
+    // Move past checksum.
+    stream.moveForwardsBy(4);
+
+    while (stream.hasMore()) {
+        const uncompSize = stream.readInt(4, "be");
+
+        // If data has no length, break.
+        if (uncompSize === 0)
+            break;
+
+        const compSize = stream.readInt(4, "be");
+
+        const numCheckSumSkip = (uncompSize === compSize) ? numCheckSumD : numCheckSumD + numCheckSumC;
+
+        // skip forwards by compressed data size and the size of the checksum(s).
+        stream.moveForwardsBy(compSize + (numCheckSumSkip * 4));
+    }
+    return stream.carve();
+
+}

src/core/lib/FileType.mjs

@@ -0,0 +1,267 @@
+/**
+ * File type functions
+ *
+ * @author n1474335 [n1474335@gmail.com]
+ * @copyright Crown Copyright 2018
+ * @license Apache-2.0
+ *
+ */
+import {FILE_SIGNATURES} from "./FileSignatures.mjs";
+import {sendStatusMessage} from "../Utils.mjs";
+
+
+/**
+ * Checks whether a signature matches a buffer.
+ *
+ * @param {Object|Object[]} sig - A dictionary of offsets with values assigned to them.
+ *   These values can be numbers for static checks, arrays of potential valid matches,
+ *   or bespoke functions to check the validity of the buffer value at that offset.
+ * @param {Uint8Array} buf
+ * @param {number} [offset=0] Where in the buffer to start searching from
+ * @returns {boolean}
+ */
+function signatureMatches(sig, buf, offset=0) {
+    // Using a length check seems to be more performant than `sig instanceof Array`
+    if (sig.length) {
+        // sig is an Array - return true if any of them match
+        // The following `reduce` method is nice, but performance matters here, so we
+        // opt for a faster, if less elegant, for loop.
+        // return sig.reduce((acc, s) => acc || bytesMatch(s, buf, offset), false);
+        for (let i = 0; i < sig.length; i++) {
+            if (bytesMatch(sig[i], buf, offset)) return true;
+        }
+        return false;
+    } else {
+        return bytesMatch(sig, buf, offset);
+    }
+}
+
+
+/**
+ * Checks whether a set of bytes match the given buffer.
+ *
+ * @param {Object} sig - A dictionary of offsets with values assigned to them.
+ *   These values can be numbers for static checks, arrays of potential valid matches,
+ *   or bespoke functions to check the validity of the buffer value at that offset.
+ * @param {Uint8Array} buf
+ * @param {number} [offset=0] Where in the buffer to start searching from
+ * @returns {boolean}
+ */
+function bytesMatch(sig, buf, offset=0) {
+    for (const sigoffset in sig) {
+        const pos = parseInt(sigoffset, 10) + offset;
+        switch (typeof sig[sigoffset]) {
+            case "number": // Static check
+                if (buf[pos] !== sig[sigoffset])
+                    return false;
+                break;
+            case "object": // Array of options
+                if (sig[sigoffset].indexOf(buf[pos]) < 0)
+                    return false;
+                break;
+            case "function": // More complex calculation
+                if (!sig[sigoffset](buf[pos]))
+                    return false;
+                break;
+            default:
+                throw new Error(`Unrecognised signature type at offset ${sigoffset}`);
+        }
+    }
+    return true;
+}
+
+
+/**
+ * Given a buffer, detects magic byte sequences at specific positions and returns the
+ * extension and mime type.
+ *
+ * @param {Uint8Array|ArrayBuffer} buf
+ * @param {string[]} [categories=All] - Which categories of file to look for
+ * @returns {Object[]} types
+ * @returns {string} type.name - Name of file type
+ * @returns {string} type.ext - File extension
+ * @returns {string} type.mime - Mime type
+ * @returns {string} [type.desc] - Description
+ */
+export function detectFileType(buf, categories=Object.keys(FILE_SIGNATURES)) {
+    if (buf instanceof ArrayBuffer) {
+        buf = new Uint8Array(buf);
+    }
+
+    if (!(buf && buf.length > 1)) {
+        return [];
+    }
+
+    const matchingFiles = [];
+    const signatures = {};
+
+    for (const cat in FILE_SIGNATURES) {
+        if (categories.includes(cat)) {
+            signatures[cat] = FILE_SIGNATURES[cat];
+        }
+    }
+
+    for (const cat in signatures) {
+        const category = signatures[cat];
+
+        category.forEach(filetype => {
+            if (signatureMatches(filetype.signature, buf)) {
+                matchingFiles.push(filetype);
+            }
+        });
+    }
+    return matchingFiles;
+}
+
+
+/**
+ * Given a buffer, searches for magic byte sequences at all possible positions and returns
+ * the extensions and mime types.
+ *
+ * @param {Uint8Array} buf
+ * @param {string[]} [categories=All] - Which categories of file to look for
+ * @returns {Object[]} foundFiles
+ * @returns {number} foundFiles.offset - The position in the buffer at which this file was found
+ * @returns {Object} foundFiles.fileDetails
+ * @returns {string} foundFiles.fileDetails.name - Name of file type
+ * @returns {string} foundFiles.fileDetails.ext - File extension
+ * @returns {string} foundFiles.fileDetails.mime - Mime type
+ * @returns {string} [foundFiles.fileDetails.desc] - Description
+ */
+export function scanForFileTypes(buf, categories=Object.keys(FILE_SIGNATURES)) {
+    if (!(buf && buf.length > 1)) {
+        return [];
+    }
+
+    const foundFiles = [];
+    const signatures = {};
+
+    for (const cat in FILE_SIGNATURES) {
+        if (categories.includes(cat)) {
+            signatures[cat] = FILE_SIGNATURES[cat];
+        }
+    }
+
+    for (const cat in signatures) {
+        const category = signatures[cat];
+
+        for (let i = 0; i < category.length; i++) {
+            const filetype = category[i];
+            const sigs = filetype.signature.length ? filetype.signature : [filetype.signature];
+
+            sigs.forEach(sig => {
+                let pos = 0;
+                while ((pos = locatePotentialSig(buf, sig, pos)) >= 0) {
+                    if (bytesMatch(sig, buf, pos)) {
+                        sendStatusMessage(`Found potential signature for ${filetype.name} at pos ${pos}`);
+                        foundFiles.push({
+                            offset: pos,
+                            fileDetails: filetype
+                        });
+                    }
+                    pos++;
+                }
+            });
+        }
+    }
+
+    // Return found files in order of increasing offset
+    return foundFiles.sort((a, b) => {
+        return a.offset - b.offset;
+    });
+}
+
+
+/**
+ * Fastcheck function to quickly scan the buffer for the first byte in a signature.
+ *
+ * @param {Uint8Array} buf - The buffer to search
+ * @param {Object} sig - A single signature object (Not an array of signatures)
+ * @param {number} offset - Where to start search from
+ * @returns {number} The position of the match or -1 if one cannot be found.
+ */
+function locatePotentialSig(buf, sig, offset) {
+    // Find values for first key and value in sig
+    const k = parseInt(Object.keys(sig)[0], 10);
+    const v = Object.values(sig)[0];
+    switch (typeof v) {
+        case "number":
+            return buf.indexOf(v, offset + k) - k;
+        case "object":
+            for (let i = offset + k; i < buf.length; i++) {
+                if (v.indexOf(buf[i]) >= 0) return i - k;
+            }
+            return -1;
+        case "function":
+            for (let i = offset + k; i < buf.length; i++) {
+                if (v(buf[i])) return i - k;
+            }
+            return -1;
+        default:
+            throw new Error("Unrecognised signature type");
+    }
+}
+
+
+/**
+ * Detects whether the given buffer is a file of the type specified.
+ *
+ * @param {string|RegExp} type
+ * @param {Uint8Array|ArrayBuffer} buf
+ * @returns {string|false} The mime type or false if the type does not match
+ */
+export function isType(type, buf) {
+    const types = detectFileType(buf);
+
+    if (!types.length) return false;
+
+    if (typeof type === "string") {
+        return types.reduce((acc, t) => {
+            const mime = t.mime.startsWith(type) ? t.mime : false;
+            return acc || mime;
+        }, false);
+    } else if (type instanceof RegExp) {
+        return types.reduce((acc, t) => {
+            const mime = type.test(t.mime) ? t.mime : false;
+            return acc || mime;
+        }, false);
+    } else {
+        throw new Error("Invalid type input.");
+    }
+}
+
+
+/**
+ * Detects whether the given buffer contains an image file.
+ *
+ * @param {Uint8Array|ArrayBuffer} buf
+ * @returns {string|false} The mime type or false if the type does not match
+ */
+export function isImage(buf) {
+    return isType("image", buf);
+}
+
+
+/**
+ * Attempts to extract a file from a data stream given its offset and extractor function.
+ *
+ * @param {Uint8Array} bytes
+ * @param {Object} fileDetail
+ * @param {string} fileDetail.mime
+ * @param {string} fileDetail.extension
+ * @param {Function} fileDetail.extractor
+ * @param {number} offset
+ * @returns {File}
+ */
+export function extractFile(bytes, fileDetail, offset) {
+    if (fileDetail.extractor) {
+        sendStatusMessage(`Attempting to extract ${fileDetail.name} at pos ${offset}...`);
+        const fileData = fileDetail.extractor(bytes, offset);
+        const ext = fileDetail.extension.split(",")[0];
+        return new File([fileData], `extracted_at_0x${offset.toString(16)}.${ext}`, {
+            type: fileDetail.mime
+        });
+    }
+
+    throw new Error(`No extraction algorithm available for "${fileDetail.mime}" files`);
+}

src/core/lib/FuzzyMatch.mjs

@@ -0,0 +1,253 @@
+/**
+ * LICENSE
+ *
+ *   This software is dual-licensed to the public domain and under the following
+ *   license: you are granted a perpetual, irrevocable license to copy, modify,
+ *   publish, and distribute this file as you see fit.
+ *
+ * VERSION
+ *   0.1.0  (2016-03-28)  Initial release
+ *
+ * AUTHOR
+ *   Forrest Smith
+ *
+ * CONTRIBUTORS
+ *   J�rgen Tjern� - async helper
+ *   Anurag Awasthi - updated to 0.2.0
+ */
+
+export const DEFAULT_WEIGHTS = {
+    sequentialBonus: 15, // bonus for adjacent matches
+    separatorBonus: 30, // bonus if match occurs after a separator
+    camelBonus: 30, // bonus if match is uppercase and prev is lower
+    firstLetterBonus: 15, // bonus if the first letter is matched
+
+    leadingLetterPenalty: -5, // penalty applied for every letter in str before the first match
+    maxLeadingLetterPenalty: -15, // maximum penalty for leading letters
+    unmatchedLetterPenalty: -1
+};
+
+/**
+ * Does a fuzzy search to find pattern inside a string.
+ * @param {string} pattern        pattern to search for
+ * @param {string} str            string which is being searched
+ * @param {boolean} global        whether to search for all matches or just one
+ * @returns [boolean, number]       a boolean which tells if pattern was
+ *                                  found or not and a search score
+ */
+export function fuzzyMatch(pattern, str, global=false, weights=DEFAULT_WEIGHTS) {
+    const recursionCount = 0;
+    const recursionLimit = 10;
+    const matches = [];
+    const maxMatches = 256;
+
+    if (!global) {
+        return fuzzyMatchRecursive(
+            pattern,
+            str,
+            0 /* patternCurIndex */,
+            0 /* strCurrIndex */,
+            null /* srcMatches */,
+            matches,
+            maxMatches,
+            0 /* nextMatch */,
+            recursionCount,
+            recursionLimit,
+            weights
+        );
+    }
+
+    // Return all matches
+    let foundMatch = true,
+        score,
+        idxs,
+        strCurrIndex = 0;
+    const results = [];
+
+    while (foundMatch) {
+        [foundMatch, score, idxs] = fuzzyMatchRecursive(
+            pattern,
+            str,
+            0 /* patternCurIndex */,
+            strCurrIndex,
+            null /* srcMatches */,
+            matches,
+            maxMatches,
+            0 /* nextMatch */,
+            recursionCount,
+            recursionLimit,
+            weights
+        );
+        if (foundMatch) results.push([foundMatch, score, [...idxs]]);
+        strCurrIndex = idxs[idxs.length - 1] + 1;
+    }
+    return results;
+}
+
+/**
+ * Recursive helper function
+ */
+function fuzzyMatchRecursive(
+    pattern,
+    str,
+    patternCurIndex,
+    strCurrIndex,
+    srcMatches,
+    matches,
+    maxMatches,
+    nextMatch,
+    recursionCount,
+    recursionLimit,
+    weights
+) {
+    let outScore = 0;
+
+    // Return if recursion limit is reached.
+    if (++recursionCount >= recursionLimit) {
+        return [false, outScore, []];
+    }
+
+    // Return if we reached ends of strings.
+    if (patternCurIndex === pattern.length || strCurrIndex === str.length) {
+        return [false, outScore, []];
+    }
+
+    // Recursion params
+    let recursiveMatch = false;
+    let bestRecursiveMatches = [];
+    let bestRecursiveScore = 0;
+
+    // Loop through pattern and str looking for a match.
+    let firstMatch = true;
+    while (patternCurIndex < pattern.length && strCurrIndex < str.length) {
+        // Match found.
+        if (
+            pattern[patternCurIndex].toLowerCase() === str[strCurrIndex].toLowerCase()
+        ) {
+            if (nextMatch >= maxMatches) {
+                return [false, outScore, []];
+            }
+
+            if (firstMatch && srcMatches) {
+                matches = [...srcMatches];
+                firstMatch = false;
+            }
+
+            const [matched, recursiveScore, recursiveMatches] = fuzzyMatchRecursive(
+                pattern,
+                str,
+                patternCurIndex,
+                strCurrIndex + 1,
+                matches,
+                recursiveMatches,
+                maxMatches,
+                nextMatch,
+                recursionCount,
+                recursionLimit,
+                weights
+            );
+
+            if (matched) {
+                // Pick best recursive score.
+                if (!recursiveMatch || recursiveScore > bestRecursiveScore) {
+                    bestRecursiveMatches = [...recursiveMatches];
+                    bestRecursiveScore = recursiveScore;
+                }
+                recursiveMatch = true;
+            }
+
+            matches[nextMatch++] = strCurrIndex;
+            ++patternCurIndex;
+        }
+        ++strCurrIndex;
+    }
+
+    const matched = patternCurIndex === pattern.length;
+
+    if (matched) {
+        outScore = 100;
+
+        // Apply leading letter penalty
+        let penalty = weights.leadingLetterPenalty * matches[0];
+        penalty =
+            penalty < weights.maxLeadingLetterPenalty ?
+                weights.maxLeadingLetterPenalty :
+                penalty;
+        outScore += penalty;
+
+        // Apply unmatched penalty
+        const unmatched = str.length - nextMatch;
+        outScore += weights.unmatchedLetterPenalty * unmatched;
+
+        // Apply ordering bonuses
+        for (let i = 0; i < nextMatch; i++) {
+            const currIdx = matches[i];
+
+            if (i > 0) {
+                const prevIdx = matches[i - 1];
+                if (currIdx === prevIdx + 1) {
+                    outScore += weights.sequentialBonus;
+                }
+            }
+
+            // Check for bonuses based on neighbor character value.
+            if (currIdx > 0) {
+                // Camel case
+                const neighbor = str[currIdx - 1];
+                const curr = str[currIdx];
+                if (
+                    neighbor !== neighbor.toUpperCase() &&
+                    curr !== curr.toLowerCase()
+                ) {
+                    outScore += weights.camelBonus;
+                }
+                const isNeighbourSeparator = neighbor === "_" || neighbor === " ";
+                if (isNeighbourSeparator) {
+                    outScore += weights.separatorBonus;
+                }
+            } else {
+                // First letter
+                outScore += weights.firstLetterBonus;
+            }
+        }
+
+        // Return best result
+        if (recursiveMatch && (!matched || bestRecursiveScore > outScore)) {
+            // Recursive score is better than "this"
+            matches = bestRecursiveMatches;
+            outScore = bestRecursiveScore;
+            return [true, outScore, matches];
+        } else if (matched) {
+            // "this" score is better than recursive
+            return [true, outScore, matches];
+        } else {
+            return [false, outScore, matches];
+        }
+    }
+    return [false, outScore, matches];
+}
+
+/**
+ * Turns a list of match indexes into a list of match ranges
+ *
+ * @author n1474335 [n1474335@gmail.com]
+ * @param [number] matches
+ * @returns [[number]]
+ */
+export function calcMatchRanges(matches) {
+    const ranges = [];
+    let start = matches[0],
+        curr = start;
+
+    matches.forEach(m => {
+        if (m === curr || m === curr + 1) curr = m;
+        else {
+            ranges.push([start, curr - start + 1]);
+            start = m;
+            curr = m;
+        }
+    });
+
+    ranges.push([start, curr - start + 1]);
+    return ranges;
+}

src/core/lib/Hash.mjs

@@ -7,8 +7,8 @@
  * @license Apache-2.0
  */
 
-import Utils from "../Utils";
-import CryptoApi from "crypto-api/src/crypto-api";
+import Utils from "../Utils.mjs";
+import CryptoApi from "crypto-api/src/crypto-api.mjs";
 
 
 /**

src/core/lib/Hex.mjs

@@ -6,13 +6,14 @@
  * @license Apache-2.0
  */
 
-import Utils from "../Utils";
+import Utils from "../Utils.mjs";
+import OperationError from "../errors/OperationError.mjs";
 
 
 /**
  * Convert a byte array into a hex string.
  *
- * @param {Uint8Array|byteArray} data
+ * @param {byteArray|Uint8Array|ArrayBuffer} data
  * @param {string} [delim=" "]
  * @param {number} [padding=2]
  * @returns {string}
@@ -23,31 +24,46 @@ import Utils from "../Utils";
  *
  * // returns "0a:14:1e"
  * toHex([10,20,30], ":");
+ *
+ * // returns "0x0a,0x14,0x1e"
+ * toHex([10,20,30], "0x", 2, ",")
  */
-export function toHex(data, delim=" ", padding=2) {
+export function toHex(data, delim=" ", padding=2, extraDelim="", lineSize=0) {
     if (!data) return "";
+    if (data instanceof ArrayBuffer) data = new Uint8Array(data);
 
     let output = "";
+    const prepend = (delim === "0x" || delim === "\\x");
 
     for (let i = 0; i < data.length; i++) {
-        output += data[i].toString(16).padStart(padding, "0") + delim;
+        const hex = data[i].toString(16).padStart(padding, "0");
+        output += prepend ? delim + hex : hex + delim;
+
+        if (extraDelim) {
+            output += extraDelim;
+        }
+        // Add LF after each lineSize amount of bytes but not at the end
+        if ((i !== data.length - 1) && ((i + 1) % lineSize === 0)) {
+            output += "\n";
+        }
     }
 
-    // Add \x or 0x to beginning
-    if (delim === "0x") output = "0x" + output;
-    if (delim === "\\x") output = "\\x" + output;
-
-    if (delim.length)
-        return output.slice(0, -delim.length);
-    else
+    // Remove the extraDelim at the end (if there is one)
+    // and remove the delim at the end, but if it's prepended there's nothing to remove
+    const rTruncLen = extraDelim.length + (prepend ? 0 : delim.length);
+    if (rTruncLen) {
+        // If rTruncLen === 0 then output.slice(0,0) will be returned, which is nothing
+        return output.slice(0, -rTruncLen);
+    } else {
         return output;
+    }
 }
 
 
 /**
  * Convert a byte array into a hex string as efficiently as possible with no options.
  *
- * @param {byteArray} data
+ * @param {byteArray|Uint8Array|ArrayBuffer} data
  * @returns {string}
  *
  * @example
@@ -56,6 +72,7 @@ export function toHex(data, delim=" ", padding=2) {
  */
 export function toHexFast(data) {
     if (!data) return "";
+    if (data instanceof ArrayBuffer) data = new Uint8Array(data);
 
     const output = [];
 
@@ -84,14 +101,21 @@ export function toHexFast(data) {
  * fromHex("0a:14:1e", "Colon");
  */
 export function fromHex(data, delim="Auto", byteLen=2) {
+    if (byteLen < 1 || Math.round(byteLen) !== byteLen)
+        throw new OperationError("Byte length must be a positive integer");
+
     if (delim !== "None") {
-        const delimRegex = delim === "Auto" ? /[^a-f\d]/gi : Utils.regexRep(delim);
-        data = data.replace(delimRegex, "");
+        const delimRegex = delim === "Auto" ? /[^a-f\d]|0x/gi : Utils.regexRep(delim);
+        data = data.split(delimRegex);
+    } else {
+        data = [data];
     }
 
     const output = [];
-    for (let i = 0; i < data.length; i += byteLen) {
-        output.push(parseInt(data.substr(i, byteLen), 16));
+    for (let i = 0; i < data.length; i++) {
+        for (let j = 0; j < data[i].length; j += byteLen) {
+            output.push(parseInt(data[i].substr(j, byteLen), 16));
+        }
     }
     return output;
 }
@@ -100,7 +124,7 @@ export function fromHex(data, delim="Auto", byteLen=2) {
 /**
  * To Hexadecimal delimiters.
  */
-export const TO_HEX_DELIM_OPTIONS = ["Space", "Comma", "Semi-colon", "Colon", "Line feed", "CRLF", "0x", "\\x", "None"];
+export const TO_HEX_DELIM_OPTIONS = ["Space", "Percent", "Comma", "Semi-colon", "Colon", "Line feed", "CRLF", "0x", "0x with comma", "\\x", "None"];
 
 
 /**

src/core/lib/IP.mjs

@@ -8,8 +8,8 @@
  * @license Apache-2.0
  */
 
-import Utils from "../Utils";
-import OperationError from "../errors/OperationError";
+import Utils from "../Utils.mjs";
+import OperationError from "../errors/OperationError.mjs";
 
 /**
  * Parses an IPv4 CIDR range (e.g. 192.168.0.0/24) and displays information about it.
@@ -26,7 +26,7 @@ export function ipv4CidrRange(cidr, includeNetworkInfo, enumerateAddresses, allo
     let output = "";
 
     if (cidrRange < 0 || cidrRange > 31) {
-        return "IPv4 CIDR must be less than 32";
+        throw new OperationError("IPv4 CIDR must be less than 32");
     }
 
     const mask = ~(0xFFFFFFFF >>> cidrRange),
@@ -64,7 +64,7 @@ export function ipv6CidrRange(cidr, includeNetworkInfo) {
         cidrRange = parseInt(cidr[cidr.length-1], 10);
 
     if (cidrRange < 0 || cidrRange > 127) {
-        return "IPv6 CIDR must be less than 128";
+        throw new OperationError("IPv6 CIDR must be less than 128");
     }
 
     const ip1 = new Array(8),
@@ -211,7 +211,7 @@ export function ipv4ListedRange(match, includeNetworkInfo, enumerateAddresses, a
         const network = strToIpv4(ipv4CidrList[i].split("/")[0]);
         const cidrRange = parseInt(ipv4CidrList[i].split("/")[1], 10);
         if (cidrRange < 0 || cidrRange > 31) {
-            return "IPv4 CIDR must be less than 32";
+            throw new OperationError("IPv4 CIDR must be less than 32");
         }
         const mask = ~(0xFFFFFFFF >>> cidrRange),
             cidrIp1 = network & mask,
@@ -241,7 +241,7 @@ export function ipv6ListedRange(match, includeNetworkInfo) {
     ipv6List = ipv6List.filter(function(str) {
         return str.trim();
     });
-    for (let i =0; i < ipv6List.length; i++){
+    for (let i =0; i < ipv6List.length; i++) {
         ipv6List[i] = ipv6List[i].trim();
     }
     const ipv6CidrList = ipv6List.filter(function(a) {
@@ -254,7 +254,7 @@ export function ipv6ListedRange(match, includeNetworkInfo) {
         const cidrRange = parseInt(ipv6CidrList[i].split("/")[1], 10);
 
         if (cidrRange < 0 || cidrRange > 127) {
-            return "IPv6 CIDR must be less than 128";
+            throw new OperationError("IPv6 CIDR must be less than 128");
         }
 
         const cidrIp1 = new Array(8),
@@ -502,8 +502,8 @@ export function ipv6Compare(a, b) {
     const a_ = strToIpv6(a),
         b_ = strToIpv6(b);
 
-    for (let i = 0; i < a_.length; i++){
-        if (a_[i] !== b_[i]){
+    for (let i = 0; i < a_.length; i++) {
+        if (a_[i] !== b_[i]) {
             return a_[i] - b_[i];
         }
     }

src/core/lib/ImageManipulation.mjs

@@ -0,0 +1,251 @@
+/**
+ * Image manipulation resources
+ *
+ * @author j433866 [j433866@gmail.com]
+ * @copyright Crown Copyright 2019
+ * @license Apache-2.0
+ */
+
+import OperationError from "../errors/OperationError.mjs";
+
+/**
+ * Gaussian blurs an image.
+ *
+ * @param {jimp} input
+ * @param {number} radius
+ * @param {boolean} fast
+ * @returns {jimp}
+ */
+export function gaussianBlur (input, radius) {
+    try {
+        // From http://blog.ivank.net/fastest-gaussian-blur.html
+        const boxes = boxesForGauss(radius, 3);
+        for (let i = 0; i < 3; i++) {
+            input = boxBlur(input, (boxes[i] - 1) / 2);
+        }
+    } catch (err) {
+        throw new OperationError(`Error blurring image. (${err})`);
+    }
+
+    return input;
+}
+
+/**
+ *
+ * @param {number} radius
+ * @param {number} numBoxes
+ * @returns {Array}
+ */
+function boxesForGauss(radius, numBoxes) {
+    const idealWidth = Math.sqrt((12 * radius * radius / numBoxes) + 1);
+
+    let wl = Math.floor(idealWidth);
+
+    if (wl % 2 === 0) {
+        wl--;
+    }
+
+    const wu = wl + 2;
+
+    const mIdeal = (12 * radius * radius - numBoxes * wl * wl - 4 * numBoxes * wl - 3 * numBoxes) / (-4 * wl - 4);
+    const m = Math.round(mIdeal);
+
+    const sizes = [];
+    for (let i = 0; i < numBoxes; i++) {
+        sizes.push(i < m ? wl : wu);
+    }
+    return sizes;
+}
+
+/**
+ * Applies a box blur effect to the image
+ *
+ * @param {jimp} source
+ * @param {number} radius
+ * @returns {jimp}
+ */
+function boxBlur (source, radius) {
+    const width = source.bitmap.width;
+    const height = source.bitmap.height;
+    let output = source.clone();
+    output = boxBlurH(source, output, width, height, radius);
+    source = boxBlurV(output, source, width, height, radius);
+
+    return source;
+}
+
+/**
+ * Applies the horizontal blur
+ *
+ * @param {jimp} source
+ * @param {jimp} output
+ * @param {number} width
+ * @param {number} height
+ * @param {number} radius
+ * @returns {jimp}
+ */
+function boxBlurH (source, output, width, height, radius) {
+    const iarr = 1 / (radius + radius + 1);
+    for (let i = 0; i < height; i++) {
+        let ti = 0,
+            li = ti,
+            ri = ti + radius;
+        const idx = source.getPixelIndex(ti, i);
+        const firstValRed = source.bitmap.data[idx],
+            firstValGreen = source.bitmap.data[idx + 1],
+            firstValBlue = source.bitmap.data[idx + 2],
+            firstValAlpha = source.bitmap.data[idx + 3];
+
+        const lastIdx = source.getPixelIndex(width - 1, i),
+            lastValRed = source.bitmap.data[lastIdx],
+            lastValGreen = source.bitmap.data[lastIdx + 1],
+            lastValBlue = source.bitmap.data[lastIdx + 2],
+            lastValAlpha = source.bitmap.data[lastIdx + 3];
+
+        let red = (radius + 1) * firstValRed;
+        let green = (radius + 1) * firstValGreen;
+        let blue = (radius + 1) * firstValBlue;
+        let alpha = (radius + 1) * firstValAlpha;
+
+        for (let j = 0; j < radius; j++) {
+            const jIdx = source.getPixelIndex(ti + j, i);
+            red += source.bitmap.data[jIdx];
+            green += source.bitmap.data[jIdx + 1];
+            blue += source.bitmap.data[jIdx + 2];
+            alpha += source.bitmap.data[jIdx + 3];
+        }
+
+        for (let j = 0; j <= radius; j++) {
+            const jIdx = source.getPixelIndex(ri++, i);
+            red += source.bitmap.data[jIdx] - firstValRed;
+            green += source.bitmap.data[jIdx + 1] - firstValGreen;
+            blue += source.bitmap.data[jIdx + 2] - firstValBlue;
+            alpha += source.bitmap.data[jIdx + 3] - firstValAlpha;
+
+            const tiIdx = source.getPixelIndex(ti++, i);
+            output.bitmap.data[tiIdx] = Math.round(red * iarr);
+            output.bitmap.data[tiIdx + 1] = Math.round(green * iarr);
+            output.bitmap.data[tiIdx + 2] = Math.round(blue * iarr);
+            output.bitmap.data[tiIdx + 3] = Math.round(alpha * iarr);
+        }
+
+        for (let j = radius + 1; j < width - radius; j++) {
+            const riIdx = source.getPixelIndex(ri++, i);
+            const liIdx = source.getPixelIndex(li++, i);
+            red += source.bitmap.data[riIdx] - source.bitmap.data[liIdx];
+            green += source.bitmap.data[riIdx + 1] - source.bitmap.data[liIdx + 1];
+            blue += source.bitmap.data[riIdx + 2] - source.bitmap.data[liIdx + 2];
+            alpha += source.bitmap.data[riIdx + 3] - source.bitmap.data[liIdx + 3];
+
+            const tiIdx = source.getPixelIndex(ti++, i);
+            output.bitmap.data[tiIdx] = Math.round(red * iarr);
+            output.bitmap.data[tiIdx + 1] = Math.round(green * iarr);
+            output.bitmap.data[tiIdx + 2] = Math.round(blue * iarr);
+            output.bitmap.data[tiIdx + 3] = Math.round(alpha * iarr);
+        }
+
+        for (let j = width - radius; j < width; j++) {
+            const liIdx = source.getPixelIndex(li++, i);
+            red += lastValRed - source.bitmap.data[liIdx];
+            green += lastValGreen - source.bitmap.data[liIdx + 1];
+            blue += lastValBlue - source.bitmap.data[liIdx + 2];
+            alpha += lastValAlpha - source.bitmap.data[liIdx + 3];
+
+            const tiIdx = source.getPixelIndex(ti++, i);
+            output.bitmap.data[tiIdx] = Math.round(red * iarr);
+            output.bitmap.data[tiIdx + 1] = Math.round(green * iarr);
+            output.bitmap.data[tiIdx + 2] = Math.round(blue * iarr);
+            output.bitmap.data[tiIdx + 3] = Math.round(alpha * iarr);
+        }
+    }
+    return output;
+}
+
+/**
+ * Applies the vertical blur
+ *
+ * @param {jimp} source
+ * @param {jimp} output
+ * @param {number} width
+ * @param {number} height
+ * @param {number} radius
+ * @returns {jimp}
+ */
+function boxBlurV (source, output, width, height, radius) {
+    const iarr = 1 / (radius + radius + 1);
+    for (let i = 0; i < width; i++) {
+        let ti = 0,
+            li = ti,
+            ri = ti + radius;
+
+        const idx = source.getPixelIndex(i, ti);
+
+        const firstValRed = source.bitmap.data[idx],
+            firstValGreen = source.bitmap.data[idx + 1],
+            firstValBlue = source.bitmap.data[idx + 2],
+            firstValAlpha = source.bitmap.data[idx + 3];
+
+        const lastIdx = source.getPixelIndex(i, height - 1),
+            lastValRed = source.bitmap.data[lastIdx],
+            lastValGreen = source.bitmap.data[lastIdx + 1],
+            lastValBlue = source.bitmap.data[lastIdx + 2],
+            lastValAlpha = source.bitmap.data[lastIdx + 3];
+
+        let red = (radius + 1) * firstValRed;
+        let green = (radius + 1) * firstValGreen;
+        let blue = (radius + 1) * firstValBlue;
+        let alpha = (radius + 1) * firstValAlpha;
+
+        for (let j = 0; j < radius; j++) {
+            const jIdx = source.getPixelIndex(i, ti + j);
+            red += source.bitmap.data[jIdx];
+            green += source.bitmap.data[jIdx + 1];
+            blue += source.bitmap.data[jIdx + 2];
+            alpha += source.bitmap.data[jIdx + 3];
+        }
+
+        for (let j = 0; j <= radius; j++) {
+            const riIdx = source.getPixelIndex(i, ri++);
+            red += source.bitmap.data[riIdx] - firstValRed;
+            green += source.bitmap.data[riIdx + 1] - firstValGreen;
+            blue += source.bitmap.data[riIdx + 2] - firstValBlue;
+            alpha += source.bitmap.data[riIdx + 3] - firstValAlpha;
+
+            const tiIdx = source.getPixelIndex(i, ti++);
+            output.bitmap.data[tiIdx] = Math.round(red * iarr);
+            output.bitmap.data[tiIdx + 1] = Math.round(green * iarr);
+            output.bitmap.data[tiIdx + 2] = Math.round(blue * iarr);
+            output.bitmap.data[tiIdx + 3] = Math.round(alpha * iarr);
+        }
+
+        for (let j = radius + 1; j < height - radius; j++) {
+            const riIdx = source.getPixelIndex(i, ri++);
+            const liIdx = source.getPixelIndex(i, li++);
+            red += source.bitmap.data[riIdx] - source.bitmap.data[liIdx];
+            green += source.bitmap.data[riIdx + 1] - source.bitmap.data[liIdx + 1];
+            blue += source.bitmap.data[riIdx + 2] - source.bitmap.data[liIdx + 2];
+            alpha += source.bitmap.data[riIdx + 3] - source.bitmap.data[liIdx + 3];
+
+            const tiIdx = source.getPixelIndex(i, ti++);
+            output.bitmap.data[tiIdx] = Math.round(red * iarr);
+            output.bitmap.data[tiIdx + 1] = Math.round(green * iarr);
+            output.bitmap.data[tiIdx + 2] = Math.round(blue * iarr);
+            output.bitmap.data[tiIdx + 3] = Math.round(alpha * iarr);
+        }
+
+        for (let j = height - radius; j < height; j++) {
+            const liIdx = source.getPixelIndex(i, li++);
+            red += lastValRed - source.bitmap.data[liIdx];
+            green += lastValGreen - source.bitmap.data[liIdx + 1];
+            blue += lastValBlue - source.bitmap.data[liIdx + 2];
+            alpha += lastValAlpha - source.bitmap.data[liIdx + 3];
+
+            const tiIdx = source.getPixelIndex(i, ti++);
+            output.bitmap.data[tiIdx] = Math.round(red * iarr);
+            output.bitmap.data[tiIdx + 1] = Math.round(green * iarr);
+            output.bitmap.data[tiIdx + 2] = Math.round(blue * iarr);
+            output.bitmap.data[tiIdx + 3] = Math.round(alpha * iarr);
+        }
+    }
+    return output;
+}

src/core/lib/JWT.mjs

@@ -0,0 +1,24 @@
+/**
+ * JWT resources
+ *
+ * @author mt3571 [mt3571@protonmail.com]
+ * @copyright Crown Copyright 2020
+ * @license Apache-2.0
+ */
+
+
+/**
+ * List of the JWT algorithms that can be used
+ */
+export const JWT_ALGORITHMS = [
+    "HS256",
+    "HS384",
+    "HS512",
+    "RS256",
+    "RS384",
+    "RS512",
+    "ES256",
+    "ES384",
+    "ES512",
+    "None"
+];

src/core/lib/LS47.mjs

@@ -0,0 +1,244 @@
+/**
+ * @author n1073645 [n1073645@gmail.com]
+ * @copyright Crown Copyright 2020
+ * @license Apache-2.0
+ */
+
+import OperationError from "../errors/OperationError.mjs";
+
+const letters = "_abcdefghijklmnopqrstuvwxyz.0123456789,-+*/:?!'()";
+const tiles = [];
+
+/**
+ * Initialises the tiles with values and positions.
+ */
+export function initTiles() {
+    for (let i = 0; i < 49; i++)
+        tiles.push([letters.charAt(i), [Math.floor(i/7), i % 7]]);
+}
+
+/**
+ * Rotates the key "down".
+ *
+ * @param {string} key
+ * @param {number} col
+ * @param {number} n
+ * @returns {string}
+ */
+function rotateDown(key, col, n) {
+    const lines = [];
+    for (let i = 0; i < 7; i++)
+        lines.push(key.slice(i*7, (i + 1) * 7));
+    const lefts = [];
+    let mids = [];
+    const rights = [];
+    lines.forEach((element) => {
+        lefts.push(element.slice(0, col));
+        mids.push(element.charAt(col));
+        rights.push(element.slice(col+1));
+    });
+    n = (7 - n % 7) % 7;
+    mids = mids.slice(n).concat(mids.slice(0, n));
+    let result = "";
+    for (let i = 0; i < 7; i++)
+        result += lefts[i] + mids[i] + rights[i];
+    return result;
+}
+
+/**
+ * Rotates the key "right".
+ *
+ * @param {string} key
+ * @param {number} row
+ * @param {number} n
+ * @returns {string}
+ */
+function rotateRight(key, row, n) {
+    const mid = key.slice(row * 7, (row + 1) * 7);
+    n = (7 - n % 7) % 7;
+    return key.slice(0, 7 * row) + mid.slice(n) + mid.slice(0, n) + key.slice(7 * (row + 1));
+}
+
+/**
+ * Finds the position of a letter in the tiles.
+ *
+ * @param {string} letter
+ * @returns {string}
+ */
+function findIx(letter) {
+    for (let i = 0; i < tiles.length; i++)
+        if (tiles[i][0] === letter)
+            return tiles[i][1];
+    throw new OperationError("Letter " + letter + " is not included in LS47");
+}
+
+/**
+ * Derives key from the input password.
+ *
+ * @param {string} password
+ * @returns {string}
+ */
+export function deriveKey(password) {
+    let i = 0;
+    let k = letters;
+    for (const c of password) {
+        const [row, col] = findIx(c);
+        k = rotateDown(rotateRight(k, i, col), i, row);
+        i = (i + 1) % 7;
+    }
+    return k;
+}
+
+/**
+ * Checks the key is a valid key.
+ *
+ * @param {string} key
+ */
+function checkKey(key) {
+    if (key.length !== letters.length)
+        throw new OperationError("Wrong key size");
+    const counts = new Array();
+    for (let i = 0; i < letters.length; i++)
+        counts[letters.charAt(i)] = 0;
+    for (const elem of letters) {
+        if (letters.indexOf(elem) === -1)
+            throw new OperationError("Letter " + elem + " not in LS47");
+        counts[elem]++;
+        if (counts[elem] > 1)
+            throw new OperationError("Letter duplicated in the key");
+    }
+}
+
+/**
+ * Finds the position of a letter in they key.
+ *
+ * @param {letter} key
+ * @param {string} letter
+ * @returns {object}
+ */
+function findPos (key, letter) {
+    const index = key.indexOf(letter);
+    if (index >= 0 && index < 49)
+        return [Math.floor(index/7), index%7];
+    throw new OperationError("Letter " + letter + " is not in the key");
+}
+
+/**
+ * Returns the character at the position on the tiles.
+ *
+ * @param {string} key
+ * @param {object} coord
+ * @returns {string}
+ */
+function findAtPos(key, coord) {
+    return key.charAt(coord[1] + (coord[0] * 7));
+}
+
+/**
+ * Returns new position by adding two positions.
+ *
+ * @param {object} a
+ * @param {object} b
+ * @returns {object}
+ */
+function addPos(a, b) {
+    return [(a[0] + b[0]) % 7, (a[1] + b[1]) % 7];
+}
+
+/**
+ * Returns new position by subtracting two positions.
+ * Note: We have to manually do the remainder division, since JS does not
+ * operate correctly on negative numbers (e.g. -3 % 4 = -3 when it should be 1).
+ *
+ * @param {object} a
+ * @param {object} b
+ * @returns {object}
+ */
+function subPos(a, b) {
+    const asub = a[0] - b[0];
+    const bsub = a[1] - b[1];
+    return [asub - (Math.floor(asub/7) * 7), bsub - (Math.floor(bsub/7) * 7)];
+}
+
+/**
+ * Encrypts the plaintext string.
+ *
+ * @param {string} key
+ * @param {string} plaintext
+ * @returns {string}
+ */
+function encrypt(key, plaintext) {
+    checkKey(key);
+    let mp = [0, 0];
+    let ciphertext = "";
+    for (const p of plaintext) {
+        const pp = findPos(key, p);
+        const mix = findIx(findAtPos(key, mp));
+        let cp = addPos(pp, mix);
+        const c = findAtPos(key, cp);
+        ciphertext += c;
+        key = rotateRight(key, pp[0], 1);
+        cp = findPos(key, c);
+        key = rotateDown(key, cp[1], 1);
+        mp = addPos(mp, findIx(c));
+    }
+    return ciphertext;
+}
+
+/**
+ * Decrypts the ciphertext string.
+ *
+ * @param {string} key
+ * @param {string} ciphertext
+ * @returns {string}
+ */
+function decrypt(key, ciphertext) {
+    checkKey(key);
+    let mp = [0, 0];
+    let plaintext = "";
+    for (const c of ciphertext) {
+        let cp = findPos(key, c);
+        const mix = findIx(findAtPos(key, mp));
+        const pp = subPos(cp, mix);
+        const p = findAtPos(key, pp);
+        plaintext += p;
+        key = rotateRight(key, pp[0], 1);
+        cp = findPos(key, c);
+        key = rotateDown(key, cp[1], 1);
+        mp = addPos(mp, findIx(c));
+    }
+    return plaintext;
+}
+
+/**
+ * Adds padding to the input.
+ *
+ * @param {string} key
+ * @param {string} plaintext
+ * @param {string} signature
+ * @param {number} paddingSize
+ * @returns {string}
+ */
+export function encryptPad(key, plaintext, signature, paddingSize) {
+    initTiles();
+    checkKey(key);
+    let padding = "";
+    for (let i = 0; i < paddingSize; i++) {
+        padding += letters.charAt(Math.floor(Math.random() * letters.length));
+    }
+    return encrypt(key, padding+plaintext+"---"+signature);
+}
+
+/**
+ * Removes padding from the ouput.
+ *
+ * @param {string} key
+ * @param {string} ciphertext
+ * @param {number} paddingSize
+ * @returns {string}
+ */
+export function decryptPad(key, ciphertext, paddingSize) {
+    initTiles();
+    checkKey(key);
+    return decrypt(key, ciphertext).slice(paddingSize);
+}

src/core/lib/LZNT1.mjs

@@ -0,0 +1,88 @@
+/**
+ *
+ * LZNT1 Decompress.
+ *
+ * @author 0xThiebaut [thiebaut.dev]
+ * @copyright Crown Copyright 2023
+ * @license Apache-2.0
+ *
+ * https://github.com/Velocidex/go-ntfs/blob/master/parser%2Flznt1.go
+ */
+
+import Utils from "../Utils.mjs";
+import OperationError from "../errors/OperationError.mjs";
+
+const COMPRESSED_MASK = 1 << 15,
+    SIZE_MASK = (1 << 12) - 1;
+
+/**
+ * @param {number} offset
+ * @returns {number}
+ */
+function getDisplacement(offset) {
+    let result = 0;
+    while (offset >= 0x10) {
+        offset >>= 1;
+        result += 1;
+    }
+    return result;
+}
+
+/**
+ * @param {byteArray} compressed
+ * @returns {byteArray}
+ */
+export function decompress(compressed) {
+    const decompressed = Array();
+    let coffset = 0;
+
+    while (coffset + 2 <= compressed.length) {
+        const doffset = decompressed.length;
+
+        const blockHeader = Utils.byteArrayToInt(compressed.slice(coffset, coffset + 2), "little");
+        coffset += 2;
+
+        const size = blockHeader & SIZE_MASK;
+        const blockEnd = coffset + size + 1;
+
+        if (size === 0) {
+            break;
+        } else if (compressed.length < coffset + size) {
+            throw new OperationError("Malformed LZNT1 stream: Block too small! Has the stream been truncated?");
+        }
+
+        if ((blockHeader & COMPRESSED_MASK) !== 0) {
+            while (coffset < blockEnd) {
+                let header = compressed[coffset++];
+
+                for (let i = 0; i < 8 && coffset < blockEnd; i++) {
+                    if ((header & 1) === 0) {
+                        decompressed.push(compressed[coffset++]);
+                    } else {
+                        const pointer = Utils.byteArrayToInt(compressed.slice(coffset, coffset + 2), "little");
+                        coffset += 2;
+
+                        const displacement = getDisplacement(decompressed.length - doffset - 1);
+                        const symbolOffset = (pointer >> (12 - displacement)) + 1;
+                        const symbolLength = (pointer & (0xFFF >> displacement)) + 2;
+                        const shiftOffset = decompressed.length - symbolOffset;
+
+                        for (let shiftDelta = 0; shiftDelta < symbolLength + 1; shiftDelta++) {
+                            const shift = shiftOffset + shiftDelta;
+                            if (shift < 0 || decompressed.length <= shift) {
+                                throw new OperationError("Malformed LZNT1 stream: Invalid shift!");
+                            }
+                            decompressed.push(decompressed[shift]);
+                        }
+                    }
+                    header >>= 1;
+                }
+            }
+        } else {
+            decompressed.push(...compressed.slice(coffset, coffset + size + 1));
+            coffset += size + 1;
+        }
+    }
+
+    return decompressed;
+}

src/core/lib/LZString.mjs

@@ -0,0 +1,21 @@
+/**
+ * lz-string exports.
+ *
+ * @author crespyl [peter@crespyl.net]
+ * @copyright Peter Jacobs 2021
+ * @license Apache-2.0
+ */
+
+import LZString from "lz-string";
+
+export const COMPRESSION_OUTPUT_FORMATS = ["default", "UTF16", "Base64"];
+export const COMPRESSION_FUNCTIONS = {
+    "default": LZString.compress,
+    "UTF16":   LZString.compressToUTF16,
+    "Base64":  LZString.compressToBase64,
+};
+export const DECOMPRESSION_FUNCTIONS = {
+    "default": LZString.decompress,
+    "UTF16":   LZString.decompressFromUTF16,
+    "Base64":  LZString.decompressFromBase64,
+};

src/core/lib/LoremIpsum.mjs

@@ -85,7 +85,7 @@ function getWords(length=3) {
     const words = [];
     let word;
     let previousWord;
-    while (words.length < length){
+    while (words.length < length) {
         do {
             word = wordList[Math.floor(Math.random() * wordList.length)];
         } while (previousWord === word);

src/core/lib/Lorenz.mjs

@@ -0,0 +1,156 @@
+/**
+ * Resources required by the Lorenz SZ40/42 and Colossus
+ *
+ * @author VirtualColossus [martin@virtualcolossus.co.uk]
+ * @copyright Crown Copyright 2019
+ * @license Apache-2.0
+ */
+
+export const SWITCHES = [
+    {name: "Up (.)", value: "."},
+    {name: "Centre", value: ""},
+    {name: "Down (x)", value: "x"}
+];
+
+export const VALID_ITA2 = "ABCDEFGHIJKLMNOPQRSTUVWXYZ34589+-./";
+
+export const ITA2_TABLE = {
+    "A": "11000",
+    "B": "10011",
+    "C": "01110",
+    "D": "10010",
+    "E": "10000",
+    "F": "10110",
+    "G": "01011",
+    "H": "00101",
+    "I": "01100",
+    "J": "11010",
+    "K": "11110",
+    "L": "01001",
+    "M": "00111",
+    "N": "00110",
+    "O": "00011",
+    "P": "01101",
+    "Q": "11101",
+    "R": "01010",
+    "S": "10100",
+    "T": "00001",
+    "U": "11100",
+    "V": "01111",
+    "W": "11001",
+    "X": "10111",
+    "Y": "10101",
+    "Z": "10001",
+    "3": "00010",
+    "4": "01000",
+    "9": "00100",
+    "/": "00000",
+    " ": "00100",
+    ".": "00100",
+    "8": "11111",
+    "5": "11011",
+    "-": "11111",
+    "+": "11011"
+};
+
+export const ROTOR_SIZES = {
+    S1: 43,
+    S2: 47,
+    S3: 51,
+    S4: 53,
+    S5: 59,
+    M37: 37,
+    M61: 61,
+    X1: 41,
+    X2: 31,
+    X3: 29,
+    X4: 26,
+    X5: 23
+};
+
+/**
+ * Initial rotor patterns
+ */
+export const INIT_PATTERNS = {
+    "No Pattern": {
+        "X": {
+            1: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
+            2: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
+            3: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
+            4: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
+            5: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
+        },
+        "S": {
+            1: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
+            2: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
+            3: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
+            4: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
+            5: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
+        },
+        "M": {
+            1: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
+            2: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
+        }
+
+    },
+    "KH Pattern": {
+        "X": {
+            1: [0, 1, 0, 0, 0, 1, 1, 1, 0, 1, 0, 1, 1, 1, 1, 0, 1, 0, 0, 0, 1, 0, 1, 0, 0, 1, 1, 1, 0, 0, 0, 0, 1, 1, 0, 1, 1, 1, 1, 0, 0],
+            2: [1, 0, 0, 1, 1, 1, 0, 0, 0, 1, 0, 1, 1, 1, 1, 0, 0, 1, 1, 0, 0, 1, 0, 0, 1, 1, 0, 1, 1, 0, 0],
+            3: [0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 1, 0, 0, 0, 1, 1, 0, 0, 0, 1, 1, 0, 0, 1, 1, 0, 1, 1, 0],
+            4: [1, 1, 0, 0, 1, 0, 0, 1, 1, 1, 1, 0, 0, 1, 1, 0, 1, 1, 1, 0, 0, 0, 0, 1, 0, 0],
+            5: [1, 1, 0, 0, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1, 0, 1, 0, 0, 1, 0, 1, 0, 0]
+        },
+        "S": {
+            1: [0, 1, 0, 0, 0, 1, 1, 0, 1, 0, 1, 0, 0, 1, 1, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 0, 1, 0, 1, 1, 0, 1],
+            2: [0, 1, 1, 0, 1, 0, 1, 1, 1, 0, 0, 1, 0, 1, 0, 1, 0, 0, 1, 0, 1, 1, 0, 1, 0, 1, 1, 1, 0, 1, 0, 0, 0, 0, 1, 0, 1, 1, 0, 1, 0, 1, 0, 1, 0, 0, 1],
+            3: [0, 1, 0, 1, 0, 1, 0, 0, 1, 1, 1, 0, 0, 0, 0, 1, 0, 1, 0, 1, 1, 0, 1, 0, 1, 0, 1, 0, 0, 1, 1, 1, 0, 1, 0, 1, 0, 0, 1, 0, 1, 0, 1, 1, 0, 0, 1, 0, 1, 0, 1],
+            4: [0, 1, 1, 0, 0, 0, 1, 1, 1, 1, 1, 0, 1, 0, 1, 0, 1, 1, 0, 0, 0, 1, 0, 1, 1, 0, 1, 0, 1, 0, 0, 1, 0, 1, 0, 1, 1, 0, 1, 0, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0],
+            5: [1, 1, 0, 0, 0, 1, 1, 0, 1, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 1, 0, 0, 1, 1, 1, 1, 0, 1, 0, 1, 0, 0, 0, 1, 1, 0, 1, 0, 0, 1, 0]
+        },
+        "M":  {
+            1: [0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 1, 0, 1, 1, 0, 0, 0, 0, 1, 1, 1, 0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 1, 0, 0, 0],
+            2: [1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 0, 0, 1, 0, 1, 0, 1, 0, 0, 0, 1, 0, 1, 0, 1, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0]
+        }
+    },
+    "ZMUG Pattern":  {
+        "X": {
+            1: [0, 1, 1, 0, 1, 1, 0, 0, 0, 1, 1, 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 1, 1, 1, 0, 0, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 0, 0, 1, 1, 0],
+            2: [1, 1, 0, 1, 1, 0, 0, 0, 0, 1, 1, 1, 0, 1, 1, 1, 1, 0, 1, 0, 0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 0],
+            3: [0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 1, 1, 0, 0, 0, 1, 1, 1, 0, 0, 0, 1, 1, 0, 1, 1, 1, 1, 0],
+            4: [1, 0, 1, 0, 1, 0, 0, 1, 1, 0, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 1, 0, 0, 1, 0, 1],
+            5: [0, 1, 0, 0, 1, 1, 1, 1, 0, 0, 0, 1, 0, 1, 1, 1, 0, 0, 0, 0, 1, 0, 1]
+        },
+        "S": {
+            1: [1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 0, 1, 1, 1, 0, 0, 1, 1, 0, 0, 0, 1, 1, 0, 0, 0, 1, 1, 1, 1, 0, 0, 1, 1, 1, 0, 0, 1, 1, 1, 0, 0, 0],
+            2: [0, 0, 0, 1, 0, 0, 0, 1, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1, 1, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 1, 1, 0, 0, 1, 1, 1, 0, 0, 1, 1, 1, 0, 0, 1, 0, 1, 1],
+            3: [0, 1, 0, 0, 1, 1, 0, 0, 1, 1, 1, 0, 0, 1, 1, 1, 0, 0, 1, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, 1, 0, 0, 0, 1, 1, 1, 0, 0, 0, 1, 1, 0, 0, 0, 1, 1, 0, 0, 1, 1, 1],
+            4: [0, 0, 1, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1, 1, 0, 0, 1, 1, 1, 1, 0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 0, 1, 1, 1, 0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 1, 1, 0, 0, 1, 1, 1, 0, 1],
+            5: [1, 0, 0, 1, 1, 1, 0, 0, 0, 1, 0, 0, 0, 1, 1, 1, 1, 0, 0, 1, 1, 1, 0, 0, 1, 0, 0, 1, 1, 1, 1, 0, 0, 0, 1, 1, 0, 0, 1, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1, 1, 0, 0, 1, 0, 0, 0, 1, 1, 0]
+        },
+        "M": {
+            1: [1, 0, 1, 1, 0, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 0, 1, 0, 1, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 0, 1, 0, 1, 0, 1],
+            2: [0, 1, 0, 1, 0, 1, 1, 0, 1, 0, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 0, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 1]
+        }
+    },
+    "BREAM Pattern": {
+        "X": {
+            1: [0, 1, 1, 1, 1, 0, 1, 0, 1, 1, 0, 1, 0, 1, 1, 0, 0, 1, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0],
+            2: [0, 1, 1, 1, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 1, 0, 1, 0, 0, 0, 1, 1, 0, 1, 1, 1, 0, 0, 1, 1],
+            3: [1, 1, 0, 0, 1, 1, 0, 1, 1, 0, 0, 1, 1, 1, 0, 0, 0, 0, 1, 0, 0, 1, 1, 0, 1, 1, 1, 0, 0],
+            4: [1, 1, 1, 1, 0, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0],
+            5: [0, 1, 1, 1, 0, 1, 1, 1, 0, 0, 0, 1, 0, 0, 1, 1, 0, 1, 0, 0, 0, 1, 0]
+        },
+        "S": {
+            1: [0, 0, 0, 1, 1, 1, 0, 0, 1, 1, 1, 0, 1, 1, 0, 0, 1, 0, 1, 0, 1, 1, 0, 1, 1, 0, 1, 0, 0, 1, 0, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 0],
+            2: [1, 1, 0, 1, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 1, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 1, 0, 1, 0, 0],
+            3: [1, 0, 0, 1, 0, 0, 1, 1, 0, 1, 1, 1, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1, 0, 1, 0, 1, 0, 1, 1, 0, 0, 1, 0, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1],
+            4: [0, 1, 0, 0, 0, 0, 1, 0, 0, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 0, 0, 1, 0, 1, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 1, 0, 0, 1, 0, 1],
+            5: [1, 0, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 1, 0, 1, 1, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 1, 0, 1, 1, 1, 1, 0, 1, 1, 1, 0, 0, 1, 0, 1, 0, 0, 0, 1, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0]
+        },
+        "M": {
+            1: [1, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 1, 0, 0, 1, 1, 0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 1, 0, 1, 1, 0, 1, 0, 1, 1, 1, 1, 0, 0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1, 0, 1, 0, 1, 1, 1],
+            2: [0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 1, 1, 0, 1, 0, 1, 0, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 1, 1, 0, 1]
+        }
+    }
+};

src/core/lib/Magic.mjs

@@ -1,7 +1,8 @@
-import OperationConfig from "../config/OperationConfig.json";
-import Utils from "../Utils";
-import Recipe from "../Recipe";
-import Dish from "../Dish";
+import OperationConfig from "../config/OperationConfig.json" assert {type: "json"};
+import Utils, { isWorkerEnvironment } from "../Utils.mjs";
+import Recipe from "../Recipe.mjs";
+import Dish from "../Dish.mjs";
+import {detectFileType, isType} from "./FileType.mjs";
 import chiSquared from "chi-squared";
 
 /**
@@ -18,31 +19,38 @@ class Magic {
      * Magic constructor.
      *
      * @param {ArrayBuffer} buf
-     * @param {Object[]} [opPatterns]
+     * @param {Object[]} [opCriteria]
+     * @param {Object} [prevOp]
      */
-    constructor(buf, opPatterns) {
+    constructor(buf, opCriteria=Magic._generateOpCriteria(), prevOp=null) {
         this.inputBuffer = new Uint8Array(buf);
         this.inputStr = Utils.arrayBufferToStr(buf);
-        this.opPatterns = opPatterns || Magic._generateOpPatterns();
+        this.opCriteria = opCriteria;
+        this.prevOp = prevOp;
     }
 
     /**
-     * Finds operations that claim to be able to decode the input based on regular
-     * expression matches.
+     * Finds operations that claim to be able to decode the input based on various criteria.
      *
      * @returns {Object[]}
      */
-    findMatchingOps() {
-        const matches = [];
-
-        for (let i = 0; i < this.opPatterns.length; i++) {
-            const pattern = this.opPatterns[i],
-                regex = new RegExp(pattern.match, pattern.flags);
+    findMatchingInputOps() {
+        const matches = [],
+            inputEntropy = this.calcEntropy();
+
+        this.opCriteria.forEach(check => {
+            // If the input doesn't lie in the required entropy range, move on
+            if (check.entropyRange &&
+                (inputEntropy < check.entropyRange[0] ||
+                inputEntropy > check.entropyRange[1]))
+                return;
+            // If the input doesn't match the pattern, move on
+            if (check.pattern &&
+                !check.pattern.test(this.inputStr))
+                return;
 
-            if (regex.test(this.inputStr)) {
-                matches.push(pattern);
-            }
-        }
+            matches.push(check);
+        });
 
         return matches;
     }
@@ -92,7 +100,15 @@ class Magic {
      * @returns {string} [type.desc] - Description
      */
     detectFileType() {
-        return Magic.magicFileType(this.inputBuffer);
+        const fileType = detectFileType(this.inputBuffer);
+
+        if (!fileType.length) return null;
+        return {
+            name: fileType[0].name,
+            ext: fileType[0].extension,
+            mime: fileType[0].mime,
+            desc: fileType[0].description
+        };
     }
 
     /**
@@ -176,8 +192,10 @@ class Magic {
      *
      * @returns {number}
      */
-    calcEntropy() {
-        const prob = this._freqDist();
+    calcEntropy(data=this.inputBuffer, standalone=false) {
+        if (!standalone && this.inputEntropy) return this.inputEntropy;
+
+        const prob = this._freqDist(data, standalone);
         let entropy = 0,
             p;
 
@@ -186,6 +204,8 @@ class Magic {
             if (p === 0) continue;
             entropy += p * Math.log(p) / Math.log(2);
         }
+
+        if (!standalone) this.inputEntropy = -entropy;
         return -entropy;
     }
 
@@ -255,25 +275,59 @@ class Magic {
         return results;
     }
 
+    /**
+     * Checks whether the data passes output criteria for an operation check
+     *
+     * @param {ArrayBuffer} data
+     * @param {Object} criteria
+     * @returns {boolean}
+     */
+    outputCheckPasses(data, criteria) {
+        if (criteria.pattern) {
+            const dataStr = Utils.arrayBufferToStr(data),
+                regex = new RegExp(criteria.pattern, criteria.flags);
+            if (!regex.test(dataStr))
+                return false;
+        }
+        if (criteria.entropyRange) {
+            const dataEntropy = this.calcEntropy(data, true);
+            if (dataEntropy < criteria.entropyRange[0] || dataEntropy > criteria.entropyRange[1])
+                return false;
+        }
+        if (criteria.mime &&
+            !isType(criteria.mime, data))
+            return false;
+
+        return true;
+    }
+
     /**
      * Speculatively executes matching operations, recording metadata of each result.
      *
      * @param {number} [depth=0] - How many levels to try to execute
      * @param {boolean} [extLang=false] - Extensive language support (false = only check the most
-     *                                    common Internet languages)
+     *     common Internet languages)
      * @param {boolean} [intensive=false] - Run brute-forcing on each branch (significantly affects
-     *                                      performance)
+     *     performance)
      * @param {Object[]} [recipeConfig=[]] - The recipe configuration up to this point
      * @param {boolean} [useful=false] - Whether the current recipe should be scored highly
-     * @param {string} [crib=null] - The regex crib provided by the user, for filtering the operation output
+     * @param {string} [crib=null] - The regex crib provided by the user, for filtering the operation
+     *     output
      * @returns {Object[]} - A sorted list of the recipes most likely to result in correct decoding
      */
-    async speculativeExecution(depth=0, extLang=false, intensive=false, recipeConfig=[], useful=false, crib=null) {
+    async speculativeExecution(
+        depth=0,
+        extLang=false,
+        intensive=false,
+        recipeConfig=[],
+        useful=false,
+        crib=null) {
+
+        // If we have reached the recursion depth, return
         if (depth < 0) return [];
 
         // Find any operations that can be run on this data
-        const matchingOps = this.findMatchingOps();
-
+        const matchingOps = this.findMatchingInputOps();
         let results = [];
 
         // Record the properties of the current data
@@ -299,12 +353,21 @@ class Magic {
                 },
                 output = await this._runRecipe([opConfig]);
 
+            // If the recipe returned an empty buffer, do not continue
+            if (_buffersEqual(output, new ArrayBuffer())) {
+                return;
+            }
+
             // If the recipe is repeating and returning the same data, do not continue
             if (prevOp && op.op === prevOp.op && _buffersEqual(output, this.inputBuffer)) {
                 return;
             }
 
-            const magic = new Magic(output, this.opPatterns),
+            // If the output criteria for this op doesn't match the output, do not continue
+            if (op.output && !this.outputCheckPasses(output, op.output))
+                return;
+
+            const magic = new Magic(output, this.opCriteria, OperationConfig[op.op]),
                 speculativeResults = await magic.speculativeExecution(
                     depth-1, extLang, intensive, [...recipeConfig, opConfig], op.useful, crib);
 
@@ -316,7 +379,7 @@ class Magic {
             const bfEncodings = await this.bruteForce();
 
             await Promise.all(bfEncodings.map(async enc => {
-                const magic = new Magic(enc.data, this.opPatterns),
+                const magic = new Magic(enc.data, this.opCriteria, undefined),
                     bfResults = await magic.speculativeExecution(
                         depth-1, extLang, false, [...recipeConfig, enc.conf], false, crib);
 
@@ -325,33 +388,34 @@ class Magic {
         }
 
         // Prune branches that result in unhelpful outputs
-        results = results.filter(r =>
+        const prunedResults = results.filter(r =>
             (r.useful || r.data.length > 0) &&          // The operation resulted in ""
             (                                           // One of the following must be true
                 r.languageScores[0].probability > 0 ||    // Some kind of language was found
                 r.fileType ||                             // A file was found
                 r.isUTF8 ||                               // UTF-8 was found
-                r.matchingOps.length                      // A matching op was found
+                r.matchingOps.length ||                   // A matching op was found
+                r.matchesCrib                             // The crib matches
             )
         );
 
         // Return a sorted list of possible recipes along with their properties
-        return results.sort((a, b) => {
+        return prunedResults.sort((a, b) => {
             // Each option is sorted based on its most likely language (lower is better)
             let aScore = a.languageScores[0].score,
                 bScore = b.languageScores[0].score;
 
-            // If a recipe results in a file being detected, it receives a relatively good score
-            if (a.fileType) aScore = 500;
-            if (b.fileType) bScore = 500;
-
             // If the result is valid UTF8, its score gets boosted (lower being better)
             if (a.isUTF8) aScore -= 100;
             if (b.isUTF8) bScore -= 100;
 
+            // If a recipe results in a file being detected, it receives a relatively good score
+            if (a.fileType && aScore > 500) aScore = 500;
+            if (b.fileType && bScore > 500) bScore = 500;
+
             // If the option is marked useful, give it a good score
-            if (a.useful) aScore = 100;
-            if (b.useful) bScore = 100;
+            if (a.useful && aScore > 100) aScore = 100;
+            if (b.useful && bScore > 100) bScore = 100;
 
             // Shorter recipes are better, so we add the length of the recipe to the score
             aScore += a.recipe.length;
@@ -362,9 +426,10 @@ class Magic {
             bScore += b.entropy;
 
             // A result with no recipe but matching ops suggests there are better options
-            if ((!a.recipe.length && a.matchingOps.length) &&
-                b.recipe.length)
+            if ((!a.recipe.length && a.matchingOps.length) && b.recipe.length)
                 return 1;
+            if ((!b.recipe.length && b.matchingOps.length) && a.recipe.length)
+                return -1;
 
             return aScore - bScore;
         });
@@ -382,12 +447,17 @@ class Magic {
         const dish = new Dish();
         dish.set(input, Dish.ARRAY_BUFFER);
 
-        if (ENVIRONMENT_IS_WORKER()) self.loadRequiredModules(recipeConfig);
+        if (isWorkerEnvironment()) self.loadRequiredModules(recipeConfig);
 
         const recipe = new Recipe(recipeConfig);
         try {
             await recipe.execute(dish);
-            return dish.get(Dish.ARRAY_BUFFER);
+            // Return an empty buffer if the recipe did not run to completion
+            if (recipe.lastRunOp === recipe.opList[recipe.opList.length - 1]) {
+                return await dish.get(Dish.ARRAY_BUFFER);
+            } else {
+                return new ArrayBuffer();
+            }
         } catch (err) {
             // If there are errors, return an empty buffer
             return new ArrayBuffer();
@@ -398,14 +468,16 @@ class Magic {
      * Calculates the number of times each byte appears in the input as a percentage
      *
      * @private
+     * @param {ArrayBuffer} [data]
+     * @param {boolean} [standalone]
      * @returns {number[]}
      */
-    _freqDist() {
-        if (this.freqDist) return this.freqDist;
+    _freqDist(data=this.inputBuffer, standalone=false) {
+        if (!standalone && this.freqDist) return this.freqDist;
 
-        const len = this.inputBuffer.length;
+        const len = data.length,
+            counts = new Array(256).fill(0);
         let i = len;
-        const counts = new Array(256).fill(0);
 
         if (!len) {
             this.freqDist = counts;
@@ -413,13 +485,15 @@ class Magic {
         }
 
         while (i--) {
-            counts[this.inputBuffer[i]]++;
+            counts[data[i]]++;
         }
 
-        this.freqDist = counts.map(c => {
+        const result = counts.map(c => {
             return c / len * 100;
         });
-        return this.freqDist;
+
+        if (!standalone) this.freqDist = result;
+        return result;
     }
 
     /**
@@ -428,24 +502,29 @@ class Magic {
      * @private
      * @returns {Object[]}
      */
-    static _generateOpPatterns() {
-        const opPatterns = [];
+    static _generateOpCriteria() {
+        const opCriteria = [];
 
         for (const op in OperationConfig) {
-            if (!OperationConfig[op].hasOwnProperty("patterns")) continue;
+            if (!("checks" in OperationConfig[op]))
+                continue;
 
-            OperationConfig[op].patterns.forEach(pattern => {
-                opPatterns.push({
+            OperationConfig[op].checks.forEach(check => {
+                // Add to the opCriteria list.
+                // Compile the regex here and cache the compiled version so we
+                // don't have to keep calculating it.
+                opCriteria.push({
                     op: op,
-                    match: pattern.match,
-                    flags: pattern.flags,
-                    args: pattern.args,
-                    useful: pattern.useful || false
+                    pattern: check.pattern ? new RegExp(check.pattern, check.flags) : null,
+                    args: check.args,
+                    useful: check.useful,
+                    entropyRange: check.entropyRange,
+                    output: check.output
                 });
             });
         }
 
-        return opPatterns;
+        return opCriteria;
     }
 
     /**
@@ -479,7 +558,7 @@ class Magic {
      * Taken from http://wikistats.wmflabs.org/display.php?t=wp
      *
      * @param {string} code - ISO 639 code
-     * @returns {string} The full name of the languge
+     * @returns {string} The full name of the language
      */
     static codeToLanguage(code) {
         return {
@@ -785,452 +864,9 @@ class Magic {
         }[code];
     }
 
-
-    /**
-     * Given a buffer, detects magic byte sequences at specific positions and returns the
-     * extension and mime type.
-     *
-     * @param {Uint8Array} buf
-     * @returns {Object} type
-     * @returns {string} type.ext - File extension
-     * @returns {string} type.mime - Mime type
-     * @returns {string} [type.desc] - Description
-     */
-    static magicFileType(buf) {
-        if (!(buf && buf.length > 1)) {
-            return null;
-        }
-
-        if (buf[0] === 0xFF && buf[1] === 0xD8 && buf[2] === 0xFF) {
-            return {
-                ext: "jpg",
-                mime: "image/jpeg"
-            };
-        }
-
-        if (buf[0] === 0x89 && buf[1] === 0x50 && buf[2] === 0x4E && buf[3] === 0x47) {
-            return {
-                ext: "png",
-                mime: "image/png"
-            };
-        }
-
-        if (buf[0] === 0x47 && buf[1] === 0x49 && buf[2] === 0x46) {
-            return {
-                ext: "gif",
-                mime: "image/gif"
-            };
-        }
-
-        if (buf[8] === 0x57 && buf[9] === 0x45 && buf[10] === 0x42 && buf[11] === 0x50) {
-            return {
-                ext: "webp",
-                mime: "image/webp"
-            };
-        }
-
-        // needs to be before `tif` check
-        if (((buf[0] === 0x49 && buf[1] === 0x49 && buf[2] === 0x2A && buf[3] === 0x0) || (buf[0] === 0x4D && buf[1] === 0x4D && buf[2] === 0x0 && buf[3] === 0x2A)) && buf[8] === 0x43 && buf[9] === 0x52) {
-            return {
-                ext: "cr2",
-                mime: "image/x-canon-cr2"
-            };
-        }
-
-        if ((buf[0] === 0x49 && buf[1] === 0x49 && buf[2] === 0x2A && buf[3] === 0x0) || (buf[0] === 0x4D && buf[1] === 0x4D && buf[2] === 0x0 && buf[3] === 0x2A)) {
-            return {
-                ext: "tif",
-                mime: "image/tiff"
-            };
-        }
-
-        if (buf[0] === 0x42 && buf[1] === 0x4D) {
-            return {
-                ext: "bmp",
-                mime: "image/bmp"
-            };
-        }
-
-        if (buf[0] === 0x49 && buf[1] === 0x49 && buf[2] === 0xBC) {
-            return {
-                ext: "jxr",
-                mime: "image/vnd.ms-photo"
-            };
-        }
-
-        if (buf[0] === 0x38 && buf[1] === 0x42 && buf[2] === 0x50 && buf[3] === 0x53) {
-            return {
-                ext: "psd",
-                mime: "image/vnd.adobe.photoshop"
-            };
-        }
-
-        // needs to be before `zip` check
-        if (buf[0] === 0x50 && buf[1] === 0x4B && buf[2] === 0x3 && buf[3] === 0x4 && buf[30] === 0x6D && buf[31] === 0x69 && buf[32] === 0x6D && buf[33] === 0x65 && buf[34] === 0x74 && buf[35] === 0x79 && buf[36] === 0x70 && buf[37] === 0x65 && buf[38] === 0x61 && buf[39] === 0x70 && buf[40] === 0x70 && buf[41] === 0x6C && buf[42] === 0x69 && buf[43] === 0x63 && buf[44] === 0x61 && buf[45] === 0x74 && buf[46] === 0x69 && buf[47] === 0x6F && buf[48] === 0x6E && buf[49] === 0x2F && buf[50] === 0x65 && buf[51] === 0x70 && buf[52] === 0x75 && buf[53] === 0x62 && buf[54] === 0x2B && buf[55] === 0x7A && buf[56] === 0x69 && buf[57] === 0x70) {
-            return {
-                ext: "epub",
-                mime: "application/epub+zip"
-            };
-        }
-
-        if (buf[0] === 0x50 && buf[1] === 0x4B && (buf[2] === 0x3 || buf[2] === 0x5 || buf[2] === 0x7) && (buf[3] === 0x4 || buf[3] === 0x6 || buf[3] === 0x8)) {
-            return {
-                ext: "zip",
-                mime: "application/zip"
-            };
-        }
-
-        if (buf[257] === 0x75 && buf[258] === 0x73 && buf[259] === 0x74 && buf[260] === 0x61 && buf[261] === 0x72) {
-            return {
-                ext: "tar",
-                mime: "application/x-tar"
-            };
-        }
-
-        if (buf[0] === 0x52 && buf[1] === 0x61 && buf[2] === 0x72 && buf[3] === 0x21 && buf[4] === 0x1A && buf[5] === 0x7 && (buf[6] === 0x0 || buf[6] === 0x1)) {
-            return {
-                ext: "rar",
-                mime: "application/x-rar-compressed"
-            };
-        }
-
-        if (buf[0] === 0x1F && buf[1] === 0x8B && buf[2] === 0x8) {
-            return {
-                ext: "gz",
-                mime: "application/gzip"
-            };
-        }
-
-        if (buf[0] === 0x42 && buf[1] === 0x5A && buf[2] === 0x68) {
-            return {
-                ext: "bz2",
-                mime: "application/x-bzip2"
-            };
-        }
-
-        if (buf[0] === 0x37 && buf[1] === 0x7A && buf[2] === 0xBC && buf[3] === 0xAF && buf[4] === 0x27 && buf[5] === 0x1C) {
-            return {
-                ext: "7z",
-                mime: "application/x-7z-compressed"
-            };
-        }
-
-        if (buf[0] === 0x78 && buf[1] === 0x01) {
-            return {
-                ext: "dmg, zlib",
-                mime: "application/x-apple-diskimage, application/x-deflate"
-            };
-        }
-
-        if ((buf[0] === 0x0 && buf[1] === 0x0 && buf[2] === 0x0 && (buf[3] === 0x18 || buf[3] === 0x20) && buf[4] === 0x66 && buf[5] === 0x74 && buf[6] === 0x79 && buf[7] === 0x70) || (buf[0] === 0x33 && buf[1] === 0x67 && buf[2] === 0x70 && buf[3] === 0x35) || (buf[0] === 0x0 && buf[1] === 0x0 && buf[2] === 0x0 && buf[3] === 0x1C && buf[4] === 0x66 && buf[5] === 0x74 && buf[6] === 0x79 && buf[7] === 0x70 && buf[8] === 0x6D && buf[9] === 0x70 && buf[10] === 0x34 && buf[11] === 0x32 && buf[16] === 0x6D && buf[17] === 0x70 && buf[18] === 0x34 && buf[19] === 0x31 && buf[20] === 0x6D && buf[21] === 0x70 && buf[22] === 0x34 && buf[23] === 0x32 && buf[24] === 0x69 && buf[25] === 0x73 && buf[26] === 0x6F && buf[27] === 0x6D)) {
-            return {
-                ext: "mp4",
-                mime: "video/mp4"
-            };
-        }
-
-        if ((buf[0] === 0x0 && buf[1] === 0x0 && buf[2] === 0x0 && buf[3] === 0x1C && buf[4] === 0x66 && buf[5] === 0x74 && buf[6] === 0x79 && buf[7] === 0x70 && buf[8] === 0x4D && buf[9] === 0x34 && buf[10] === 0x56)) {
-            return {
-                ext: "m4v",
-                mime: "video/x-m4v"
-            };
-        }
-
-        if (buf[0] === 0x4D && buf[1] === 0x54 && buf[2] === 0x68 && buf[3] === 0x64) {
-            return {
-                ext: "mid",
-                mime: "audio/midi"
-            };
-        }
-
-        // needs to be before the `webm` check
-        if (buf[31] === 0x6D && buf[32] === 0x61 && buf[33] === 0x74 && buf[34] === 0x72 && buf[35] === 0x6f && buf[36] === 0x73 && buf[37] === 0x6B && buf[38] === 0x61) {
-            return {
-                ext: "mkv",
-                mime: "video/x-matroska"
-            };
-        }
-
-        if (buf[0] === 0x1A && buf[1] === 0x45 && buf[2] === 0xDF && buf[3] === 0xA3) {
-            return {
-                ext: "webm",
-                mime: "video/webm"
-            };
-        }
-
-        if (buf[0] === 0x0 && buf[1] === 0x0 && buf[2] === 0x0 && buf[3] === 0x14 && buf[4] === 0x66 && buf[5] === 0x74 && buf[6] === 0x79 && buf[7] === 0x70) {
-            return {
-                ext: "mov",
-                mime: "video/quicktime"
-            };
-        }
-
-        if (buf[0] === 0x52 && buf[1] === 0x49 && buf[2] === 0x46 && buf[3] === 0x46 && buf[8] === 0x41 && buf[9] === 0x56 && buf[10] === 0x49) {
-            return {
-                ext: "avi",
-                mime: "video/x-msvideo"
-            };
-        }
-
-        if (buf[0] === 0x30 && buf[1] === 0x26 && buf[2] === 0xB2 && buf[3] === 0x75 && buf[4] === 0x8E && buf[5] === 0x66 && buf[6] === 0xCF && buf[7] === 0x11 && buf[8] === 0xA6 && buf[9] === 0xD9) {
-            return {
-                ext: "wmv",
-                mime: "video/x-ms-wmv"
-            };
-        }
-
-        if (buf[0] === 0x0 && buf[1] === 0x0 && buf[2] === 0x1 && buf[3].toString(16)[0] === "b") {
-            return {
-                ext: "mpg",
-                mime: "video/mpeg"
-            };
-        }
-
-        if ((buf[0] === 0x49 && buf[1] === 0x44 && buf[2] === 0x33) || (buf[0] === 0xFF && buf[1] === 0xfb)) {
-            return {
-                ext: "mp3",
-                mime: "audio/mpeg"
-            };
-        }
-
-        if ((buf[4] === 0x66 && buf[5] === 0x74 && buf[6] === 0x79 && buf[7] === 0x70 && buf[8] === 0x4D && buf[9] === 0x34 && buf[10] === 0x41) || (buf[0] === 0x4D && buf[1] === 0x34 && buf[2] === 0x41 && buf[3] === 0x20)) {
-            return {
-                ext: "m4a",
-                mime: "audio/m4a"
-            };
-        }
-
-        if (buf[0] === 0x4F && buf[1] === 0x67 && buf[2] === 0x67 && buf[3] === 0x53) {
-            return {
-                ext: "ogg",
-                mime: "audio/ogg"
-            };
-        }
-
-        if (buf[0] === 0x66 && buf[1] === 0x4C && buf[2] === 0x61 && buf[3] === 0x43) {
-            return {
-                ext: "flac",
-                mime: "audio/x-flac"
-            };
-        }
-
-        if (buf[0] === 0x52 && buf[1] === 0x49 && buf[2] === 0x46 && buf[3] === 0x46 && buf[8] === 0x57 && buf[9] === 0x41 && buf[10] === 0x56 && buf[11] === 0x45) {
-            return {
-                ext: "wav",
-                mime: "audio/x-wav"
-            };
-        }
-
-        if (buf[0] === 0x23 && buf[1] === 0x21 && buf[2] === 0x41 && buf[3] === 0x4D && buf[4] === 0x52 && buf[5] === 0x0A) {
-            return {
-                ext: "amr",
-                mime: "audio/amr"
-            };
-        }
-
-        if (buf[0] === 0x25 && buf[1] === 0x50 && buf[2] === 0x44 && buf[3] === 0x46) {
-            return {
-                ext: "pdf",
-                mime: "application/pdf"
-            };
-        }
-
-        if (buf[0] === 0x4D && buf[1] === 0x5A) {
-            return {
-                ext: "exe",
-                mime: "application/x-msdownload"
-            };
-        }
-
-        if ((buf[0] === 0x43 || buf[0] === 0x46) && buf[1] === 0x57 && buf[2] === 0x53) {
-            return {
-                ext: "swf",
-                mime: "application/x-shockwave-flash"
-            };
-        }
-
-        if (buf[0] === 0x7B && buf[1] === 0x5C && buf[2] === 0x72 && buf[3] === 0x74 && buf[4] === 0x66) {
-            return {
-                ext: "rtf",
-                mime: "application/rtf"
-            };
-        }
-
-        if (buf[0] === 0x77 && buf[1] === 0x4F && buf[2] === 0x46 && buf[3] === 0x46 && buf[4] === 0x00 && buf[5] === 0x01 && buf[6] === 0x00 && buf[7] === 0x00) {
-            return {
-                ext: "woff",
-                mime: "application/font-woff"
-            };
-        }
-
-        if (buf[0] === 0x77 && buf[1] === 0x4F && buf[2] === 0x46 && buf[3] === 0x32 && buf[4] === 0x00 && buf[5] === 0x01 && buf[6] === 0x00 && buf[7] === 0x00) {
-            return {
-                ext: "woff2",
-                mime: "application/font-woff"
-            };
-        }
-
-        if (buf[34] === 0x4C && buf[35] === 0x50 && ((buf[8] === 0x02 && buf[9] === 0x00 && buf[10] === 0x01) || (buf[8] === 0x01 && buf[9] === 0x00 && buf[10] === 0x00) || (buf[8] === 0x02 && buf[9] === 0x00 && buf[10] === 0x02))) {
-            return {
-                ext: "eot",
-                mime: "application/octet-stream"
-            };
-        }
-
-        if (buf[0] === 0x00 && buf[1] === 0x01 && buf[2] === 0x00 && buf[3] === 0x00 && buf[4] === 0x00) {
-            return {
-                ext: "ttf",
-                mime: "application/font-sfnt"
-            };
-        }
-
-        if (buf[0] === 0x4F && buf[1] === 0x54 && buf[2] === 0x54 && buf[3] === 0x4F && buf[4] === 0x00) {
-            return {
-                ext: "otf",
-                mime: "application/font-sfnt"
-            };
-        }
-
-        if (buf[0] === 0x00 && buf[1] === 0x00 && buf[2] === 0x01 && buf[3] === 0x00) {
-            return {
-                ext: "ico",
-                mime: "image/x-icon"
-            };
-        }
-
-        if (buf[0] === 0x46 && buf[1] === 0x4C && buf[2] === 0x56 && buf[3] === 0x01) {
-            return {
-                ext: "flv",
-                mime: "video/x-flv"
-            };
-        }
-
-        if (buf[0] === 0x25 && buf[1] === 0x21) {
-            return {
-                ext: "ps",
-                mime: "application/postscript"
-            };
-        }
-
-        if (buf[0] === 0xFD && buf[1] === 0x37 && buf[2] === 0x7A && buf[3] === 0x58 && buf[4] === 0x5A && buf[5] === 0x00) {
-            return {
-                ext: "xz",
-                mime: "application/x-xz"
-            };
-        }
-
-        if (buf[0] === 0x53 && buf[1] === 0x51 && buf[2] === 0x4C && buf[3] === 0x69) {
-            return {
-                ext: "sqlite",
-                mime: "application/x-sqlite3"
-            };
-        }
-
-        /**
-         *
-         * Added by n1474335 [n1474335@gmail.com] from here on
-         *
-         */
-        if ((buf[0] === 0x1F && buf[1] === 0x9D) || (buf[0] === 0x1F && buf[1] === 0xA0)) {
-            return {
-                ext: "z, tar.z",
-                mime: "application/x-gtar"
-            };
-        }
-
-        if (buf[0] === 0x7F && buf[1] === 0x45 && buf[2] === 0x4C && buf[3] === 0x46) {
-            return {
-                ext: "none, axf, bin, elf, o, prx, puff, so",
-                mime: "application/x-executable",
-                desc: "Executable and Linkable Format file. No standard file extension."
-            };
-        }
-
-        if (buf[0] === 0xCA && buf[1] === 0xFE && buf[2] === 0xBA && buf[3] === 0xBE) {
-            return {
-                ext: "class",
-                mime: "application/java-vm"
-            };
-        }
-
-        if (buf[0] === 0xEF && buf[1] === 0xBB && buf[2] === 0xBF) {
-            return {
-                ext: "txt",
-                mime: "text/plain",
-                desc: "UTF-8 encoded Unicode byte order mark detected, commonly but not exclusively seen in text files."
-            };
-        }
-
-        // Must be before Little-endian UTF-16 BOM
-        if (buf[0] === 0xFF && buf[1] === 0xFE && buf[2] === 0x00 && buf[3] === 0x00) {
-            return {
-                ext: "UTF32LE",
-                mime: "charset/utf32le",
-                desc: "Little-endian UTF-32 encoded Unicode byte order mark detected."
-            };
-        }
-
-        if (buf[0] === 0xFF && buf[1] === 0xFE) {
-            return {
-                ext: "UTF16LE",
-                mime: "charset/utf16le",
-                desc: "Little-endian UTF-16 encoded Unicode byte order mark detected."
-            };
-        }
-
-        if ((buf[0x8001] === 0x43 && buf[0x8002] === 0x44 && buf[0x8003] === 0x30 && buf[0x8004] === 0x30 && buf[0x8005] === 0x31) ||
-            (buf[0x8801] === 0x43 && buf[0x8802] === 0x44 && buf[0x8803] === 0x30 && buf[0x8804] === 0x30 && buf[0x8805] === 0x31) ||
-            (buf[0x9001] === 0x43 && buf[0x9002] === 0x44 && buf[0x9003] === 0x30 && buf[0x9004] === 0x30 && buf[0x9005] === 0x31)) {
-            return {
-                ext: "iso",
-                mime: "application/octet-stream",
-                desc: "ISO 9660 CD/DVD image file"
-            };
-        }
-
-        if (buf[0] === 0xD0 && buf[1] === 0xCF && buf[2] === 0x11 && buf[3] === 0xE0 && buf[4] === 0xA1 && buf[5] === 0xB1 && buf[6] === 0x1A && buf[7] === 0xE1) {
-            return {
-                ext: "doc, xls, ppt",
-                mime: "application/msword, application/vnd.ms-excel, application/vnd.ms-powerpoint",
-                desc: "Microsoft Office documents"
-            };
-        }
-
-        if (buf[0] === 0x64 && buf[1] === 0x65 && buf[2] === 0x78 && buf[3] === 0x0A && buf[4] === 0x30 && buf[5] === 0x33 && buf[6] === 0x35 && buf[7] === 0x00) {
-            return {
-                ext: "dex",
-                mime: "application/octet-stream",
-                desc: "Dalvik Executable (Android)"
-            };
-        }
-
-        if (buf[0] === 0x4B && buf[1] === 0x44 && buf[2] === 0x4D) {
-            return {
-                ext: "vmdk",
-                mime: "application/vmdk, application/x-virtualbox-vmdk"
-            };
-        }
-
-        if (buf[0] === 0x43 && buf[1] === 0x72 && buf[2] === 0x32 && buf[3] === 0x34) {
-            return {
-                ext: "crx",
-                mime: "application/crx",
-                desc: "Google Chrome extension or packaged app"
-            };
-        }
-
-        if (buf[0] === 0x78 && (buf[1] === 0x01 || buf[1] === 0x9C || buf[1] === 0xDA || buf[1] === 0x5e)) {
-            return {
-                ext: "zlib",
-                mime: "application/x-deflate"
-            };
-        }
-
-        return null;
-    }
-
 }
 
+
 /**
  * Byte frequencies of various languages generated from Wikipedia dumps taken in late 2017 and early 2018.
  * The Chi-Squared test cannot accept expected values of 0, so 0.0001 has been used to account for bytes

src/core/lib/PGP.mjs

@@ -10,7 +10,8 @@
  *
  */
 
-import OperationError from "../errors/OperationError";
+import OperationError from "../errors/OperationError.mjs";
+import { isWorkerEnvironment } from "../Utils.mjs";
 import kbpgp from "kbpgp";
 import * as es6promisify from "es6-promisify";
 const promisify = es6promisify.default ? es6promisify.default.promisify : es6promisify.promisify;
@@ -45,7 +46,7 @@ export const ASP = kbpgp.ASP({
                 msg = `Stage: ${info.what}`;
         }
 
-        if (ENVIRONMENT_IS_WORKER())
+        if (isWorkerEnvironment())
             self.sendStatusMessage(msg);
     }
 });

src/core/lib/Protobuf.mjs

@@ -0,0 +1,562 @@
+import Utils from "../Utils.mjs";
+import protobuf from "protobufjs";
+
+/**
+ * Protobuf lib. Contains functions to decode protobuf serialised
+ * data without a schema or .proto file.
+ *
+ * Provides utility functions to encode and decode variable length
+ * integers (varint).
+ *
+ * @author GCHQ Contributor [3]
+ * @copyright Crown Copyright 2019
+ * @license Apache-2.0
+ */
+class Protobuf {
+
+    /**
+     * Protobuf constructor
+     *
+     * @param {byteArray|Uint8Array} data
+     */
+    constructor(data) {
+        // Check we have a byteArray or Uint8Array
+        if (data instanceof Array || data instanceof Uint8Array) {
+            this.data = data;
+        } else {
+            throw new Error("Protobuf input must be a byteArray or Uint8Array");
+        }
+
+        // Set up masks
+        this.TYPE = 0x07;
+        this.NUMBER = 0x78;
+        this.MSB = 0x80;
+        this.VALUE = 0x7f;
+
+        // Declare offset, length, and field type object
+        this.offset = 0;
+        this.LENGTH = data.length;
+        this.fieldTypes = {};
+    }
+
+    // Public Functions
+
+    /**
+     * Encode a varint from a number
+     *
+     * @param {number} number
+     * @returns {byteArray}
+     */
+    static varIntEncode(number) {
+        const MSB = 0x80,
+            VALUE = 0x7f,
+            MSBALL = ~VALUE,
+            INT = Math.pow(2, 31);
+        const out = [];
+        let offset = 0;
+
+        while (number >= INT) {
+            out[offset++] = (number & 0xff) | MSB;
+            number /= 128;
+        }
+        while (number & MSBALL) {
+            out[offset++] = (number & 0xff) | MSB;
+            number >>>= 7;
+        }
+        out[offset] = number | 0;
+        return out;
+    }
+
+    /**
+     * Decode a varint from the byteArray
+     *
+     * @param {byteArray} input
+     * @returns {number}
+     */
+    static varIntDecode(input) {
+        const pb = new Protobuf(input);
+        return pb._varInt();
+    }
+
+    /**
+     * Encode input JSON according to the given schema
+     *
+     * @param {Object} input
+     * @param {Object []} args
+     * @returns {Object}
+     */
+    static encode(input, args) {
+        this.updateProtoRoot(args[0]);
+        if (!this.mainMessageName) {
+            throw new Error("Schema Error: Schema not defined");
+        }
+        const message = this.parsedProto.root.nested[this.mainMessageName];
+
+        // Convert input into instance of message, and verify instance
+        input = message.fromObject(input);
+        const error = message.verify(input);
+        if (error) {
+            throw new Error("Input Error: " + error);
+        }
+        // Encode input
+        const output = message.encode(input).finish();
+        return new Uint8Array(output).buffer;
+    }
+
+    /**
+     * Parse Protobuf data
+     *
+     * @param {byteArray} input
+     * @returns {Object}
+     */
+    static decode(input, args) {
+        this.updateProtoRoot(args[0]);
+        this.showUnknownFields = args[1];
+        this.showTypes = args[2];
+        return this.mergeDecodes(input);
+    }
+
+    /**
+     * Update the parsedProto, throw parsing errors
+     *
+     * @param {string} protoText
+     */
+    static updateProtoRoot(protoText) {
+        try {
+            this.parsedProto = protobuf.parse(protoText);
+            if (this.parsedProto.package) {
+                this.parsedProto.root = this.parsedProto.root.nested[this.parsedProto.package];
+            }
+            this.updateMainMessageName();
+        } catch (error) {
+            throw new Error("Schema " + error);
+        }
+    }
+
+    /**
+     * Set mainMessageName to the first instance of a message defined in the schema that is not a submessage
+     *
+     */
+    static updateMainMessageName() {
+        const messageNames = [];
+        const fieldTypes = [];
+        this.parsedProto.root.nestedArray.forEach(block => {
+            if (block instanceof protobuf.Type) {
+                messageNames.push(block.name);
+                this.parsedProto.root.nested[block.name].fieldsArray.forEach(field => {
+                    fieldTypes.push(field.type);
+                });
+            }
+        });
+
+        if (messageNames.length === 0) {
+            this.mainMessageName = null;
+        } else {
+            // for (const name of messageNames) {
+            //     if (!fieldTypes.includes(name)) {
+            //         this.mainMessageName = name;
+            //         break;
+            //     }
+            // }
+            this.mainMessageName = messageNames[0];
+        }
+    }
+
+    /**
+     * Decode input using Protobufjs package and raw methods, compare, and merge results
+     *
+     * @param {byteArray} input
+     * @returns {Object}
+     */
+    static mergeDecodes(input) {
+        const pb = new Protobuf(input);
+        let rawDecode = pb._parse();
+        let message;
+
+        if (this.showTypes) {
+            rawDecode = this.showRawTypes(rawDecode, pb.fieldTypes);
+            this.parsedProto.root = this.appendTypesToFieldNames(this.parsedProto.root);
+        }
+
+        try {
+            message = this.parsedProto.root.nested[this.mainMessageName];
+            const packageDecode = message.toObject(message.decode(input), {
+                bytes: String,
+                longs: Number,
+                enums: String,
+                defaults: true
+            });
+            const output = {};
+
+            if (this.showUnknownFields) {
+                output[message.name] = packageDecode;
+                output["Unknown Fields"] = this.compareFields(rawDecode, message);
+                return output;
+            } else {
+                return packageDecode;
+            }
+
+        } catch (error) {
+            if (message) {
+                throw new Error("Input " + error);
+            } else {
+                return rawDecode;
+            }
+        }
+    }
+
+    /**
+     * Replace fieldnames with fieldname and type
+     *
+     * @param {Object} schemaRoot
+     * @returns {Object}
+     */
+    static appendTypesToFieldNames(schemaRoot) {
+        for (const block of schemaRoot.nestedArray) {
+            if (block instanceof protobuf.Type) {
+                for (const [fieldName, fieldData] of Object.entries(block.fields)) {
+                    schemaRoot.nested[block.name].remove(block.fields[fieldName]);
+                    schemaRoot.nested[block.name].add(new protobuf.Field(`${fieldName} (${fieldData.type})`, fieldData.id, fieldData.type, fieldData.rule));
+                }
+            }
+        }
+        return schemaRoot;
+    }
+
+    /**
+     * Add field type to field name for fields in the raw decoded output
+     *
+     * @param {Object} rawDecode
+     * @param {Object} fieldTypes
+     * @returns {Object}
+     */
+    static showRawTypes(rawDecode, fieldTypes) {
+        for (const [fieldNum, value] of Object.entries(rawDecode)) {
+            const fieldType = fieldTypes[fieldNum];
+            let outputFieldValue;
+            let outputFieldType;
+
+            // Submessages
+            if (isNaN(fieldType)) {
+                outputFieldType = 2;
+
+                // Repeated submessages
+                if (Array.isArray(value)) {
+                    const fieldInstances = [];
+                    for (const instance of Object.keys(value)) {
+                        if (typeof(value[instance]) !== "string") {
+                            fieldInstances.push(this.showRawTypes(value[instance], fieldType));
+                        } else {
+                            fieldInstances.push(value[instance]);
+                        }
+                    }
+                    outputFieldValue = fieldInstances;
+
+                // Single submessage
+                } else {
+                    outputFieldValue = this.showRawTypes(value, fieldType);
+                }
+
+            // Non-submessage field
+            } else {
+                outputFieldType = fieldType;
+                outputFieldValue = value;
+            }
+
+            // Substitute fieldNum with field number and type
+            rawDecode[`field #${fieldNum}: ${this.getTypeInfo(outputFieldType)}`] = outputFieldValue;
+            delete rawDecode[fieldNum];
+        }
+        return rawDecode;
+    }
+
+    /**
+     * Compare raw decode to package decode and return discrepancies
+     *
+     * @param rawDecodedMessage
+     * @param schemaMessage
+     * @returns {Object}
+     */
+    static compareFields(rawDecodedMessage, schemaMessage) {
+        // Define message data using raw decode output and schema
+        const schemaFieldProperties = {};
+        const schemaFieldNames = Object.keys(schemaMessage.fields);
+        schemaFieldNames.forEach(field => schemaFieldProperties[schemaMessage.fields[field].id] = field);
+
+        // Loop over each field present in the raw decode output
+        for (const fieldName in rawDecodedMessage) {
+            let fieldId;
+            if (isNaN(fieldName)) {
+                fieldId = fieldName.match(/^field #(\d+)/)[1];
+            } else {
+                fieldId = fieldName;
+            }
+
+            // Check if this field is defined in the schema
+            if (fieldId in schemaFieldProperties) {
+                const schemaFieldName = schemaFieldProperties[fieldId];
+
+                // Extract the current field data from the raw decode and schema
+                const rawFieldData = rawDecodedMessage[fieldName];
+                const schemaField = schemaMessage.fields[schemaFieldName];
+
+                // Check for repeated fields
+                if (Array.isArray(rawFieldData) && !schemaField.repeated) {
+                    rawDecodedMessage[`(${schemaMessage.name}) ${schemaFieldName} is a repeated field`] = rawFieldData;
+                }
+
+                // Check for submessage fields
+                if (schemaField.resolvedType instanceof protobuf.Type) {
+                    const subMessageType = schemaMessage.fields[schemaFieldName].type;
+                    const schemaSubMessage = this.parsedProto.root.nested[subMessageType];
+                    const rawSubMessages = rawDecodedMessage[fieldName];
+                    let rawDecodedSubMessage = {};
+
+                    // Squash multiple submessage instances into one submessage
+                    if (Array.isArray(rawSubMessages)) {
+                        rawSubMessages.forEach(subMessageInstance => {
+                            const instanceFields = Object.entries(subMessageInstance);
+                            instanceFields.forEach(subField => {
+                                rawDecodedSubMessage[subField[0]] = subField[1];
+                            });
+                        });
+                    } else {
+                        rawDecodedSubMessage = rawSubMessages;
+                    }
+
+                    // Treat submessage as own message and compare its fields
+                    rawDecodedSubMessage = Protobuf.compareFields(rawDecodedSubMessage, schemaSubMessage);
+                    if (Object.entries(rawDecodedSubMessage).length !== 0) {
+                        rawDecodedMessage[`${schemaFieldName} (${subMessageType}) has missing fields`] = rawDecodedSubMessage;
+                    }
+                }
+                delete rawDecodedMessage[fieldName];
+            }
+        }
+        return rawDecodedMessage;
+    }
+
+    /**
+     * Returns wiretype information for input wiretype number
+     *
+     * @param {number} wireType
+     * @returns {string}
+     */
+    static getTypeInfo(wireType) {
+        switch (wireType) {
+            case 0:
+                return "VarInt (e.g. int32, bool)";
+            case 1:
+                return "64-Bit (e.g. fixed64, double)";
+            case 2:
+                return "L-delim (e.g. string, message)";
+            case 5:
+                return "32-Bit (e.g. fixed32, float)";
+        }
+    }
+
+    // Private Class Functions
+
+    /**
+     * Main private parsing function
+     *
+     * @private
+     * @returns {Object}
+     */
+    _parse() {
+        let object = {};
+        // Continue reading whilst we still have data
+        while (this.offset < this.LENGTH) {
+            const field = this._parseField();
+            object = this._addField(field, object);
+        }
+        // Throw an error if we have gone beyond the end of the data
+        if (this.offset > this.LENGTH) {
+            throw new Error("Exhausted Buffer");
+        }
+        return object;
+    }
+
+    /**
+     * Add a field read from the protobuf data into the Object. As
+     * protobuf fields can appear multiple times, if the field already
+     * exists we need to add the new field into an array of fields
+     * for that key.
+     *
+     * @private
+     * @param {Object} field
+     * @param {Object} object
+     * @returns {Object}
+     */
+    _addField(field, object) {
+        // Get the field key/values
+        const key = field.key;
+        const value = field.value;
+        object[key] = Object.prototype.hasOwnProperty.call(object, key) ?
+            object[key] instanceof Array ?
+                object[key].concat([value]) :
+                [object[key], value] :
+            value;
+        return object;
+    }
+
+    /**
+     * Parse a field and return the Object read from the record
+     *
+     * @private
+     * @returns {Object}
+     */
+    _parseField() {
+        // Get the field headers
+        const header = this._fieldHeader();
+        const type = header.type;
+        const key = header.key;
+
+        if (typeof(this.fieldTypes[key]) !== "object") {
+            this.fieldTypes[key] = type;
+        }
+
+        switch (type) {
+            // varint
+            case 0:
+                return { "key": key, "value": this._varInt() };
+            // fixed 64
+            case 1:
+                return { "key": key, "value": this._uint64() };
+            // length delimited
+            case 2:
+                return { "key": key, "value": this._lenDelim(key) };
+            // fixed 32
+            case 5:
+                return { "key": key, "value": this._uint32() };
+            // unknown type
+            default:
+                throw new Error("Unknown type 0x" + type.toString(16));
+        }
+    }
+
+    /**
+     * Parse the field header and return the type and key
+     *
+     * @private
+     * @returns {Object}
+     */
+    _fieldHeader() {
+        // Make sure we call type then number to preserve offset
+        return { "type": this._fieldType(), "key": this._fieldNumber() };
+    }
+
+    /**
+     * Parse the field type from the field header. Type is stored in the
+     * lower 3 bits of the tag byte. This does not move the offset on as
+     * we need to read the field number from the tag byte too.
+     *
+     * @private
+     * @returns {number}
+     */
+    _fieldType() {
+        // Field type stored in lower 3 bits of tag byte
+        return this.data[this.offset] & this.TYPE;
+    }
+
+    /**
+     * Parse the field number (i.e. the key) from the field header. The
+     * field number is stored in the upper 5 bits of the tag byte - but
+     * is also varint encoded so the follow on bytes may need to be read
+     * when field numbers are > 15.
+     *
+     * @private
+     * @returns {number}
+     */
+    _fieldNumber() {
+        let shift = -3;
+        let fieldNumber = 0;
+        do {
+            fieldNumber += shift < 28 ?
+                shift === -3 ?
+                    (this.data[this.offset] & this.NUMBER) >> -shift :
+                    (this.data[this.offset] & this.VALUE) << shift :
+                (this.data[this.offset] & this.VALUE) * Math.pow(2, shift);
+            shift += 7;
+        } while ((this.data[this.offset++] & this.MSB) === this.MSB);
+        return fieldNumber;
+    }
+
+    // Field Parsing Functions
+
+    /**
+     * Read off a varint from the data
+     *
+     * @private
+     * @returns {number}
+     */
+    _varInt() {
+        let value = 0;
+        let shift = 0;
+        // Keep reading while upper bit set
+        do {
+            value += shift < 28 ?
+                (this.data[this.offset] & this.VALUE) << shift :
+                (this.data[this.offset] & this.VALUE) * Math.pow(2, shift);
+            shift += 7;
+        } while ((this.data[this.offset++] & this.MSB) === this.MSB);
+        return value;
+    }
+
+    /**
+     * Read off a 64 bit unsigned integer from the data
+     *
+     * @private
+     * @returns {number}
+     */
+    _uint64() {
+        // Read off a Uint64 with little-endian
+        const lowerHalf = this.data[this.offset++] + (this.data[this.offset++] * 0x100) + (this.data[this.offset++] * 0x10000) + this.data[this.offset++] * 0x1000000;
+        const upperHalf = this.data[this.offset++] + (this.data[this.offset++] * 0x100) + (this.data[this.offset++] * 0x10000) + this.data[this.offset++] * 0x1000000;
+        return upperHalf * 0x100000000 + lowerHalf;
+    }
+
+    /**
+     * Read off a length delimited field from the data
+     *
+     * @private
+     * @returns {Object|string}
+     */
+    _lenDelim(fieldNum) {
+        // Read off the field length
+        const length = this._varInt();
+        const fieldBytes = this.data.slice(this.offset, this.offset + length);
+        let field;
+        try {
+            // Attempt to parse as a new Protobuf Object
+            const pbObject = new Protobuf(fieldBytes);
+            field = pbObject._parse();
+
+            // Set field types object
+            this.fieldTypes[fieldNum] = {...this.fieldTypes[fieldNum], ...pbObject.fieldTypes};
+
+        } catch (err) {
+            // Otherwise treat as bytes
+            field = Utils.byteArrayToChars(fieldBytes);
+        }
+        // Move the offset and return the field
+        this.offset += length;
+        return field;
+    }
+
+    /**
+     * Read a 32 bit unsigned integer from the data
+     *
+     * @private
+     * @returns {number}
+     */
+    _uint32() {
+        // Use a dataview to read off the integer
+        const dataview = new DataView(new Uint8Array(this.data.slice(this.offset, this.offset + 4)).buffer);
+        const value = dataview.getUint32(0, true);
+        this.offset += 4;
+        return value;
+    }
+}
+
+export default Protobuf;

src/core/lib/Protocol.mjs

@@ -0,0 +1,47 @@
+/**
+ * Protocol parsing functions.
+ *
+ * @author n1474335 [n1474335@gmail.com]
+ * @copyright Crown Copyright 2022
+ * @license Apache-2.0
+ */
+
+import BigNumber from "bignumber.js";
+import {toHexFast} from "../lib/Hex.mjs";
+
+/**
+ * Recursively displays a JSON object as an HTML table
+ *
+ * @param {Object} obj
+ * @returns string
+ */
+export function objToTable(obj, nested=false) {
+    let html = `<table
+        class='table table-sm table-nonfluid ${nested ? "mb-0 table-borderless" : "table-bordered"}'
+        style='table-layout: fixed; ${nested ? "margin: -1px !important;" : ""}'>`;
+    if (!nested)
+        html += `<tr>
+            <th>Field</th>
+            <th>Value</th>
+        </tr>`;
+
+    for (const key in obj) {
+        html += `<tr><td style='word-wrap: break-word'>${key}</td>`;
+        if (typeof obj[key] === "object")
+            html += `<td style='padding: 0'>${objToTable(obj[key], true)}</td>`;
+        else
+            html += `<td>${obj[key]}</td>`;
+        html += "</tr>";
+    }
+    html += "</table>";
+    return html;
+}
+
+/**
+ * Converts bytes into a BigNumber string
+ * @param {Uint8Array} bs
+ * @returns {string}
+ */
+export function bytesToLargeNumber(bs) {
+    return BigNumber(toHexFast(bs), 16).toString();
+}

src/core/lib/PublicKey.mjs

@@ -6,38 +6,28 @@
  * @license Apache-2.0
  */
 
-import { toHex, fromHex } from "./Hex";
+import { toHex, fromHex } from "./Hex.mjs";
 
 /**
- * Formats Distinguished Name (DN) strings.
+ * Formats Distinguished Name (DN) objects to strings.
  *
- * @param {string} dnStr
+ * @param {Object} dnObj
  * @param {number} indent
  * @returns {string}
  */
-export function formatDnStr (dnStr, indent) {
-    const fields = dnStr.substr(1).replace(/([^\\])\//g, "$1$1/").split(/[^\\]\//);
-    let output = "",
-        maxKeyLen = 0,
-        key,
-        value,
-        i,
-        str;
-
-    for (i = 0; i < fields.length; i++) {
-        if (!fields[i].length) continue;
-
-        key = fields[i].split("=")[0];
+export function formatDnObj(dnObj, indent) {
+    let output = "";
 
-        maxKeyLen = key.length > maxKeyLen ? key.length : maxKeyLen;
-    }
+    const maxKeyLen = dnObj.array.reduce((max, item) => {
+        return item[0].type.length > max ? item[0].type.length : max;
+    }, 0);
 
-    for (i = 0; i < fields.length; i++) {
-        if (!fields[i].length) continue;
+    for (let i = 0; i < dnObj.array.length; i++) {
+        if (!dnObj.array[i].length) continue;
 
-        key = fields[i].split("=")[0];
-        value = fields[i].split("=")[1];
-        str = key.padEnd(maxKeyLen, " ") + " = " + value + "\n";
+        const key = dnObj.array[i][0].type;
+        const value = dnObj.array[i][0].value;
+        const str = `${key.padEnd(maxKeyLen, " ")} = ${value}\n`;
 
         output += str.padStart(indent + str.length, " ");
     }
@@ -54,7 +44,7 @@ export function formatDnStr (dnStr, indent) {
  * @param {number} indent
  * @returns {string}
  */
-export function formatByteStr (byteStr, length, indent) {
+export function formatByteStr(byteStr, length, indent) {
     byteStr = toHex(fromHex(byteStr), ":");
     length = length * 3;
     let output = "";

src/core/lib/QRCode.mjs

@@ -0,0 +1,93 @@
+/**
+ * QR code resources
+ *
+ * @author j433866 [j433866@gmail.com]
+ * @copyright Crown Copyright 2019
+ * @license Apache-2.0
+ */
+
+import OperationError from "../errors/OperationError.mjs";
+import jsQR from "jsqr";
+import qr from "qr-image";
+import Utils from "../Utils.mjs";
+import jimp from "jimp";
+
+/**
+ * Parses a QR code image from an image
+ *
+ * @param {ArrayBuffer} input
+ * @param {boolean} normalise
+ * @returns {string}
+ */
+export async function parseQrCode(input, normalise) {
+    let image;
+    try {
+        image = await jimp.read(input);
+    } catch (err) {
+        throw new OperationError(`Error opening image. (${err})`);
+    }
+
+    try {
+        if (normalise) {
+            image.rgba(false);
+            image.background(0xFFFFFFFF);
+            image.normalize();
+            image.greyscale();
+            image = await image.getBufferAsync(jimp.MIME_JPEG);
+            image = await jimp.read(image);
+        }
+    } catch (err) {
+        throw new OperationError(`Error normalising image. (${err})`);
+    }
+
+    const qrData = jsQR(image.bitmap.data, image.getWidth(), image.getHeight());
+    if (qrData) {
+        return qrData.data;
+    } else {
+        throw new OperationError("Could not read a QR code from the image.");
+    }
+}
+
+/**
+ * Generates a QR code from the input string
+ *
+ * @param {string} input
+ * @param {string} format
+ * @param {number} moduleSize
+ * @param {number} margin
+ * @param {string} errorCorrection
+ * @returns {ArrayBuffer}
+ */
+export function generateQrCode(input, format, moduleSize, margin, errorCorrection) {
+    const formats = ["SVG", "EPS", "PDF", "PNG"];
+    if (!formats.includes(format.toUpperCase())) {
+        throw new OperationError("Unsupported QR code format.");
+    }
+
+    let qrImage;
+    try {
+        qrImage = qr.imageSync(input, {
+            type: format,
+            size: moduleSize,
+            margin: margin,
+            "ec_level": errorCorrection.charAt(0).toUpperCase()
+        });
+    } catch (err) {
+        throw new OperationError(`Error generating QR code. (${err})`);
+    }
+
+    if (!qrImage) {
+        throw new OperationError("Error generating QR code.");
+    }
+
+    switch (format) {
+        case "SVG":
+        case "EPS":
+        case "PDF":
+            return Utils.strToArrayBuffer(qrImage);
+        case "PNG":
+            return qrImage.buffer;
+        default:
+            throw new OperationError("Unsupported QR code format.");
+    }
+}

src/core/lib/RSA.mjs

@@ -0,0 +1,17 @@
+/**
+ * RSA resources.
+ *
+ * @author Matt C [me@mitt.dev]
+ * @copyright Crown Copyright 2021
+ * @license Apache-2.0
+ */
+
+import forge from "node-forge";
+
+export const MD_ALGORITHMS = {
+    "SHA-1": forge.md.sha1,
+    "MD5": forge.md.md5,
+    "SHA-256": forge.md.sha256,
+    "SHA-384": forge.md.sha384,
+    "SHA-512": forge.md.sha512,
+};

src/core/lib/SIGABA.mjs

@@ -0,0 +1,502 @@
+/**
+ * Emulation of the SIGABA machine
+ *
+ * @author hettysymes
+ * @copyright hettysymes 2020
+ * @license Apache-2.0
+ */
+
+/**
+ * A set of randomised example SIGABA cipher/control rotors (these rotors are interchangeable). Cipher and control rotors can be referred to as C and R rotors respectively.
+ */
+export const CR_ROTORS = [
+    {name: "Example 1", value: "SRGWANHPJZFXVIDQCEUKBYOLMT"},
+    {name: "Example 2", value: "THQEFSAZVKJYULBODCPXNIMWRG"},
+    {name: "Example 3", value: "XDTUYLEVFNQZBPOGIRCSMHWKAJ"},
+    {name: "Example 4", value: "LOHDMCWUPSTNGVXYFJREQIKBZA"},
+    {name: "Example 5", value: "ERXWNZQIJYLVOFUMSGHTCKPBDA"},
+    {name: "Example 6", value: "FQECYHJIOUMDZVPSLKRTGWXBAN"},
+    {name: "Example 7", value: "TBYIUMKZDJSOPEWXVANHLCFQGR"},
+    {name: "Example 8", value: "QZUPDTFNYIAOMLEBWJXCGHKRSV"},
+    {name: "Example 9", value: "CZWNHEMPOVXLKRSIDGJFYBTQAU"},
+    {name: "Example 10", value: "ENPXJVKYQBFZTICAGMOHWRLDUS"}
+];
+
+/**
+ * A set of randomised example SIGABA index rotors (may be referred to as I rotors).
+ */
+export const I_ROTORS = [
+    {name: "Example 1", value: "6201348957"},
+    {name: "Example 2", value: "6147253089"},
+    {name: "Example 3", value: "8239647510"},
+    {name: "Example 4", value: "7194835260"},
+    {name: "Example 5", value: "4873205916"}
+];
+
+export const NUMBERS = "0123456789".split("");
+
+/**
+ * Converts a letter to uppercase (if it already isn't)
+ *
+ * @param {char} letter - letter to convert to uppercase
+ * @returns {char}
+ */
+export function convToUpperCase(letter) {
+    const charCode = letter.charCodeAt();
+    if (97<=charCode && charCode<=122) {
+        return String.fromCharCode(charCode-32);
+    }
+    return letter;
+}
+
+/**
+ * The SIGABA machine consisting of the 3 rotor banks: cipher, control and index banks.
+ */
+export class SigabaMachine {
+
+    /**
+     * SigabaMachine constructor
+     *
+     * @param {Object[]} cipherRotors - list of CRRotors
+     * @param {Object[]} controlRotors - list of CRRotors
+     * @param {object[]} indexRotors - list of IRotors
+     */
+    constructor(cipherRotors, controlRotors, indexRotors) {
+        this.cipherBank = new CipherBank(cipherRotors);
+        this.controlBank = new ControlBank(controlRotors);
+        this.indexBank = new IndexBank(indexRotors);
+    }
+
+    /**
+     * Steps all the correct rotors in the machine.
+     */
+    step() {
+        const controlOut = this.controlBank.goThroughControl();
+        const indexOut = this.indexBank.goThroughIndex(controlOut);
+        this.cipherBank.step(indexOut);
+    }
+
+    /**
+     * Encrypts a letter. A space is converted to a "Z" before encryption, and a "Z" is converted to an "X". This allows spaces to be encrypted.
+     *
+     * @param {char} letter - letter to encrypt
+     * @returns {char}
+     */
+    encryptLetter(letter) {
+        letter = convToUpperCase(letter);
+        if (letter === " ") {
+            letter = "Z";
+        } else if (letter === "Z") {
+            letter = "X";
+        }
+        const encryptedLetter = this.cipherBank.encrypt(letter);
+        this.step();
+        return encryptedLetter;
+    }
+
+    /**
+     * Decrypts a letter. A letter decrypted as a "Z" is converted to a space before it is output, since spaces are converted to "Z"s before encryption.
+     *
+     * @param {char} letter - letter to decrypt
+     * @returns {char}
+     */
+    decryptLetter(letter) {
+        letter = convToUpperCase(letter);
+        let decryptedLetter = this.cipherBank.decrypt(letter);
+        if (decryptedLetter === "Z") {
+            decryptedLetter = " ";
+        }
+        this.step();
+        return decryptedLetter;
+    }
+
+    /**
+     * Encrypts a message of one or more letters
+     *
+     * @param {string} msg - message to encrypt
+     * @returns {string}
+     */
+    encrypt(msg) {
+        let ciphertext = "";
+        for (const letter of msg) {
+            ciphertext = ciphertext.concat(this.encryptLetter(letter));
+        }
+        return ciphertext;
+    }
+
+    /**
+     * Decrypts a message of one or more letters
+     *
+     * @param {string} msg - message to decrypt
+     * @returns {string}
+     */
+    decrypt(msg) {
+        let plaintext = "";
+        for (const letter of msg) {
+            plaintext = plaintext.concat(this.decryptLetter(letter));
+        }
+        return plaintext;
+    }
+
+}
+
+/**
+ * The cipher rotor bank consists of 5 cipher rotors in either a forward or reversed orientation.
+ */
+export class CipherBank {
+
+    /**
+     * CipherBank constructor
+     *
+     * @param {Object[]} rotors - list of CRRotors
+     */
+    constructor(rotors) {
+        this.rotors = rotors;
+    }
+
+    /**
+     * Encrypts a letter through the cipher rotors (signal goes from left-to-right)
+     *
+     * @param {char} inputPos - the input position of the signal (letter to be encrypted)
+     * @returns {char}
+     */
+    encrypt(inputPos) {
+        for (const rotor of this.rotors) {
+            inputPos = rotor.crypt(inputPos, "leftToRight");
+        }
+        return inputPos;
+    }
+
+    /**
+     * Decrypts a letter through the cipher rotors (signal goes from right-to-left)
+     *
+     * @param {char} inputPos - the input position of the signal (letter to be decrypted)
+     * @returns {char}
+     */
+    decrypt(inputPos) {
+        const revOrderedRotors = [...this.rotors].reverse();
+        for (const rotor of revOrderedRotors) {
+            inputPos = rotor.crypt(inputPos, "rightToLeft");
+        }
+        return inputPos;
+    }
+
+    /**
+     * Step the cipher rotors forward according to the inputs from the index rotors
+     *
+     * @param {number[]} indexInputs - the inputs from the index rotors
+     */
+    step(indexInputs) {
+        const logicDict = {0: [0, 9], 1: [7, 8], 2: [5, 6], 3: [3, 4], 4: [1, 2]};
+        const rotorsToMove = [];
+        for (const key in logicDict) {
+            const item = logicDict[key];
+            for (const i of indexInputs) {
+                if (item.includes(i)) {
+                    rotorsToMove.push(this.rotors[key]);
+                    break;
+                }
+            }
+        }
+        for (const rotor of rotorsToMove) {
+            rotor.step();
+        }
+    }
+
+}
+
+/**
+ * The control rotor bank consists of 5 control rotors in either a forward or reversed orientation. Signals to the control rotor bank always go from right-to-left.
+ */
+export class ControlBank {
+
+    /**
+     * ControlBank constructor. The rotors have been reversed as signals go from right-to-left through the control rotors.
+     *
+     * @param {Object[]} rotors - list of CRRotors
+     */
+    constructor(rotors) {
+        this.rotors = [...rotors].reverse();
+    }
+
+    /**
+     * Encrypts a letter.
+     *
+     * @param {char} inputPos - the input position of the signal
+     * @returns {char}
+     */
+    crypt(inputPos) {
+        for (const rotor of this.rotors) {
+            inputPos = rotor.crypt(inputPos, "rightToLeft");
+        }
+        return inputPos;
+    }
+
+    /**
+     * Gets the outputs of the control rotors. The inputs to the control rotors are always "F", "G", "H" and "I".
+     *
+     * @returns {number[]}
+     */
+    getOutputs() {
+        const outputs = [this.crypt("F"), this.crypt("G"), this.crypt("H"), this.crypt("I")];
+        const logicDict = {1: "B", 2: "C", 3: "DE", 4: "FGH", 5: "IJK", 6: "LMNO", 7: "PQRST", 8: "UVWXYZ", 9: "A"};
+        const numberOutputs = [];
+        for (const key in logicDict) {
+            const item = logicDict[key];
+            for (const output of outputs) {
+                if (item.includes(output)) {
+                    numberOutputs.push(key);
+                    break;
+                }
+            }
+        }
+        return numberOutputs;
+    }
+
+    /**
+     * Steps the control rotors. Only 3 of the control rotors step: one after every encryption, one after every 26, and one after every 26 squared.
+     */
+    step() {
+        const MRotor = this.rotors[1], FRotor = this.rotors[2], SRotor = this.rotors[3];
+        // 14 is the offset of "O" from "A" - the next rotor steps once the previous rotor reaches "O"
+        if (FRotor.state === 14) {
+            if (MRotor.state === 14) {
+                SRotor.step();
+            }
+            MRotor.step();
+        }
+        FRotor.step();
+    }
+
+    /**
+     * The goThroughControl function combines getting the outputs from the control rotor bank and then stepping them.
+     *
+     * @returns {number[]}
+     */
+    goThroughControl() {
+        const outputs = this.getOutputs();
+        this.step();
+        return outputs;
+    }
+
+}
+
+/**
+ * The index rotor bank consists of 5 index rotors all placed in the forwards orientation.
+ */
+export class IndexBank {
+
+    /**
+     * IndexBank constructor
+     *
+     * @param {Object[]} rotors - list of IRotors
+     */
+    constructor(rotors) {
+        this.rotors = rotors;
+    }
+
+    /**
+     * Encrypts a number.
+     *
+     * @param {number} inputPos - the input position of the signal
+     * @returns {number}
+     */
+    crypt(inputPos) {
+        for (const rotor of this.rotors) {
+            inputPos = rotor.crypt(inputPos);
+        }
+        return inputPos;
+    }
+
+    /**
+     * The goThroughIndex function takes the inputs from the control rotor bank and returns the list of outputs after encryption through the index rotors.
+     *
+     * @param {number[]} controlInputs - inputs from the control rotors
+     * @returns {number[]}
+     */
+    goThroughIndex(controlInputs) {
+        const outputs = [];
+        for (const inp of controlInputs) {
+            outputs.push(this.crypt(inp));
+        }
+        return outputs;
+    }
+
+}
+
+/**
+ * Rotor class
+ */
+export class Rotor {
+
+    /**
+     * Rotor constructor
+     *
+     * @param {number[]} wireSetting - the wirings within the rotor: mapping from left-to-right, the index of the number in the list maps onto the number at that index
+     * @param {bool} rev - true if the rotor is reversed, false if it isn't
+     * @param {number} key - the starting position or state of the rotor
+     */
+    constructor(wireSetting, key, rev) {
+        this.state = key;
+        this.numMapping = this.getNumMapping(wireSetting, rev);
+        this.posMapping = this.getPosMapping(rev);
+    }
+
+    /**
+     * Get the number mapping from the wireSetting (only different from wireSetting if rotor is reversed)
+     *
+     * @param {number[]} wireSetting - the wirings within the rotors
+     * @param {bool} rev - true if reversed, false if not
+     * @returns {number[]}
+     */
+    getNumMapping(wireSetting, rev) {
+        if (rev===false) {
+            return wireSetting;
+        } else {
+            const length = wireSetting.length;
+            const tempMapping = new Array(length);
+            for (let i=0; i<length; i++) {
+                tempMapping[wireSetting[i]] = i;
+            }
+            return tempMapping;
+        }
+    }
+
+    /**
+     * Get the position mapping (how the position numbers map onto the numbers of the rotor)
+     *
+     * @param {bool} rev - true if reversed, false if not
+     * @returns {number[]}
+     */
+    getPosMapping(rev) {
+        const length = this.numMapping.length;
+        const posMapping = [];
+        if (rev===false) {
+            for (let i = this.state; i < this.state+length; i++) {
+                let res = i%length;
+                if (res<0) {
+                    res += length;
+                }
+                posMapping.push(res);
+            }
+        } else {
+            for (let i = this.state; i > this.state-length; i--) {
+                let res = i%length;
+                if (res<0) {
+                    res += length;
+                }
+                posMapping.push(res);
+            }
+        }
+        return posMapping;
+    }
+
+    /**
+     * Encrypt/decrypt data. This process is identical to the rotors of cipher machines such as Enigma or Typex.
+     *
+     * @param {number} inputPos - the input position of the signal (the data to encrypt/decrypt)
+     * @param {string} direction - one of "leftToRight" and "rightToLeft", states the direction in which the signal passes through the rotor
+     * @returns {number}
+     */
+    cryptNum(inputPos, direction) {
+        const inpNum = this.posMapping[inputPos];
+        let outNum;
+        if (direction === "leftToRight") {
+            outNum = this.numMapping[inpNum];
+        } else if (direction === "rightToLeft") {
+            outNum = this.numMapping.indexOf(inpNum);
+        }
+        const outPos = this.posMapping.indexOf(outNum);
+        return outPos;
+    }
+
+    /**
+     * Steps the rotor. The number at position 0 will be moved to position 1 etc.
+     */
+    step() {
+        const lastNum = this.posMapping.pop();
+        this.posMapping.splice(0, 0, lastNum);
+        this.state = this.posMapping[0];
+    }
+
+}
+
+/**
+ * A CRRotor is a cipher (C) or control (R) rotor. These rotors are identical and interchangeable. A C or R rotor consists of 26 contacts, one for each letter, and may be put into either a forwards of reversed orientation.
+ */
+export class CRRotor extends Rotor {
+
+    /**
+     * CRRotor constructor
+     *
+     * @param {string} wireSetting - the rotor wirings (string of letters)
+     * @param {char} key - initial state of rotor
+     * @param {bool} rev - true if reversed, false if not
+     */
+    constructor(wireSetting, key, rev=false) {
+        wireSetting = wireSetting.split("").map(CRRotor.letterToNum);
+        super(wireSetting, CRRotor.letterToNum(key), rev);
+    }
+
+    /**
+     * Static function which converts a letter into its number i.e. its offset from the letter "A"
+     *
+     * @param {char} letter - letter to convert to number
+     * @returns {number}
+     */
+    static letterToNum(letter) {
+        return letter.charCodeAt()-65;
+    }
+
+    /**
+     * Static function which converts a number (a letter's offset from "A") into its letter
+     *
+     * @param {number} num - number to convert to letter
+     * @returns {char}
+     */
+    static numToLetter(num) {
+        return String.fromCharCode(num+65);
+    }
+
+    /**
+     * Encrypts/decrypts a letter.
+     *
+     * @param {char} inputPos - the input position of the signal ("A" refers to position 0 etc.)
+     * @param {string} direction - one of "leftToRight" and "rightToLeft"
+     * @returns {char}
+     */
+    crypt(inputPos, direction) {
+        inputPos = CRRotor.letterToNum(inputPos);
+        const outPos = this.cryptNum(inputPos, direction);
+        return CRRotor.numToLetter(outPos);
+    }
+
+}
+
+/**
+ * An IRotor is an index rotor, which consists of 10 contacts each numbered from 0 to 9. Unlike C and R rotors, they cannot be put in the reversed orientation. The index rotors do not step at any point during encryption or decryption.
+ */
+export class IRotor extends Rotor {
+
+    /**
+     * IRotor constructor
+     *
+     * @param {string} wireSetting - the rotor wirings (string of numbers)
+     * @param {char} key - initial state of rotor
+     */
+    constructor(wireSetting, key) {
+        wireSetting = wireSetting.split("").map(Number);
+        super(wireSetting, Number(key), false);
+    }
+
+    /**
+     * Encrypts a number
+     *
+     * @param {number} inputPos - the input position of the signal
+     * @returns {number}
+     */
+    crypt(inputPos) {
+        return this.cryptNum(inputPos, "leftToRight");
+    }
+
+}

src/core/lib/SM4.mjs

@@ -0,0 +1,331 @@
+/**
+ * Complete implementation of SM4 cipher encryption/decryption with
+ * ECB, CBC, CFB, OFB, CTR block modes.
+ * These modes are specified in IETF draft-ribose-cfrg-sm4-09, see:
+ * https://tools.ietf.org/id/draft-ribose-cfrg-sm4-09.html
+ * for details.
+ *
+ * Follows spec from Cryptography Standardization Technical Comittee:
+ * http://www.gmbz.org.cn/upload/2018-04-04/1522788048733065051.pdf
+ *
+ * @author swesven
+ * @copyright 2021
+ * @license Apache-2.0
+ */
+
+import OperationError from "../errors/OperationError.mjs";
+
+/** Number of rounds */
+const NROUNDS = 32;
+
+/** block size in bytes */
+const BLOCKSIZE = 16;
+
+/** The S box, 256 8-bit values */
+const Sbox = [
+    0xd6, 0x90, 0xe9, 0xfe, 0xcc, 0xe1, 0x3d, 0xb7, 0x16, 0xb6, 0x14, 0xc2, 0x28, 0xfb, 0x2c, 0x05,
+    0x2b, 0x67, 0x9a, 0x76, 0x2a, 0xbe, 0x04, 0xc3, 0xaa, 0x44, 0x13, 0x26, 0x49, 0x86, 0x06, 0x99,
+    0x9c, 0x42, 0x50, 0xf4, 0x91, 0xef, 0x98, 0x7a, 0x33, 0x54, 0x0b, 0x43, 0xed, 0xcf, 0xac, 0x62,
+    0xe4, 0xb3, 0x1c, 0xa9, 0xc9, 0x08, 0xe8, 0x95, 0x80, 0xdf, 0x94, 0xfa, 0x75, 0x8f, 0x3f, 0xa6,
+    0x47, 0x07, 0xa7, 0xfc, 0xf3, 0x73, 0x17, 0xba, 0x83, 0x59, 0x3c, 0x19, 0xe6, 0x85, 0x4f, 0xa8,
+    0x68, 0x6b, 0x81, 0xb2, 0x71, 0x64, 0xda, 0x8b, 0xf8, 0xeb, 0x0f, 0x4b, 0x70, 0x56, 0x9d, 0x35,
+    0x1e, 0x24, 0x0e, 0x5e, 0x63, 0x58, 0xd1, 0xa2, 0x25, 0x22, 0x7c, 0x3b, 0x01, 0x21, 0x78, 0x87,
+    0xd4, 0x00, 0x46, 0x57, 0x9f, 0xd3, 0x27, 0x52, 0x4c, 0x36, 0x02, 0xe7, 0xa0, 0xc4, 0xc8, 0x9e,
+    0xea, 0xbf, 0x8a, 0xd2, 0x40, 0xc7, 0x38, 0xb5, 0xa3, 0xf7, 0xf2, 0xce, 0xf9, 0x61, 0x15, 0xa1,
+    0xe0, 0xae, 0x5d, 0xa4, 0x9b, 0x34, 0x1a, 0x55, 0xad, 0x93, 0x32, 0x30, 0xf5, 0x8c, 0xb1, 0xe3,
+    0x1d, 0xf6, 0xe2, 0x2e, 0x82, 0x66, 0xca, 0x60, 0xc0, 0x29, 0x23, 0xab, 0x0d, 0x53, 0x4e, 0x6f,
+    0xd5, 0xdb, 0x37, 0x45, 0xde, 0xfd, 0x8e, 0x2f, 0x03, 0xff, 0x6a, 0x72, 0x6d, 0x6c, 0x5b, 0x51,
+    0x8d, 0x1b, 0xaf, 0x92, 0xbb, 0xdd, 0xbc, 0x7f, 0x11, 0xd9, 0x5c, 0x41, 0x1f, 0x10, 0x5a, 0xd8,
+    0x0a, 0xc1, 0x31, 0x88, 0xa5, 0xcd, 0x7b, 0xbd, 0x2d, 0x74, 0xd0, 0x12, 0xb8, 0xe5, 0xb4, 0xb0,
+    0x89, 0x69, 0x97, 0x4a, 0x0c, 0x96, 0x77, 0x7e, 0x65, 0xb9, 0xf1, 0x09, 0xc5, 0x6e, 0xc6, 0x84,
+    0x18, 0xf0, 0x7d, 0xec, 0x3a, 0xdc, 0x4d, 0x20, 0x79, 0xee, 0x5f, 0x3e, 0xd7, 0xcb, 0x39, 0x48
+];
+
+/** "Fixed parameter CK" used in key expansion */
+const CK = [
+    0x00070e15, 0x1c232a31, 0x383f464d, 0x545b6269,
+    0x70777e85, 0x8c939aa1, 0xa8afb6bd, 0xc4cbd2d9,
+    0xe0e7eef5, 0xfc030a11, 0x181f262d, 0x343b4249,
+    0x50575e65, 0x6c737a81, 0x888f969d, 0xa4abb2b9,
+    0xc0c7ced5, 0xdce3eaf1, 0xf8ff060d, 0x141b2229,
+    0x30373e45, 0x4c535a61, 0x686f767d, 0x848b9299,
+    0xa0a7aeb5, 0xbcc3cad1, 0xd8dfe6ed, 0xf4fb0209,
+    0x10171e25, 0x2c333a41, 0x484f565d, 0x646b7279
+];
+
+/** "System parameter FK" */
+const FK = [0xa3b1bac6, 0x56aa3350, 0x677d9197, 0xb27022dc];
+
+/**
+ * Rotating 32-bit shift left
+ *
+ * (Note that although JS integers are stored in doubles and thus have 53 bits,
+ * the JS bitwise operations are 32-bit)
+ */
+function ROL(i, n) {
+    return (i << n) | (i >>> (32 - n));
+}
+
+/**
+ * Linear transformation L
+ *
+ * @param {integer} b - a 32 bit integer
+ */
+function transformL(b) {
+    /* Replace each of the 4 bytes in b with the value at its offset in the Sbox */
+    b = (Sbox[(b >>> 24) & 0xFF] << 24) | (Sbox[(b >>> 16) & 0xFF] << 16) |
+        (Sbox[(b >>> 8) & 0xFF] << 8) | Sbox[b & 0xFF];
+    /* circular rotate and xor */
+    return b ^ ROL(b, 2) ^ ROL(b, 10) ^ ROL(b, 18) ^ ROL(b, 24);
+}
+
+/**
+ * Linear transformation L'
+ *
+ * @param {integer} b - a 32 bit integer
+ */
+function transformLprime(b) {
+    /* Replace each of the 4 bytes in b with the value at its offset in the Sbox */
+    b = (Sbox[(b >>> 24) & 0xFF] << 24) | (Sbox[(b >>> 16) & 0xFF] << 16) |
+        (Sbox[(b >>> 8) & 0xFF] << 8) | Sbox[b & 0xFF];
+    return b ^ ROL(b, 13) ^ ROL(b, 23); /* circular rotate and XOR */
+}
+
+/**
+ * Initialize the round key
+ */
+function initSM4RoundKey(rawkey) {
+    const K = rawkey.map((a, i) => a ^ FK[i]);    /* K = rawkey ^ FK */
+    const roundKey = [];
+    for (let i = 0; i < 32; i++)
+        roundKey[i] = K[i + 4] = K[i] ^ transformLprime(K[i + 1] ^ K[i + 2] ^ K[i + 3] ^ CK[i]);
+    return roundKey;
+}
+
+/**
+ * Encrypts/decrypts a single block X (4 32-bit values) with a prepared round key.
+ *
+ * @param {intArray} X - A cleartext block.
+ * @param {intArray} roundKey - The round key from initSMRoundKey for encrypting (reversed for decrypting).
+ * @returns {byteArray} - The cipher text.
+ */
+function encryptBlockSM4(X, roundKey) {
+    for (let i = 0; i < NROUNDS; i++)
+        X[i + 4] = X[i] ^ transformL(X[i + 1] ^ X[i + 2] ^ X[i + 3] ^ roundKey[i]);
+    return [X[35], X[34], X[33], X[32]];
+}
+
+/**
+ * Takes 16 bytes from an offset in an array and returns an array of 4 32-bit Big-Endian values.
+ * (DataView won't work portably here as we need Big-Endian)
+ *
+ * @param {byteArray} bArray - the array of bytes
+ * @param {integer} offset - starting offset in the array; 15 bytes must follow it.
+ */
+function bytesToInts(bArray, offs=0) {
+    let offset = offs;
+    const A = (bArray[offset] << 24) | (bArray[offset + 1] << 16) | (bArray[offset + 2] << 8) | bArray[offset + 3];
+    offset += 4;
+    const B = (bArray[offset] << 24) | (bArray[offset + 1] << 16) | (bArray[offset + 2] << 8) | bArray[offset + 3];
+    offset += 4;
+    const C = (bArray[offset] << 24) | (bArray[offset + 1] << 16) | (bArray[offset + 2] << 8) | bArray[offset + 3];
+    offset += 4;
+    const D = (bArray[offset] << 24) | (bArray[offset + 1] << 16) | (bArray[offset + 2] << 8) | bArray[offset + 3];
+    return [A, B, C, D];
+}
+
+/**
+ * Inverse of bytesToInts above; takes an array of 32-bit integers and turns it into an array of bytes.
+ * Again, Big-Endian order.
+ */
+function intsToBytes(ints) {
+    const bArr = [];
+    for (let i = 0; i < ints.length; i++) {
+        bArr.push((ints[i] >> 24) & 0xFF);
+        bArr.push((ints[i] >> 16) & 0xFF);
+        bArr.push((ints[i] >> 8) & 0xFF);
+        bArr.push(ints[i] & 0xFF);
+    }
+    return bArr;
+}
+
+/**
+ * Encrypt using SM4 using a given block cipher mode.
+ *
+ * @param {byteArray} message - The clear text message; any length under 32 Gb or so.
+ * @param {byteArray} key - The cipher key, 16 bytes.
+ * @param {byteArray} iv - The IV or nonce, 16 bytes (not used with ECB mode)
+ * @param {string} mode - The block cipher mode "CBC", "ECB", "CFB", "OFB", "CTR".
+ * @param {boolean} noPadding - Don't add PKCS#7 padding if set.
+ * @returns {byteArray} - The cipher text.
+ */
+export function encryptSM4(message, key, iv, mode="ECB", noPadding=false) {
+    const messageLength = message.length;
+    if (messageLength === 0)
+        return [];
+    const roundKey = initSM4RoundKey(bytesToInts(key, 0));
+
+    /* Pad with PKCS#7 if requested for ECB/CBC else add zeroes (which are sliced off at the end) */
+    let padByte = 0;
+    let nPadding = 16 - (message.length & 0xF);
+    if (mode === "ECB" || mode === "CBC") {
+        if (noPadding) {
+            if (nPadding !== 16)
+                throw new OperationError(`No padding requested in ${mode} mode but input is not a 16-byte multiple.`);
+            nPadding = 0;
+        } else
+            padByte = nPadding;
+    }
+    for (let i = 0; i < nPadding; i++)
+        message.push(padByte);
+
+    const cipherText = [];
+    switch (mode) {
+        case "ECB":
+            for (let i = 0; i < message.length; i += BLOCKSIZE)
+                Array.prototype.push.apply(cipherText, intsToBytes(encryptBlockSM4(bytesToInts(message, i), roundKey)));
+            break;
+        case "CBC":
+            iv = bytesToInts(iv, 0);
+            for (let i = 0; i < message.length; i += BLOCKSIZE) {
+                const block = bytesToInts(message, i);
+                block[0] ^= iv[0]; block[1] ^= iv[1];
+                block[2] ^= iv[2]; block[3] ^= iv[3];
+                iv = encryptBlockSM4(block, roundKey);
+                Array.prototype.push.apply(cipherText, intsToBytes(iv));
+            }
+            break;
+        case "CFB":
+            iv = bytesToInts(iv, 0);
+            for (let i = 0; i < message.length; i += BLOCKSIZE) {
+                iv = encryptBlockSM4(iv, roundKey);
+                const block = bytesToInts(message, i);
+                block[0] ^= iv[0]; block[1] ^= iv[1];
+                block[2] ^= iv[2]; block[3] ^= iv[3];
+                Array.prototype.push.apply(cipherText, intsToBytes(block));
+                iv = block;
+            }
+            break;
+        case "OFB":
+            iv = bytesToInts(iv, 0);
+            for (let i = 0; i < message.length; i += BLOCKSIZE) {
+                iv = encryptBlockSM4(iv, roundKey);
+                const block = bytesToInts(message, i);
+                block[0] ^= iv[0]; block[1] ^= iv[1];
+                block[2] ^= iv[2]; block[3] ^= iv[3];
+                Array.prototype.push.apply(cipherText, intsToBytes(block));
+            }
+            break;
+        case "CTR":
+            iv = bytesToInts(iv, 0);
+            for (let i = 0; i < message.length; i += BLOCKSIZE) {
+                let iv2 = [...iv]; /* containing the IV + counter */
+                iv2[3] += (i >> 4);/* Using a 32 bit counter here. 64 Gb encrypts should be enough for everyone. */
+                iv2 = encryptBlockSM4(iv2, roundKey);
+                const block = bytesToInts(message, i);
+                block[0] ^= iv2[0]; block[1] ^= iv2[1];
+                block[2] ^= iv2[2]; block[3] ^= iv2[3];
+                Array.prototype.push.apply(cipherText, intsToBytes(block));
+            }
+            break;
+        default:
+            throw new OperationError("Invalid block cipher mode: "+mode);
+    }
+    if (mode !== "ECB" && mode !== "CBC")
+        return cipherText.slice(0, messageLength);
+    return cipherText;
+}
+
+/**
+ * Decrypt using SM4 using a given block cipher mode.
+ *
+ * @param {byteArray} cipherText - The ciphertext
+ * @param {byteArray} key - The cipher key, 16 bytes.
+ * @param {byteArray} iv - The IV or nonce, 16 bytes (not used with ECB mode)
+ * @param {string} mode - The block cipher mode "CBC", "ECB", "CFB", "OFB", "CTR"
+ * @param {boolean] ignorePadding - If true, ignore padding issues in ECB/CBC mode.
+ * @returns {byteArray} - The cipher text.
+ */
+export function decryptSM4(cipherText, key, iv, mode="ECB", ignorePadding=false) {
+    const originalLength = cipherText.length;
+    if (originalLength === 0)
+        return [];
+    let roundKey = initSM4RoundKey(bytesToInts(key, 0));
+
+    if (mode === "ECB" || mode === "CBC") {
+        /* Init decryption key */
+        roundKey = roundKey.reverse();
+        if ((originalLength & 0xF) !== 0 && !ignorePadding)
+            throw new OperationError(`With ECB or CBC modes, the input must be divisible into 16 byte blocks. (${cipherText.length & 0xF} bytes extra)`);
+    } else { /* Pad dummy bytes for other modes, chop them off at the end */
+        while ((cipherText.length & 0xF) !== 0)
+            cipherText.push(0);
+    }
+
+    const clearText = [];
+    switch (mode) {
+        case "ECB":
+            for (let i = 0; i < cipherText.length; i += BLOCKSIZE)
+                Array.prototype.push.apply(clearText, intsToBytes(encryptBlockSM4(bytesToInts(cipherText, i), roundKey)));
+            break;
+        case "CBC":
+            iv = bytesToInts(iv, 0);
+            for (let i = 0; i < cipherText.length; i += BLOCKSIZE) {
+                const block = encryptBlockSM4(bytesToInts(cipherText, i), roundKey);
+                block[0] ^= iv[0]; block[1] ^= iv[1];
+                block[2] ^= iv[2]; block[3] ^= iv[3];
+                Array.prototype.push.apply(clearText, intsToBytes(block));
+                iv = bytesToInts(cipherText, i);
+            }
+            break;
+        case "CFB":
+            iv = bytesToInts(iv, 0);
+            for (let i = 0; i < cipherText.length; i += BLOCKSIZE) {
+                iv = encryptBlockSM4(iv, roundKey);
+                const block = bytesToInts(cipherText, i);
+                block[0] ^= iv[0]; block[1] ^= iv[1];
+                block[2] ^= iv[2]; block[3] ^= iv[3];
+                Array.prototype.push.apply(clearText, intsToBytes(block));
+                iv = bytesToInts(cipherText, i);
+            }
+            break;
+        case "OFB":
+            iv = bytesToInts(iv, 0);
+            for (let i = 0; i < cipherText.length; i += BLOCKSIZE) {
+                iv = encryptBlockSM4(iv, roundKey);
+                const block = bytesToInts(cipherText, i);
+                block[0] ^= iv[0]; block[1] ^= iv[1];
+                block[2] ^= iv[2]; block[3] ^= iv[3];
+                Array.prototype.push.apply(clearText, intsToBytes(block));
+            }
+            break;
+        case "CTR":
+            iv = bytesToInts(iv, 0);
+            for (let i = 0; i < cipherText.length; i += BLOCKSIZE) {
+                let iv2 = [...iv]; /* containing the IV + counter */
+                iv2[3] += (i >> 4);/* Using a 32 bit counter here. 64 Gb encrypts should be enough for everyone. */
+                iv2 = encryptBlockSM4(iv2, roundKey);
+                const block = bytesToInts(cipherText, i);
+                block[0] ^= iv2[0]; block[1] ^= iv2[1];
+                block[2] ^= iv2[2]; block[3] ^= iv2[3];
+                Array.prototype.push.apply(clearText, intsToBytes(block));
+            }
+            break;
+        default:
+            throw new OperationError(`Invalid block cipher mode: ${mode}`);
+    }
+    /* Check PKCS#7 padding */
+    if (mode === "ECB" || mode === "CBC") {
+        if (ignorePadding)
+            return clearText;
+        const padByte = clearText[clearText.length - 1];
+        if (padByte > 16)
+            throw new OperationError("Invalid PKCS#7 padding.");
+        for (let i = 0; i < padByte; i++)
+            if (clearText[clearText.length -i - 1] !== padByte)
+                throw new OperationError("Invalid PKCS#7 padding.");
+        return clearText.slice(0, clearText.length - padByte);
+    }
+    return clearText.slice(0, originalLength);
+}
+

src/core/lib/Sort.mjs

@@ -0,0 +1,117 @@
+/**
+ * Sorting functions
+ *
+ * @author n1474335 [n1474335@gmail.com]
+ * @copyright Crown Copyright 2022
+ * @license Apache-2.0
+ *
+ */
+
+/**
+ * Comparison operation for sorting of strings ignoring case.
+ *
+ * @param {string} a
+ * @param {string} b
+ * @returns {number}
+ */
+export function caseInsensitiveSort(a, b) {
+    return a.toLowerCase().localeCompare(b.toLowerCase());
+}
+
+
+/**
+ * Comparison operation for sorting of IPv4 addresses.
+ *
+ * @param {string} a
+ * @param {string} b
+ * @returns {number}
+ */
+export function ipSort(a, b) {
+    let a_ = a.split("."),
+        b_ = b.split(".");
+
+    a_ = a_[0] * 0x1000000 + a_[1] * 0x10000 + a_[2] * 0x100 + a_[3] * 1;
+    b_ = b_[0] * 0x1000000 + b_[1] * 0x10000 + b_[2] * 0x100 + b_[3] * 1;
+
+    if (isNaN(a_) && !isNaN(b_)) return 1;
+    if (!isNaN(a_) && isNaN(b_)) return -1;
+    if (isNaN(a_) && isNaN(b_)) return a.localeCompare(b);
+
+    return a_ - b_;
+}
+
+/**
+ * Comparison operation for sorting of numeric values.
+ *
+ * @author Chris van Marle
+ * @param {string} a
+ * @param {string} b
+ * @returns {number}
+ */
+export function numericSort(a, b) {
+    const a_ = a.split(/([^\d]+)/),
+        b_ = b.split(/([^\d]+)/);
+
+    for (let i = 0; i < a_.length && i < b.length; ++i) {
+        if (isNaN(a_[i]) && !isNaN(b_[i])) return 1; // Numbers after non-numbers
+        if (!isNaN(a_[i]) && isNaN(b_[i])) return -1;
+        if (isNaN(a_[i]) && isNaN(b_[i])) {
+            const ret = a_[i].localeCompare(b_[i]); // Compare strings
+            if (ret !== 0) return ret;
+        }
+        if (!isNaN(a_[i]) && !isNaN(b_[i])) { // Compare numbers
+            if (a_[i] - b_[i] !== 0) return a_[i] - b_[i];
+        }
+    }
+
+    return a.localeCompare(b);
+}
+
+/**
+ * Comparison operation for sorting of hexadecimal values.
+ *
+ * @author Chris van Marle
+ * @param {string} a
+ * @param {string} b
+ * @returns {number}
+ */
+export function hexadecimalSort(a, b) {
+    let a_ = a.split(/([^\da-f]+)/i),
+        b_ = b.split(/([^\da-f]+)/i);
+
+    a_ = a_.map(v => {
+        const t = parseInt(v, 16);
+        return isNaN(t) ? v : t;
+    });
+
+    b_ = b_.map(v => {
+        const t = parseInt(v, 16);
+        return isNaN(t) ? v : t;
+    });
+
+    for (let i = 0; i < a_.length && i < b.length; ++i) {
+        if (isNaN(a_[i]) && !isNaN(b_[i])) return 1; // Numbers after non-numbers
+        if (!isNaN(a_[i]) && isNaN(b_[i])) return -1;
+        if (isNaN(a_[i]) && isNaN(b_[i])) {
+            const ret = a_[i].localeCompare(b_[i]); // Compare strings
+            if (ret !== 0) return ret;
+        }
+        if (!isNaN(a_[i]) && !isNaN(b_[i])) { // Compare numbers
+            if (a_[i] - b_[i] !== 0) return a_[i] - b_[i];
+        }
+    }
+
+    return a.localeCompare(b);
+}
+
+/**
+ * Comparison operation for sorting by length
+ *
+ * @param {string} a
+ * @param {string} b
+ * @returns {number}
+ */
+export function lengthSort(a, b) {
+    return a.length - b.length;
+}
+

src/core/lib/Stream.mjs

@@ -0,0 +1,327 @@
+/**
+ * Stream class for parsing binary protocols.
+ *
+ * @author n1474335 [n1474335@gmail.com]
+ * @author tlwr [toby@toby.codes]
+ * @copyright Crown Copyright 2018
+ * @license Apache-2.0
+ *
+ */
+
+/**
+ * A Stream can be used to traverse a binary blob, interpreting sections of it
+ * as various data types.
+ */
+export default class Stream {
+
+    /**
+     * Stream constructor.
+     *
+     * @param {Uint8Array} input
+     */
+    constructor(input) {
+        this.bytes = input;
+        this.length = this.bytes.length;
+        this.position = 0;
+        this.bitPos = 0;
+    }
+
+    /**
+     * Get a number of bytes from the current position, or all remaining bytes.
+     *
+     * @param {number} [numBytes=null]
+     * @returns {Uint8Array}
+     */
+    getBytes(numBytes=null) {
+        if (this.position > this.length) return undefined;
+
+        const newPosition = numBytes !== null ?
+            this.position + numBytes :
+            this.length;
+        const bytes = this.bytes.slice(this.position, newPosition);
+        this.position = newPosition;
+        this.bitPos = 0;
+        return bytes;
+    }
+
+    /**
+     * Interpret the following bytes as a string, stopping at the next null byte or
+     * the supplied limit.
+     *
+     * @param {number} [numBytes=-1]
+     * @returns {string}
+     */
+    readString(numBytes=-1) {
+        if (this.position > this.length) return undefined;
+
+        if (numBytes === -1) numBytes = this.length - this.position;
+
+        let result = "";
+        for (let i = this.position; i < this.position + numBytes; i++) {
+            const currentByte = this.bytes[i];
+            if (currentByte === 0) break;
+            result += String.fromCharCode(currentByte);
+        }
+        this.position += numBytes;
+        this.bitPos = 0;
+        return result;
+    }
+
+    /**
+     * Interpret the following bytes as an integer in big or little endian.
+     *
+     * @param {number} numBytes
+     * @param {string} [endianness="be"]
+     * @returns {number}
+     */
+    readInt(numBytes, endianness="be") {
+        if (this.position > this.length) return undefined;
+
+        let val = 0;
+        if (endianness === "be") {
+            for (let i = this.position; i < this.position + numBytes; i++) {
+                val = val << 8;
+                val |= this.bytes[i];
+            }
+        } else {
+            for (let i = this.position + numBytes - 1; i >= this.position; i--) {
+                val = val << 8;
+                val |= this.bytes[i];
+            }
+        }
+        this.position += numBytes;
+        this.bitPos = 0;
+        return val;
+    }
+
+    /**
+     * Reads a number of bits from the buffer in big or little endian.
+     *
+     * @param {number} numBits
+     * @param {string} [endianness="be"]
+     * @returns {number}
+     */
+    readBits(numBits, endianness="be") {
+        if (this.position > this.length) return undefined;
+
+        let bitBuf = 0,
+            bitBufLen = 0;
+
+        // Add remaining bits from current byte
+        bitBuf = this.bytes[this.position++] & bitMask(this.bitPos);
+        if (endianness !== "be") bitBuf >>>= this.bitPos;
+        bitBufLen = 8 - this.bitPos;
+        this.bitPos = 0;
+
+        // Not enough bits yet
+        while (bitBufLen < numBits) {
+            if (endianness === "be")
+                bitBuf = (bitBuf << bitBufLen) | this.bytes[this.position++];
+            else
+                bitBuf |= this.bytes[this.position++] << bitBufLen;
+            bitBufLen += 8;
+        }
+
+        // Reverse back to numBits
+        if (bitBufLen > numBits) {
+            const excess = bitBufLen - numBits;
+            if (endianness === "be")
+                bitBuf >>>= excess;
+            else
+                bitBuf &= (1 << numBits) - 1;
+            bitBufLen -= excess;
+            this.position--;
+            this.bitPos = 8 - excess;
+        }
+
+        return bitBuf;
+
+        /**
+         * Calculates the bit mask based on the current bit position.
+         *
+         * @param {number} bitPos
+         * @returns {number} The bit mask
+         */
+        function bitMask(bitPos) {
+            return endianness === "be" ?
+                (1 << (8 - bitPos)) - 1 :
+                256 - (1 << bitPos);
+        }
+    }
+
+    /**
+     * Consume the stream until we reach the specified byte or sequence of bytes.
+     *
+     * @param {number|List<number>} val
+     */
+    continueUntil(val) {
+        if (this.position > this.length) return;
+
+        this.bitPos = 0;
+
+        if (typeof val === "number") {
+            while (++this.position < this.length && this.bytes[this.position] !== val) {
+                continue;
+            }
+            return;
+        }
+
+        // val is an array
+
+        /**
+         * Builds the skip forward table from the value to be searched.
+         *
+         * @param {Uint8Array} val
+         * @param {Number} len
+         * @returns {Uint8Array}
+         */
+        function preprocess(val, len) {
+            const skiptable = new Array();
+            val.forEach((element, index) => {
+                skiptable[element] = len - index;
+            });
+            return skiptable;
+        }
+
+        const length = val.length;
+        const initial = val[length-1];
+        this.position = length;
+
+        // Get the skip table.
+        const skiptable = preprocess(val, length);
+        let found;
+
+        while (this.position < this.length) {
+            // Until we hit the final element of val in the stream.
+            while ((this.position < this.length) && (this.bytes[this.position++] !== initial));
+
+            found = true;
+
+            // Loop through the elements comparing them to val.
+            for (let x = length-1; x >= 0; x--) {
+                if (this.bytes[this.position - length + x] !== val[x]) {
+                    found = false;
+
+                    // If element is not equal to val's element then jump forward by the correct amount.
+                    this.position += skiptable[val[x]];
+                    break;
+                }
+            }
+            if (found) {
+                this.position -= length;
+                break;
+            }
+        }
+    }
+
+
+    /**
+     * Consume bytes if they match the supplied value.
+     *
+     * @param {Number} val
+     */
+    consumeWhile(val) {
+        while (this.position < this.length) {
+            if (this.bytes[this.position] !== val) {
+                break;
+            }
+            this.position++;
+        }
+        this.bitPos = 0;
+    }
+
+    /**
+     * Consume the next byte if it matches the supplied value.
+     *
+     * @param {number} val
+     */
+    consumeIf(val) {
+        if (this.bytes[this.position] === val) {
+            this.position++;
+            this.bitPos = 0;
+        }
+    }
+
+    /**
+     * Move forwards through the stream by the specified number of bytes.
+     *
+     * @param {number} numBytes
+     */
+    moveForwardsBy(numBytes) {
+        const pos = this.position + numBytes;
+        if (pos < 0 || pos > this.length)
+            throw new Error("Cannot move to position " + pos + " in stream. Out of bounds.");
+        this.position = pos;
+        this.bitPos = 0;
+    }
+
+    /**
+     * Move backwards through the stream by the specified number of bytes.
+     *
+     * @param {number} numBytes
+     */
+    moveBackwardsBy(numBytes) {
+        const pos = this.position - numBytes;
+        if (pos < 0 || pos > this.length)
+            throw new Error("Cannot move to position " + pos + " in stream. Out of bounds.");
+        this.position = pos;
+        this.bitPos = 0;
+    }
+
+    /**
+     * Move backwards through the strem by the specified number of bits.
+     *
+     * @param {number} numBits
+     */
+    moveBackwardsByBits(numBits) {
+        if (numBits <= this.bitPos) {
+            this.bitPos -= numBits;
+        } else {
+            if (this.bitPos > 0) {
+                numBits -= this.bitPos;
+                this.bitPos = 0;
+            }
+
+            while (numBits > 0) {
+                this.moveBackwardsBy(1);
+                this.bitPos = 8;
+                this.moveBackwardsByBits(numBits);
+                numBits -= 8;
+            }
+        }
+    }
+
+    /**
+     * Move to a specified position in the stream.
+     *
+     * @param {number} pos
+     */
+    moveTo(pos) {
+        if (pos < 0 || pos > this.length)
+            throw new Error("Cannot move to position " + pos + " in stream. Out of bounds.");
+        this.position = pos;
+        this.bitPos = 0;
+    }
+
+    /**
+     * Returns true if there are more bytes left in the stream.
+     *
+     * @returns {boolean}
+     */
+    hasMore() {
+        return this.position < this.length;
+    }
+
+    /**
+     * Returns a slice of the stream up to the current position.
+     *
+     * @param {number} [start=0]
+     * @param {number} [finish=this.position]
+     * @returns {Uint8Array}
+     */
+    carve(start=0, finish=this.position) {
+        if (this.bitPos > 0) finish++;
+        return this.bytes.slice(start, finish);
+    }
+
+}