Tests now display a progress bar and report long running tests

.travis.yml

@@ -11,7 +11,7 @@ before_script:
   - export NODE_OPTIONS=--max_old_space_size=2048
 script:
   - grunt lint
-  - grunt test
+  - npm test
   - grunt testnodeconsumer
   - grunt prod --msg="$COMPILE_MSG"
   - xvfb-run --server-args="-screen 0 1200x800x24" grunt testui

Gruntfile.js

@@ -36,11 +36,10 @@ module.exports = function (grunt) {
             "clean:node", "clean:config", "clean:nodeConfig", "exec:generateConfig", "exec:generateNodeIndex"
         ]);
 
-    grunt.registerTask("test",
-        "A task which runs all the operation tests in the tests directory.",
+    grunt.registerTask("configTests",
+        "A task which configures config files in preparation for tests to be run. Use `npm tests` to run tests.",
         [
-            "clean:config", "clean:nodeConfig", "exec:generateConfig", "exec:generateNodeIndex",
-            "exec:nodeTests", "exec:opTests"
+            "clean:config", "clean:nodeConfig", "exec:generateConfig", "exec:generateNodeIndex"
         ]);
 
     grunt.registerTask("testui",
@@ -55,7 +54,6 @@ module.exports = function (grunt) {
         "Lints the code base",
         ["eslint", "exec:repoSize"]);
 
-    grunt.registerTask("tests", "test");
     grunt.registerTask("lint", "eslint");
 
     grunt.registerTask("findModules",
@@ -385,15 +383,9 @@ module.exports = function (grunt) {
                 ]),
                 sync: true
             },
-            opTests: {
-                command: "node --experimental-modules --no-warnings --no-deprecation tests/operations/index.mjs"
-            },
             browserTests: {
                 command: "./node_modules/.bin/nightwatch --env prod"
             },
-            nodeTests: {
-                command: "node --experimental-modules --no-warnings --no-deprecation tests/node/index.mjs"
-            },
             setupNodeConsumers: {
                 command: chainCommands([
                     "echo '\n--- Testing node consumers ---'",

package-lock.json

@@ -4413,6 +4413,49 @@
         "restore-cursor": "^3.1.0"
       }
     },
+    "cli-progress": {
+      "version": "3.6.0",
+      "resolved": "https://registry.npmjs.org/cli-progress/-/cli-progress-3.6.0.tgz",
+      "integrity": "sha512-elg6jkiDedYrvwqWSae2FGvtbMo37Lo04oI9jJ5cI43Ge3jrDPWzeL3axv7MgBLYHDY/kGio/CXa49m4MWMrNw==",
+      "dev": true,
+      "requires": {
+        "colors": "^1.1.2",
+        "string-width": "^2.1.1"
+      },
+      "dependencies": {
+        "ansi-regex": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-3.0.0.tgz",
+          "integrity": "sha1-7QMXwyIGT3lGbAKWa922Bas32Zg=",
+          "dev": true
+        },
+        "is-fullwidth-code-point": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz",
+          "integrity": "sha1-o7MKXE8ZkYMWeqq5O+764937ZU8=",
+          "dev": true
+        },
+        "string-width": {
+          "version": "2.1.1",
+          "resolved": "https://registry.npmjs.org/string-width/-/string-width-2.1.1.tgz",
+          "integrity": "sha512-nOqH59deCq9SRHlxq1Aw85Jnt4w6KvLKqWVik6oA9ZklXLNIOlqg4F2yrT1MVaTjAqvVwdfeZ7w7aCvJD7ugkw==",
+          "dev": true,
+          "requires": {
+            "is-fullwidth-code-point": "^2.0.0",
+            "strip-ansi": "^4.0.0"
+          }
+        },
+        "strip-ansi": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-4.0.0.tgz",
+          "integrity": "sha1-qEeQIusaw2iocTibY1JixQXuNo8=",
+          "dev": true,
+          "requires": {
+            "ansi-regex": "^3.0.0"
+          }
+        }
+      }
+    },
     "cli-spinners": {
       "version": "2.2.0",
       "resolved": "https://registry.npmjs.org/cli-spinners/-/cli-spinners-2.2.0.tgz",
@@ -12204,6 +12247,14 @@
         "request": "^2.81.0",
         "request-progress": "^2.0.1",
         "which": "^1.2.10"
+      },
+      "dependencies": {
+        "progress": {
+          "version": "1.1.8",
+          "resolved": "https://registry.npmjs.org/progress/-/progress-1.1.8.tgz",
+          "integrity": "sha1-4mDHj2Fhzdmw5WzD4Khd4Xx6V74=",
+          "dev": true
+        }
       }
     },
     "phin": {
@@ -12589,11 +12640,6 @@
       "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==",
       "dev": true
     },
-    "progress": {
-      "version": "1.1.8",
-      "resolved": "https://registry.npmjs.org/progress/-/progress-1.1.8.tgz",
-      "integrity": "sha1-4mDHj2Fhzdmw5WzD4Khd4Xx6V74="
-    },
     "promise-inflight": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/promise-inflight/-/promise-inflight-1.0.1.tgz",
@@ -15027,6 +15073,13 @@
         "more-entropy": ">=0.0.7",
         "progress": "~1.1.2",
         "uglify-js": "^3.1.9"
+      },
+      "dependencies": {
+        "progress": {
+          "version": "1.1.8",
+          "resolved": "https://registry.npmjs.org/progress/-/progress-1.1.8.tgz",
+          "integrity": "sha1-4mDHj2Fhzdmw5WzD4Khd4Xx6V74="
+        }
       }
     },
     "true-case-path": {

package.json

@@ -44,6 +44,7 @@
     "babel-loader": "^8.0.6",
     "babel-plugin-dynamic-import-node": "^2.3.0",
     "chromedriver": "^80.0.1",
+    "cli-progress": "^3.6.0",
     "colors": "^1.4.0",
     "copy-webpack-plugin": "^5.0.5",
     "css-loader": "^3.2.1",
@@ -159,7 +160,7 @@
     "start": "grunt dev",
     "build": "grunt prod",
     "repl": "node src/node/repl.js",
-    "test": "grunt test",
+    "test": "grunt configTests && node --experimental-modules --no-warnings --no-deprecation tests/node/index.mjs && node --experimental-modules --no-warnings --no-deprecation tests/operations/index.mjs",
     "test-node-consumer": "grunt testnodeconsumer",
     "testui": "grunt testui",
     "testuidev": "npx nightwatch --env=dev",

src/core/Utils.mjs

@@ -1335,7 +1335,7 @@ export function sendStatusMessage(msg) {
         self.sendStatusMessage(msg);
     else if (isWebEnvironment())
         app.alert(msg, 10000);
-    else if (isNodeEnvironment())
+    else if (isNodeEnvironment() && !global.TESTING)
         // eslint-disable-next-line no-console
         console.debug(msg);
 }

src/node/api.mjs

@@ -282,11 +282,11 @@ export function help(input) {
         .map(result => result.hydrated);
 
     if (matches && matches.length) {
-        console.log(`${matches.length} result${matches.length > 1 ? "s" : ""} found.`);
+        // console.log(`${matches.length} result${matches.length > 1 ? "s" : ""} found.`);
         return matches;
     }
 
-    console.log("No results found.");
+    // console.log("No results found.");
     return null;
 }
 

tests/lib/TestRegister.mjs

@@ -10,6 +10,8 @@
  * @license Apache-2.0
  */
 import Chef from "../../src/core/Chef.mjs";
+import Utils from "../../src/core/Utils.mjs";
+import cliProgress from "cli-progress";
 
 /**
  * Object to store and run the list of tests.
@@ -47,68 +49,99 @@ class TestRegister {
     /**
      * Runs all the tests in the register.
      */
-    runTests () {
-        console.log("Running tests...");
-        return Promise.all(
-            this.tests.map(function(test, i) {
-                const chef = new Chef();
-
-                return chef.bake(
-                    test.input,
-                    test.recipeConfig,
-                    {},
-                    0,
-                    false
-                ).then(function(result) {
-                    const ret = {
-                        test: test,
-                        status: null,
-                        output: null,
-                    };
-
-                    if (result.error) {
-                        if (test.expectedError) {
-                            ret.status = "passing";
-                        } else {
-                            ret.status = "erroring";
-                            ret.output = result.error.displayStr;
-                        }
-                    } else {
-                        if (test.expectedError) {
-                            ret.status = "failing";
-                            ret.output = "Expected an error but did not receive one.";
-                        } else if (result.result === test.expectedOutput) {
-                            ret.status = "passing";
-                        } else if ("expectedMatch" in test && test.expectedMatch.test(result.result)) {
-                            ret.status = "passing";
-                        } else {
-                            ret.status = "failing";
-                            const expected = test.expectedOutput ? test.expectedOutput :
-                                test.expectedMatch ? test.expectedMatch.toString() : "unknown";
-                            ret.output = [
-                                "Expected",
-                                "\t" + expected.replace(/\n/g, "\n\t"),
-                                "Received",
-                                "\t" + result.result.replace(/\n/g, "\n\t"),
-                            ].join("\n");
-                        }
-                    }
-
-                    return ret;
-                });
-            })
-        );
+    async runTests () {
+        const progBar = new cliProgress.SingleBar({
+            format: formatter,
+            stopOnComplete: true
+        }, cliProgress.Presets.shades_classic);
+        const testResults = [];
+
+        console.log("Running operation tests...");
+        progBar.start(this.tests.length, 0, {
+            msg: "Setting up"
+        });
+
+        for (const test of this.tests) {
+            progBar.update(testResults.length, {
+                msg: test.name
+            });
+
+            const chef = new Chef();
+            const result = await chef.bake(
+                test.input,
+                test.recipeConfig,
+                {},
+                0,
+                false
+            );
+
+            const ret = {
+                test: test,
+                status: null,
+                output: null,
+                duration: result.duration
+            };
+
+            if (result.error) {
+                if (test.expectedError) {
+                    ret.status = "passing";
+                } else {
+                    ret.status = "erroring";
+                    ret.output = result.error.displayStr;
+                }
+            } else {
+                if (test.expectedError) {
+                    ret.status = "failing";
+                    ret.output = "Expected an error but did not receive one.";
+                } else if (result.result === test.expectedOutput) {
+                    ret.status = "passing";
+                } else if ("expectedMatch" in test && test.expectedMatch.test(result.result)) {
+                    ret.status = "passing";
+                } else {
+                    ret.status = "failing";
+                    const expected = test.expectedOutput ? test.expectedOutput :
+                        test.expectedMatch ? test.expectedMatch.toString() : "unknown";
+                    ret.output = [
+                        "Expected",
+                        "\t" + expected.replace(/\n/g, "\n\t"),
+                        "Received",
+                        "\t" + result.result.replace(/\n/g, "\n\t"),
+                    ].join("\n");
+                }
+            }
+
+            testResults.push(ret);
+            progBar.increment();
+        }
+
+        return testResults;
     }
 
     /**
      * Run all api related tests and wrap results in report format
      */
-    runApiTests() {
-        return Promise.all(this.apiTests.map(async function(test, i) {
+    async runApiTests() {
+        const progBar = new cliProgress.SingleBar({
+            format: formatter,
+            stopOnComplete: true
+        }, cliProgress.Presets.shades_classic);
+        const testResults = [];
+
+        console.log("Running Node API tests...");
+        progBar.start(this.apiTests.length, 0, {
+            msg: "Setting up"
+        });
+
+        global.TESTING = true;
+        for (const test of this.apiTests) {
+            progBar.update(testResults.length, {
+                msg: test.name
+            });
+
             const result = {
                 test: test,
                 status: null,
-                output: null,
+                output: null
             };
             try {
                 await test.run();
@@ -117,10 +150,37 @@ class TestRegister {
                 result.status = "erroring";
                 result.output = e.message;
             }
-            return result;
-        }));
+
+            testResults.push(result);
+            progBar.increment();
+        }
+
+        return testResults;
     }
 }
 
+
+/**
+ * Formatter for the progress bar
+ *
+ * @param {Object} options
+ * @param {Object} params
+ * @param {Object} payload
+ * @returns {string}
+ */
+function formatter(options, params, payload) {
+    const bar = options.barCompleteString.substr(0, Math.round(params.progress * options.barsize)) +
+        options.barIncompleteString.substr(0, Math.round((1-params.progress) * options.barsize));
+
+    const percentage = Math.floor(params.progress * 100),
+        duration = Math.floor((Date.now() - params.startTime) / 1000);
+
+    let testName = payload.msg ? payload.msg : "";
+    if (params.value >= params.total) testName = "Tests completed";
+    testName = Utils.truncate(testName, 25).padEnd(25, " ");
+
+    return `${testName} ${bar} ${params.value}/${params.total} | ${percentage}% | Duration: ${duration}s`;
+}
+
 // Export an instance to make a singleton
 export default new TestRegister();

tests/lib/utils.mjs

@@ -33,6 +33,10 @@ function handleTestResult(testStatus, testResult) {
     testStatus.allTestsPassing = testStatus.allTestsPassing && testResult.status === "passing";
     testStatus.counts[testResult.status] = (testStatus.counts[testResult.status] || 0) + 1;
     testStatus.counts.total += 1;
+
+    if (testResult.duration > 2000) {
+        console.log(`'${testResult.test.name}' took ${(testResult.duration / 1000).toFixed(2)}s to complete`);
+    }
 }
 
 /**
@@ -42,8 +46,6 @@ function handleTestResult(testStatus, testResult) {
  * @param {Object[]} results - results from TestRegister
  */
 export function logTestReport(testStatus, results) {
-    console.log("Tests completed.");
-
     results.forEach(r => handleTestResult(testStatus, r));
 
     console.log();
@@ -80,8 +82,9 @@ export function logTestReport(testStatus, results) {
  * Fail if the process takes longer than 60 seconds.
  */
 export function setLongTestFailure() {
+    const timeLimit = 60;
     setTimeout(function() {
-        console.log("Tests took longer than 60 seconds to run, returning.");
+        console.log(`Tests took longer than ${timeLimit} seconds to run, returning.`);
         process.exit(1);
-    }, 60 * 1000);
+    }, timeLimit * 1000);
 }

tests/node/index.mjs

@@ -35,6 +35,7 @@ setLongTestFailure();
 
 const logOpsTestReport = logTestReport.bind(null, testStatus);
 
-TestRegister.runApiTests()
-    .then(logOpsTestReport);
-
+(async function() {
+    const results = await TestRegister.runApiTests();
+    logOpsTestReport(results);
+})();

tests/operations/index.mjs

@@ -116,5 +116,8 @@ setLongTestFailure();
 
 const logOpsTestReport = logTestReport.bind(null, testStatus);
 
-TestRegister.runTests()
-    .then(logOpsTestReport);
+(async function() {
+    const results = await TestRegister.runTests();
+    logOpsTestReport(results);
+})();
+