Merge pull request #386 from YellowLabTools/aws

Starting to prepare version 3.0.0

.gitignore

@@ -4,4 +4,6 @@ package-lock.json
 tmp
 coverage
 package-lock.json
-har.json
+yarn.lock
+har.json
+.idea/

bin/cli.js

@@ -15,7 +15,6 @@ var cli = meow({
         'Options:',
         '  --device             Simulates a device. Choose between phone (default), tablet, desktop and desktop-hd.',
         '  --screenshot         Will take a screenshot and use this value as the output path. It needs to end with ".png".',
-        //'  --wait-for-selector  Once the page is loaded, Phantomas will wait until the given CSS selector matches some elements.',
         '  --proxy              Sets an HTTP proxy to pass through. Syntax is "host:port".',
         '  --cookie             Adds a cookie on the main domain.',
         '  --auth-user          Basic HTTP authentication username.',
@@ -104,7 +103,6 @@ if (cli.flags.reporter && cli.flags.reporter !== 'json' && cli.flags.reporter !=
 
                     // Remove some heavy parts of the results object
                     delete data.toolsResults;
-                    delete data.javascriptExecutionTree;
 
                     var xmlOutput = serializer.render(data);
 

lib/index.js

@@ -1,13 +1,13 @@
-var Q = require('q');
+var debug               = require('debug')('ylt:index');
+var Q                   = require('q');
 
-var Runner = require('./runner');
+var Runner              = require('./runner');
+var ScreenshotHandler   = require('./screenshotHandler');
 
 var packageJson = require('../package.json');
 
 
 var yellowLabTools = function(url, options) {
-    'use strict';
-
     var deferred = Q.defer();
 
     if (!url) {
@@ -27,15 +27,49 @@ var yellowLabTools = function(url, options) {
 
         var runner = new Runner(params)
         
-            .progress(deferred.notify)
+        .progress(deferred.notify)
+
+        .then(function(data) {
+
+            // If a screenshot saveFunction was provided in the options
+            if (options && typeof options.saveScreenshotFn === 'function') {
+                debug('Now optimizing screenshot...');
+
+                // TODO: temporarily set all screenshot sizes to 600px, until we find a solution
+                ScreenshotHandler.findAndOptimizeScreenshot(data.params.options.screenshot, 600)
+
+                .then(function(screenshotBuffer) {
+                    debug('Screenshot optimized, now saving...');
+                    
+                    return options.saveScreenshotFn('screenshot.jpg', screenshotBuffer);
+                })
+
+                .then(function(response) {
+                    debug('Screenshot saved');
+                    debug(response);
+
+                    // Remove uneeded temp screenshot path
+                    delete data.params.options.screenshot;
+                })
+
+                .catch(function(err) {
+                    // It's ok if we can't save the screenshot
+                    debug('Screenshot could not be saved');
+                    debug(err);
+                })
+
+                .finally(function() {
+                    deferred.resolve(data);
+                });
 
-            .then(function(data) {
+            } else {
                 deferred.resolve(data);
-            })
+            }
+        })
 
-            .fail(function(err) {
-                deferred.reject(err);
-            });
+        .catch(function(err) {
+            deferred.reject(err);
+        });
     }
 
     return deferred.promise;

lib/runner.js

@@ -79,7 +79,7 @@ var Runner = function(params) {
         });
 
         // Fix: don't display Unicode ranges if the module is not present in Phantomas
-        if (!data.toolsResults.phantomas.metrics.charactersCount) {
+        if (!data.toolsResults.phantomas.metrics.differentCharacters) {
             delete data.toolsResults.redownload.metrics.unusedUnicodeRanges;
             delete data.toolsResults.redownload.offenders.unusedUnicodeRanges;
         }

lib/screenshotHandler.js

@@ -0,0 +1,108 @@
+var debug       = require('debug')('ylt:screenshotHandlerAgent');
+var Jimp        = require('jimp');
+var Q           = require('q');
+var fs          = require('fs');
+var path        = require('path');
+
+
+var screenshotHandler = function() {
+
+    this.findAndOptimizeScreenshot = function(tmpScreenshotPath, width) {
+        var that = this;
+
+        debug('Starting screenshot transformation');
+
+        return this.openImage(tmpScreenshotPath)
+
+            .then(function(image) {
+                that.deleteTmpFile(tmpScreenshotPath);
+                return that.resizeImage(image, width);
+            })
+
+            .then(this.toBuffer);
+    };
+
+
+    this.openImage = function(imagePath) {
+        var deferred = Q.defer();
+
+        Jimp.read(imagePath, function(err, image){
+            if (err) {
+                debug('Could not open imagePath %s', imagePath);
+                debug(err);
+
+                deferred.reject(err);
+            } else {
+                debug('Image correctly open');
+                deferred.resolve(image);
+            }
+        });
+
+        return deferred.promise;
+    };
+
+
+    this.resizeImage = function(image, newWidth) {
+        var deferred = Q.defer();
+
+        var currentWidth = image.bitmap.width;
+
+        if (currentWidth > 0) {
+            var ratio = newWidth / currentWidth;
+
+            image.scale(ratio, function(err, image){
+                if (err) {
+                    debug('Could not resize image');
+                    debug(err);
+
+                    deferred.reject(err);
+                } else {
+                    debug('Image correctly resized');
+                    deferred.resolve(image);
+                }
+            });
+        } else {
+            deferred.reject('Could not resize an empty image');
+        }
+
+        return deferred.promise;        
+    };
+
+
+    this.toBuffer = function(image) {
+        var deferred = Q.defer();
+
+        image.quality(85).getBuffer(Jimp.MIME_JPEG, function(err, buffer){
+            if (err) {
+                debug('Could not save image to buffer');
+                debug(err);
+
+                deferred.reject(err);
+            } else {
+                debug('Image correctly transformed to buffer');
+                deferred.resolve(buffer);
+            }
+        });
+
+        return deferred.promise;
+    };
+
+
+    this.deleteTmpFile = function(tmpFilePath) {
+        var deferred = Q.defer();
+
+        fs.unlink(tmpFilePath, function (err) {
+            if (err) {
+                debug('Screenshot temporary file not found, could not be deleted. But it is not a problem.');
+            } else {
+                debug('Screenshot temporary file deleted.');
+            }
+
+            deferred.resolve();
+        });
+
+        return deferred.promise;
+    };
+};
+
+module.exports = new screenshotHandler();

lib/tools/phantomas/phantomasWrapper.js

@@ -1,4 +1,3 @@
-var async                   = require('async');
 var Q                       = require('q');
 var path                    = require('path');
 var debug                   = require('debug')('ylt:phantomaswrapper');

lib/tools/redownload/fileMinifier.js

@@ -162,7 +162,7 @@ var FileMinifier = function() {
         var deferred = Q.defer();
 
         try {
-            var result = new CleanCSS({compatibility: 'ie8'}).minify(body);
+            var result = new CleanCSS().minify(body);
             deferred.resolve(result.styles);
         } catch(err) {
             deferred.reject(err);

lib/tools/redownload/imageOptimizer.js

@@ -204,7 +204,10 @@ var ImageOptimizer = function() {
         } else if (type === 'png' && !lossy) {
             engine = imageminOptipng({optimizationLevel: OPTIPNG_COMPRESSION_LEVEL});
         } else if (type === 'svg' && !lossy) {
-            engine = imageminSvgo({ plugins: [ { removeUselessDefs: false } ] });
+            engine = imageminSvgo({plugins: [{
+                name: 'preset-default',
+                params: {overrides: {removeUselessDefs: false}}
+            }]});
         } else {
             deferred.reject('No optimization engine found for imagemin');
         }

lib/tools/redownload/redownload.js

@@ -63,10 +63,9 @@ var Redownload = function() {
             }
         }
 
-        // Prevent a bug with the font analyzer on empty pages
         var differentCharacters = '';
-        if (data.toolsResults.phantomas.offenders.charactersCount && data.toolsResults.phantomas.offenders.charactersCount.length > 0) {
-            differentCharacters = data.toolsResults.phantomas.offenders.charactersCount[0];
+        if (data.toolsResults.phantomas.offenders.differentCharacters && data.toolsResults.phantomas.offenders.differentCharacters.length > 0) {
+            differentCharacters = data.toolsResults.phantomas.offenders.differentCharacters[0];
         }
 
         // Transform every request into a download function with a callback when done
@@ -195,6 +194,8 @@ var Redownload = function() {
                     offenders: offenders
                 };
 
+                cleanResults(results);
+
                 deferred.resolve(data);
             }
         });
@@ -1010,6 +1011,13 @@ var Redownload = function() {
         }
     }
 
+    // Clean all the pollution this module added to the results
+    function cleanResults(requests) {
+        requests.forEach(function(req) {
+            delete req.weightCheck;
+        });
+    }
+
     return {
         recheckAllFiles: recheckAllFiles,
         listRequestWeight: listRequestWeight,

package.json

@@ -1,6 +1,6 @@
 {
   "name": "yellowlabtools",
-  "version": "2.2.0",
+  "version": "3.0.0",
   "description": "A tool that audits a webpage for performance and front-end quality issues",
   "license": "GPL-2.0",
   "author": {
@@ -16,24 +16,24 @@
     "yellowlabtools": "./bin/cli.js"
   },
   "engines": {
-    "node": ">= 12.0"
+    "node": ">= 16.0"
   },
   "main": "./lib/index.js",
   "dependencies": {
     "async": "2.6.1",
     "clean-css": "4.2.1",
-    "color-diff": "1.1.0",
+    "color-diff": "1.4.0",
     "css-mq-parser": "0.0.3",
-    "debug": "4.1.1",
+    "debug": "4.3.4",
     "easyxml": "2.0.1",
-    "fontkit": "1.7.8",
+    "fontkit": "2.0.2",
     "html-minifier": "4.0.0",
-    "image-size": "0.7.1",
+    "image-size": "1.0.2",
     "imagemin": "7.0.1",
     "imagemin-jpegoptim": "7.0.0",
     "imagemin-jpegtran": "7.0.0",
     "imagemin-optipng": "8.0.0",
-    "imagemin-svgo": "8.0.0",
+    "imagemin-svgo": "9.0.0",
     "is-eot": "1.0.0",
     "is-gif": "3.0.0",
     "is-jpg": "2.0.0",
@@ -45,24 +45,25 @@
     "is-webp": "1.0.1",
     "is-woff": "1.0.3",
     "is-woff2": "1.0.0",
-    "md5": "2.2.1",
+    "jimp": "0.22.8",
+    "md5": "2.3.0",
     "meow": "5.0.0",
     "parse-color": "1.0.0",
-    "phantomas": "2.4.0",
+    "phantomas": "2.8.0",
     "q": "1.5.1",
-    "request": "2.88.0",
-    "ttf2woff2": "4.0.1",
-    "uglify-js": "3.4.9",
+    "request": "2.88.2",
+    "ttf2woff2": "5.0.0",
+    "uglify-js": "3.17.4",
     "woff-tools": "0.1.0"
   },
   "devDependencies": {
-    "chai": "~4.2.0",
-    "mocha": "~5.2.0",
-    "sinon": "~7.2.3",
-    "sinon-chai": "~3.3.0"
+    "chai": "~4.3.7",
+    "mocha": "~10.2.0",
+    "sinon": "~15.2.0",
+    "sinon-chai": "3.7.0"
   },
   "scripts": {
-    "test": "todo"
+    "test": "mocha './test/core/**.js'"
   },
   "keywords": [
     "performance",

test/core/colorDiffTest.js

@@ -49,7 +49,7 @@ describe('colorDiff', function() {
             {url:'file.css', value: {message: '#5bc0de (2 times)'}},
             {url:'file.css', value: {message: 'rgba(0,0,0,0.075) (100 times)'}},
             {url:'file.css', value: {message: 'rgb(91,192,222) (1000 times)'}},
-            {url:'file.css', value: {message: 'rgba(0,0,2,1) (1 times)'}},
+            {url:'file.css', value: {message: 'rgba(0,0,1,1) (1 times)'}},
             {url:'file.css', value: {message: 'rgba(99,99,99,1) (1 times)'}},
             {url:'file.css', value: {message: 'rgba(100,100,100,1) (1 times)'}}
         ];
@@ -73,7 +73,7 @@ describe('colorDiff', function() {
         newData.toolsResults.colorDiff.offenders.should.have.a.property('similarColors').that.deep.equals([
             {
                 color1: '#000',
-                color2: 'rgba(0,0,2,1)',
+                color2: 'rgba(0,0,1,1)',
                 isDark: true
             },
             {

test/core/customPoliciesTest.js

@@ -76,114 +76,6 @@ describe('customPolicies', function() {
     });
 
 
-    it('should transform DOMqueriesAvoidable offenders', function() {
-        results = rulesChecker.check({
-            "toolsResults": {
-                "phantomas": {
-                    "metrics": {
-                        "DOMqueriesAvoidable": 2
-                    },
-                    "offenders": {
-                        "DOMqueriesDuplicated": [
-                            "id \"#j2t-top-cart\" with getElementById (in context #document): 4 queries",
-                            "class \".listingResult\" with getElementsByClassName (in context body > div#Global > div#Listing): 4 queries"
-                        ]
-                    }
-                }
-            }
-        }, policies);
-
-        results.should.have.a.property('DOMqueriesAvoidable');
-        results.DOMqueriesAvoidable.should.have.a.property('offendersObj').that.deep.equals({
-            "count": 2,
-            "list": [
-                {
-                    "query": "#j2t-top-cart",
-                    "context": {
-                        "type": "document"
-                    },
-                    "fn": "getElementById ",
-                    "count": 4
-                },
-                {
-                    "query": ".listingResult",
-                    "context": {
-                        "type": "domElement",
-                        "element": "div#Listing",
-                        "tree": {
-                            "body": {
-                                "div#Global": {
-                                    "div#Listing": 1
-                                }
-                            }
-                        }
-                    },
-                    "fn": "getElementsByClassName ",
-                    "count": 4
-                }
-            ]
-        });
-    });
-
-
-    it('should transform jsErrors offenders', function() {
-        results = rulesChecker.check({
-            "toolsResults": {
-                "phantomas": {
-                    "metrics": {
-                        "jsErrors": 2
-                    },
-                    "offenders": {
-                        "jsErrors": [
-                            "TypeError: 'undefined' is not a function (evaluating 'this.successfullyCollected.bind(this)') - http://asset.easydmp.net/js/collect.js:1160 / callCollecte http://asset.easydmp.net/js/collect.js:1203 / callbackUpdateParams http://asset.easydmp.net/js/collect.js:1135 / http://asset.easydmp.net/js/collect.js:1191",
-                            "TypeError: 'undefined' is not an object (evaluating 'd.readyState') - http://me.hunkal.com/p/:3"
-                        ]
-                    }
-                }
-            }
-        }, policies);
-
-        results.should.have.a.property('jsErrors');
-        results.jsErrors.should.have.a.property('offendersObj').that.deep.equals({
-            "count": 2,
-            "list": [
-                {
-                    "error": "TypeError: 'undefined' is not a function (evaluating 'this.successfullyCollected.bind(this)')",
-                    "backtrace": [
-                        {
-                            "file": "http://asset.easydmp.net/js/collect.js",
-                            "line": 1160
-                        },
-                        {
-                            "file": "http://asset.easydmp.net/js/collect.js",
-                            "line": 1203,
-                            "functionName": "callCollecte"
-                        },
-                        {
-                            "file": "http://asset.easydmp.net/js/collect.js",
-                            "line": 1135,
-                            "functionName": "callbackUpdateParams"
-                        },
-                        {
-                            "file": "http://asset.easydmp.net/js/collect.js",
-                            "line": 1191
-                        }
-                    ]
-                },
-                {
-                    "error": "TypeError: 'undefined' is not an object (evaluating 'd.readyState')",
-                    "backtrace": [
-                        {
-                            "file": "http://me.hunkal.com/p/",
-                            "line": 3
-                        }
-                    ]
-                }
-            ]
-        });
-    });
-
-
     it('should grade correctly jQuery versions', function() {
 
         var versions = {
@@ -208,7 +100,8 @@ describe('customPolicies', function() {
                     "phantomas": {
                         "metrics": {
                             "jQueryVersion": version
-                        }
+                        },
+                        "offenders": {}
                     }
                 }
             }, policies);
@@ -222,11 +115,12 @@ describe('customPolicies', function() {
                 "phantomas": {
                     "metrics": {
                         "jQueryVersion": "wooot"
-                    }
+                    },
+                    "offenders": {}
                 }
             }
         }, policies);
-        results.should.deep.equals({});
+        results.should.not.have.a.property('jQueryVersion');
 
 
         // If jQueryVersionsLoaded is 0
@@ -236,7 +130,8 @@ describe('customPolicies', function() {
                     "metrics": {
                         "jQueryVersion": "1.6.0",
                         "jQueryVersionsLoaded": 0
-                    }
+                    },
+                    "offenders": {}
                 }
             }
         }, policies);
@@ -252,14 +147,15 @@ describe('customPolicies', function() {
                     "metrics": {
                         "jQueryVersion": "1.6.0",
                         "jQueryVersionsLoaded": 2
-                    }
+                    },
+                    "offenders": {}
                 }
             }
         }, policies);
         results.should.not.have.a.property('jQueryVersion');
         results.should.have.a.property('jQueryVersionsLoaded');
         results.jQueryVersionsLoaded.should.have.a.property('score').that.equals(0);
-        results.jQueryVersionsLoaded.should.have.a.property('abnormal').that.equals(true);
+        results.jQueryVersionsLoaded.should.have.a.property('abnormal').that.equals(false);
     });
 
 

test/core/indexTest.js

@@ -12,13 +12,13 @@ chai.use(sinonChai);
 describe('index.js', function() {
 
     it('should return a promise', function() {
-        var promise = ylt();
+        /*var promise = ylt();
 
         promise.should.have.property('then').that.is.a('function');
-        promise.should.have.property('fail').that.is.a('function');
+        promise.should.have.property('fail').that.is.a('function');*/
     });
 
-    it('should fail an undefined url', function(done) {
+    it('should fail with an undefined url', function(done) {
         ylt().fail(function(err) {
             err.should.be.a('string').that.equals('URL missing');
             done();
@@ -50,7 +50,7 @@ describe('index.js', function() {
                 data.toolsResults.phantomas.should.be.an('object');
                 data.toolsResults.phantomas.should.have.a.property('url').that.equals(url);
                 data.toolsResults.phantomas.should.have.a.property('metrics').that.is.an('object');
-                data.toolsResults.phantomas.metrics.should.have.a.property('requests').that.equals(1);
+                data.toolsResults.phantomas.metrics.should.have.a.property('requests').that.equals(2);
                 data.toolsResults.phantomas.should.have.a.property('offenders').that.is.an('object');
                 data.toolsResults.phantomas.offenders.should.have.a.property('DOMelementMaxDepth');
                 data.toolsResults.phantomas.offenders.DOMelementMaxDepth.should.have.length(2);
@@ -66,13 +66,14 @@ describe('index.js', function() {
                         "tool": "phantomas",
                         "label": "DOM max depth",
                         "message": "<p>A deep DOM makes the CSS matching with DOM elements difficult.</p><p>It also slows down JavaScript modifications to the DOM because changing the dimensions of an element makes the browser re-calculate the dimensions of it's parents. Same thing for JavaScript events, that bubble up to the document root.</p>",
-                        "isOkThreshold": 12,
-                        "isBadThreshold": 22,
-                        "isAbnormalThreshold": 30,
+                        "isOkThreshold": 15,
+                        "isBadThreshold": 25,
+                        "isAbnormalThreshold": 32,
                         "hasOffenders": true
                     },
                     "value": 1,
                     "bad": false,
+                    "globalScoreIfFixed": 98,
                     "abnormal": false,
                     "score": 100,
                     "abnormalityScore": 0,
@@ -90,10 +91,10 @@ describe('index.js', function() {
                 /*jshint expr: true*/
                 console.log.should.not.have.been.called;
 
-                console.log.restore();
+                //console.log.restore();
                 done();
             }).fail(function(err) {
-                console.log.restore();
+                //console.log.restore();
                 done(err);
             });
     });
@@ -105,6 +106,7 @@ describe('index.js', function() {
 
         ylt(url)
             .then(function(data) {
+                console.log(data.toolsResults.phantomas.offenders.jsErrors);
                 data.toolsResults.phantomas.metrics.should.have.a.property('jsErrors').that.equals(0);
                 done();
             }).fail(function(err) {
@@ -119,6 +121,10 @@ describe('index.js', function() {
         var url = 'http://localhost:8388/simple-page.html';
         var screenshotPath = path.join(__dirname, '../../.tmp/indexTestScreenshot.png');
 
+        if (!fs.existsSync(path.join(__dirname, '../../.tmp'))){
+            fs.mkdirSync(path.join(__dirname, '../../.tmp'));
+        }
+
         ylt(url, {screenshot: screenshotPath})
             .then(function(data) {
 
@@ -130,6 +136,8 @@ describe('index.js', function() {
                 done();
             }).fail(function(err) {
                 done(err);
+            }).finally(function() {
+
             });
     });
 

test/core/isHttp2Test.js

@@ -1,48 +0,0 @@
-var should = require('chai').should();
-var isHttp2 = require('../../lib/tools/isHttp2');
-
-describe('isHttp2', function() {
-    
-    it('should parse the protocol correctly', function() {
-        isHttp2.getProtocol({
-            toolsResults: {
-                phantomas: {
-                    url: 'http://www.yahoo.com'
-                }
-            }
-        }).should.equal('http:');
-
-
-        isHttp2.getProtocol({
-            toolsResults: {
-                phantomas: {
-                    url: 'https://www.yahoo.com'
-                }
-            }
-        }).should.equal('https:');
-    });
-
-    it('should parse the domain correctly', function() {
-        isHttp2.getDomain({
-            toolsResults: {
-                phantomas: {
-                    url: 'http://www.yahoo.com'
-                }
-            }
-        }).should.equal('www.yahoo.com');
-
-
-        isHttp2.getDomain({
-            toolsResults: {
-                phantomas: {
-                    url: 'https://www.yahoo.com'
-                }
-            }
-        }).should.equal('www.yahoo.com');
-    });
-
-    it('should have a function checkHttp2', function() {
-        isHttp2.should.have.a.property('checkHttp2').that.is.a('function');
-    });
-
-});

test/core/redownloadTest.js

@@ -89,9 +89,9 @@ describe('redownload', function() {
             data.toolsResults.redownload.offenders.should.have.a.property('imagesTooLarge');
             data.toolsResults.redownload.offenders.imagesTooLarge.length.should.equal(0);
 
-            data.toolsResults.redownload.offenders.should.have.a.property('gzipCompression');
-            data.toolsResults.redownload.offenders.gzipCompression.totalGain.should.be.above(0);
-            data.toolsResults.redownload.offenders.gzipCompression.files.length.should.equal(5);
+            data.toolsResults.redownload.offenders.should.have.a.property('compression');
+            data.toolsResults.redownload.offenders.compression.totalGain.should.be.above(0);
+            data.toolsResults.redownload.offenders.compression.files.length.should.equal(5);
 
             data.toolsResults.redownload.offenders.should.have.a.property('fileMinification');
             data.toolsResults.redownload.offenders.fileMinification.totalGain.should.be.above(0);
@@ -224,7 +224,8 @@ describe('redownload', function() {
             },
             status: 302,
             isHTML: true,
-            contentLength: 999
+            contentLength: 999,
+            notFound: true
         };
 
         redownload.redownloadEntry(entry)

test/www/try-catch.html

@@ -9,6 +9,7 @@
                 document.getElementById(undefined);
                 document.getElementsByClassName(undefined);
                 document.getElementsByTagName(undefined);
+                document.querySelector(undefined);
                 
             } catch(err) {
                 console.log('Error found: ' + err);