Merge branch 'develop' into webp-avif

lib/index.js

@@ -33,11 +33,10 @@ var yellowLabTools = function(url, options) {
 
             // If a screenshot saveFunction was provided in the options
             if (options && typeof options.saveScreenshotFn === 'function') {
-                const screenshotTmpPath = data.params.options.screenshot;
                 debug('Now optimizing screenshot...');
 
                 // TODO: temporarily set all screenshot sizes to 600px, until we find a solution
-                ScreenshotHandler.findAndOptimizeScreenshot(screenshotTmpPath, 600)
+                ScreenshotHandler.findAndOptimizeScreenshot(data.params.options.screenshot, 600)
 
                 .then(function(screenshotBuffer) {
                     debug('Screenshot optimized, now saving...');
@@ -51,7 +50,6 @@ var yellowLabTools = function(url, options) {
 
                     // Remove uneeded temp screenshot path
                     delete data.params.options.screenshot;
-                    return ScreenshotHandler.deleteTmpFile(screenshotTmpPath);
                 })
 
                 .catch(function(err) {

lib/metadata/policies.js

@@ -729,7 +729,7 @@ var policies = {
     "totalWeight": {
         "tool": "redownload",
         "label": "Total weight",
-        "message": "<p>The weight is of course very important if you want the page to load fast. Try to stay under 1.5MB.</p>",
+        "message": "<p>The weight is of course very important if you want the page to load fast. Try to stay under 1MB, which is already very long to download over a slow connection.</p>",
         "isOkThreshold": 1572864,
         "isBadThreshold": 3145728,
         "isAbnormalThreshold": 5242880,

lib/screenshotHandler.js

@@ -1,19 +1,90 @@
-var debug       = require('debug')('ylt:screenshotHandler');
-var sharp       = require('sharp');
+var debug       = require('debug')('ylt:screenshotHandlerAgent');
+var Jimp        = require('jimp');
 var Q           = require('q');
 var fs          = require('fs');
 var path        = require('path');
 
-// Disable sharp cache to reduce the "disk is full" error on Amazon Lambda
-sharp.cache(false);
 
 var screenshotHandler = function() {
 
-    this.findAndOptimizeScreenshot = async function(tmpScreenshotPath, width) {
-        return sharp(tmpScreenshotPath)
-            .resize({width: 600})
-            .jpeg({quality: 85})
-            .toBuffer();
+    this.findAndOptimizeScreenshot = function(tmpScreenshotPath, width) {
+        var that = this;
+
+        debug('Starting screenshot transformation');
+
+        return this.openImage(tmpScreenshotPath)
+
+            .then(function(image) {
+                that.deleteTmpFile(tmpScreenshotPath);
+                return that.resizeImage(image, width);
+            })
+
+            .then(this.toBuffer);
+    };
+
+
+    this.openImage = function(imagePath) {
+        var deferred = Q.defer();
+
+        Jimp.read(imagePath, function(err, image){
+            if (err) {
+                debug('Could not open imagePath %s', imagePath);
+                debug(err);
+
+                deferred.reject(err);
+            } else {
+                debug('Image correctly open');
+                deferred.resolve(image);
+            }
+        });
+
+        return deferred.promise;
+    };
+
+
+    this.resizeImage = function(image, newWidth) {
+        var deferred = Q.defer();
+
+        var currentWidth = image.bitmap.width;
+
+        if (currentWidth > 0) {
+            var ratio = newWidth / currentWidth;
+
+            image.scale(ratio, function(err, image){
+                if (err) {
+                    debug('Could not resize image');
+                    debug(err);
+
+                    deferred.reject(err);
+                } else {
+                    debug('Image correctly resized');
+                    deferred.resolve(image);
+                }
+            });
+        } else {
+            deferred.reject('Could not resize an empty image');
+        }
+
+        return deferred.promise;        
+    };
+
+
+    this.toBuffer = function(image) {
+        var deferred = Q.defer();
+
+        image.quality(85).getBuffer(Jimp.MIME_JPEG, function(err, buffer){
+            if (err) {
+                debug('Could not save image to buffer');
+                debug(err);
+
+                deferred.reject(err);
+            } else {
+                debug('Image correctly transformed to buffer');
+                deferred.resolve(buffer);
+            }
+        });
+
+        return deferred.promise;
     };
 
 

lib/tools/redownload/contentTypeChecker.js

@@ -1,12 +1,20 @@
-var debug       = require('debug')('ylt:contentTypeChecker');
-var Q           = require('q');
-var FileType    = require('file-type');
-var isSvg       = require('is-svg');
-var isJson      = require('is-json');
+var debug   = require('debug')('ylt:contentTypeChecker');
+var Q       = require('q');
+var isJpg   = require('is-jpg');
+var isPng   = require('is-png');
+var isSvg   = require('is-svg');
+var isGif   = require('is-gif');
+var isWebp  = require('is-webp');
+var isWoff  = require('is-woff');
+var isWoff2 = require('is-woff2');
+var isOtf   = require('is-otf');
+var isTtf   = require('is-ttf');
+var isEot   = require('is-eot');
+var isJson  = require('is-json');
 
 var ContentTypeChecker = function() {
 
-    async function checkContentType(entry) {
+    function checkContentType(entry) {
         var deferred = Q.defer();
         
         // Setting isSomething values:
@@ -47,12 +55,12 @@ var ContentTypeChecker = function() {
             var foundType;
 
             try {
-                foundType = await findContentType(entry.weightCheck.bodyBuffer);
+                foundType = findContentType(entry.weightCheck.bodyBuffer);
 
                 // If it's an image or a font, then rewrite.
                 if (foundType !== null && (foundType.type === 'image' || foundType.type === 'webfont' || foundType.type === 'json')) {
                     if (foundType.type !== entry.type) {
-                        debug('Content type %s is wrong for %s. It should be %s.', entry.type, entry.url, foundType.type);
+                        debug('Content type %s is wrong for %s. It should be %s.', entry.type, entry.ulr, foundType.type);
                     }
                     rewriteContentType(entry, foundType);
                 }
@@ -68,21 +76,52 @@ var ContentTypeChecker = function() {
         return deferred.promise;
     }
 
-    async function findContentType(bodyBuffer) {
+    function findContentType(bodyBuffer) {
         var bodyStr = bodyBuffer.toString();
 
+        if (isJpg(bodyBuffer)) {
+            return contentTypes.jpeg;
+        }
+
+        if (isPng(bodyBuffer)) {
+            return contentTypes.png;
+        }
+
         // https://github.com/sindresorhus/is-svg/issues/7
         if (/<svg/.test(bodyStr) && isSvg(bodyStr)) {
             return contentTypes.svg;
         }
 
-        if (isJson(bodyStr)) {
-            return contentTypes.json;
+        if (isGif(bodyBuffer)) {
+            return contentTypes.gif;
+        }
+
+        if (isWebp(bodyBuffer)) {
+            return contentTypes.webp;
+        }
+
+        if (isWoff(bodyBuffer)) {
+            return contentTypes.woff;
+        }
+
+        if (isWoff2(bodyBuffer)) {
+            return contentTypes.woff2;
+        }
+
+        if (isOtf(bodyBuffer)) {
+            return contentTypes.otf;
+        }
+
+        if (isTtf(bodyBuffer)) {
+            return contentTypes.ttf;
         }
 
-        const type = await FileType.fromBuffer(bodyBuffer);
-        if (type && type.ext && contentTypes[type.ext]) {
-            return contentTypes[type.ext];
+        if (isEot(bodyBuffer)) {
+            return contentTypes.eot;
+        }
+
+        if (isJson(bodyStr)) {
+            return contentTypes.json;
         }
 
         return null;
@@ -107,7 +146,7 @@ var ContentTypeChecker = function() {
     }
 
     var contentTypes = {
-        jpg: {
+        jpeg: {
             type: 'image',
             mimes: ['image/jpeg'],
             updateFn: function(entry) {
@@ -148,14 +187,6 @@ var ContentTypeChecker = function() {
                 entry.isImage = true;
             }
         },
-        avif: {
-            type: 'image',
-            mimes: ['image/avif'],
-            updateFn: function(entry) {
-                entry.type = 'image';
-                entry.isImage = true;
-            }
-        },
         woff: {
             type: 'webfont',
             mimes: ['application/x-font-woff', 'application/font-woff', 'font/woff'],

lib/tools/redownload/imageDimensions.js

@@ -43,10 +43,6 @@ var ImageDimensions = function() {
         return entry.isImage && entry.contentType === 'image/png';
     }
 
-    function isWebP(entry) {
-        return entry.isImage && entry.contentType === 'image/webp';
-    }
-
     return {
         getDimensions: getDimensions
     };

lib/tools/redownload/redownload.js

@@ -819,9 +819,9 @@ var Redownload = function() {
 
         debug('Downloading %s', entry.url);
 
-        // Always add compression and webp/avif headers before sending, in case the server listens to them
+        // Always add compression and webp headers before sending, in case the server listens to them
         var reqHeaders = [];
-        reqHeaders['Accept'] = '*/*,image/webp,image/avif';
+        reqHeaders['Accept'] = '*/*,image/webp';
         reqHeaders['Accept-Encoding'] = 'gzip, deflate, br';
         reqHeaders['Connection'] = 'keep-alive';
         reqHeaders['User-Agent'] = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.182 Safari/537.36';

package.json

@@ -26,7 +26,6 @@
     "css-mq-parser": "0.0.3",
     "debug": "4.3.4",
     "easyxml": "2.0.1",
-    "file-type": "16.5.3",
     "fontkit": "2.0.2",
     "html-minifier": "4.0.0",
     "image-size": "1.0.2",
@@ -35,8 +34,18 @@
     "imagemin-jpegtran": "7.0.0",
     "imagemin-optipng": "8.0.0",
     "imagemin-svgo": "9.0.0",
+    "is-eot": "1.0.0",
+    "is-gif": "3.0.0",
+    "is-jpg": "2.0.0",
     "is-json": "2.0.1",
+    "is-otf": "0.1.2",
+    "is-png": "1.1.0",
     "is-svg": "3.0.0",
+    "is-ttf": "0.2.2",
+    "is-webp": "1.0.1",
+    "is-woff": "1.0.3",
+    "is-woff2": "1.0.0",
+    "jimp": "0.22.8",
     "md5": "2.3.0",
     "meow": "5.0.0",
     "parse-color": "1.0.0",

test/core/contentTypeCheckerTest.js

@@ -10,11 +10,11 @@ describe('contentTypeChecker', function() {
     var svgImageContent = fs.readFileSync(path.resolve(__dirname, '../www/svg-image.svg'));
     var cssFileContent = fs.readFileSync(path.resolve(__dirname, '../www/unminified-stylesheet.css'));
     
-    it('detect the right content type', async function() {
-        (await contentTypeChecker.findContentType(jpgImageContent)).mimes.should.deep.equal(['image/jpeg']);
-        (await contentTypeChecker.findContentType(pngImageContent)).mimes.should.deep.equal(['image/png']);
-        (await contentTypeChecker.findContentType(svgImageContent)).mimes.should.deep.equal(['image/svg+xml']);
-        should.equal(await contentTypeChecker.findContentType(cssFileContent), null);
+    it('detect the right content type', function() {
+        contentTypeChecker.findContentType(jpgImageContent).mimes.should.deep.equal(['image/jpeg']);
+        contentTypeChecker.findContentType(pngImageContent).mimes.should.deep.equal(['image/png']);
+        contentTypeChecker.findContentType(svgImageContent).mimes.should.deep.equal(['image/svg+xml']);
+        should.equal(contentTypeChecker.findContentType(cssFileContent), null);
     });
 
 });

test/core/redownloadTest.js

@@ -160,9 +160,18 @@ describe('redownload', function() {
         redownload.redownloadEntry(entry)
 
         .then(function(newEntry) {
+
             newEntry.weightCheck.bodySize.should.equal(4193);
             newEntry.weightCheck.bodyBuffer.should.deep.equal(fileContent);
-            done();
+
+            // Opening the image in jimp to check if the format is good
+            var Jimp = require('jimp');
+            Jimp.read(newEntry.weightCheck.bodyBuffer, function(err, image) {
+                image.bitmap.width.should.equal(620);
+                image.bitmap.height.should.equal(104);
+                done(err);
+            });
+
         })
 
         .fail(function(err) {