Add a JPEG optimizer

Gruntfile.js

@@ -143,7 +143,7 @@ module.exports = function(grunt) {
                 options: {
                     reporter: 'spec',
                 },
-                src: ['test/core/phantomasWrapperTest.js']
+                src: ['test/core/imageOptimizerTest.js']
             },
             coverage: {
                 options: {

front/src/js/controllers/ruleCtrl.js

@@ -7,7 +7,7 @@ ruleCtrl.config(['ChartJsProvider', function (ChartJsProvider) {
         colours: ['#FF5252', '#FF8A80'],
         responsive: true
     });
-}])
+}]);
 
 ruleCtrl.controller('RuleCtrl', ['$scope', '$rootScope', '$routeParams', '$location', '$sce', 'Menu', 'Results', 'API', function($scope, $rootScope, $routeParams, $location, $sce, Menu, Results, API) {
     $scope.runId = $routeParams.runId;

front/src/js/directives/offendersDirectives.js

@@ -879,7 +879,7 @@
             }
 
             return mega.toFixed(1) + ' MB';
-        }
+        };
     });
 
 })();

lib/runner.js

@@ -3,7 +3,7 @@ var debug                   = require('debug')('ylt:runner');
 
 var phantomasWrapper        = require('./tools/phantomas/phantomasWrapper');
 var jsExecutionTransformer  = require('./tools/jsExecutionTransformer');
-var weightChecker           = require('./tools/weightChecker');
+var weightChecker           = require('./tools/weightChecker/weightChecker');
 var rulesChecker            = require('./rulesChecker');
 var scoreCalculator         = require('./scoreCalculator');
 
@@ -20,7 +20,9 @@ var Runner = function(params) {
     };
 
     // Execute Phantomas first
-    phantomasWrapper.execute(data).then(function(phantomasResults) {
+    phantomasWrapper.execute(data)
+
+    .then(function(phantomasResults) {
         data.toolsResults.phantomas = phantomasResults;
 
         // Treat the JS Execution Tree from offenders
@@ -29,7 +31,9 @@ var Runner = function(params) {
         // Redownload every file
         return weightChecker.recheckAllFiles(data);
 
-    }).then(function(data) {
+    })
+
+    .then(function(data) {
 
         // Rules checker
         var policies = require('./metadata/policies');
@@ -50,12 +54,16 @@ var Runner = function(params) {
 
         return data;
 
-    }).then(function(data) {
+    })
+
+    .then(function(data) {
 
         // Finished!
         deferred.resolve(data);
 
-    }).fail(function(err) {
+    })
+
+    .fail(function(err) {
         debug('Run failed');
         debug(err);
 

lib/tools/weightChecker/imageOptimizer.js

@@ -0,0 +1,121 @@
+var debug = require('debug')('ylt:imageOptimizer');
+
+var Q           = require('q');
+var Imagemin    = require('imagemin');
+var jpegoptim   = require('imagemin-jpegoptim');
+
+var ImageOptimizer = function() {
+
+    var MAX_JPEG_QUALITY = 85;
+
+    function optimizeImage(entry) {
+        var deferred = Q.defer();
+
+        var fileSize = entry.weightCheck.body.length;
+        debug('Current file size is %d', fileSize);
+
+        if (isJpeg(entry)) {
+            debug('File is a JPEG');
+
+            // Starting softly with a lossless compression
+            return compressJpegLosslessly(entry.weightCheck.body)
+
+            .then(function(newFile) {
+                var newFileSize = newFile.contents.length;
+
+                debug('JPEG lossless compression complete for %s', entry.url);
+                
+                if (newFileSize < fileSize) {
+                    entry.weightCheck.lossless = entry.weightCheck.optimized = newFileSize;
+                    entry.weightCheck.isOptimized = false;
+                    debug('Filesize is %d bytes smaller (-%d%)', fileSize - newFileSize, Math.round((fileSize - newFileSize) * 100 / fileSize));
+                }
+
+
+                // Now let's compress lossy to MAX_JPEG_QUALITY
+                return compressJpegLossly(entry.weightCheck.body);
+            })
+            
+            .then(function(newFile) {
+                var newFileSize = newFile.contents.length;
+
+                debug('JPEG lossy compression complete for %s', entry.url);
+
+                if (newFileSize < fileSize) {
+                    entry.weightCheck.lossy = newFileSize;
+                    entry.weightCheck.isOptimized = false;
+                    debug('Filesize is %d bytes smaller (-%d%)', fileSize - newFileSize, Math.round((fileSize - newFileSize) * 100 / fileSize));
+
+                    if (newFileSize < entry.weightCheck.lossless) {
+                        entry.weightCheck.optimized = newFileSize;
+                    }
+                }
+
+                return entry;
+            });
+
+        } else {
+            debug('File type is not an optimizable image');
+            deferred.resolve(entry);
+        }
+
+        return deferred.promise;
+    }
+
+    function isJpeg(entry) {
+        return entry.isImage && entry.contentType === 'image/jpeg';
+    }
+
+    function compressJpegLosslessly(imageBody) {
+        var deferred = Q.defer();
+        var startTime = Date.now();
+
+        debug('Starting JPEG lossless compression');
+
+        new Imagemin()
+            .src(imageBody)
+            .use(Imagemin.jpegtran())
+            .run(function (err, files) {
+                if (err) {
+                    deferred.reject(err);
+                } else {
+                    deferred.resolve(files[0]);
+                    var endTime = Date.now();
+                    debug('compressJpegLosslessly took %d ms', endTime - startTime);
+                }
+            });
+
+        return deferred.promise;
+    }
+
+    function compressJpegLossly(imageBody) {
+        var deferred = Q.defer();
+        var startTime = Date.now();
+
+        debug('Starting JPEG lossly compression');
+
+        new Imagemin()
+            .src(imageBody)
+            .use(jpegoptim({max: MAX_JPEG_QUALITY}))
+            .run(function (err, files) {
+                if (err) {
+                    deferred.reject(err);
+                } else {
+                    deferred.resolve(files[0]);
+                    var endTime = Date.now();
+                    debug('compressJpegLossly took %d ms', endTime - startTime);
+                }
+            });
+
+        return deferred.promise;
+    }
+
+    return {
+        //recompressIfImage: recompressIfImage,
+        optimizeImage: optimizeImage,
+        compressJpegLosslessly: compressJpegLosslessly,
+        compressJpegLossly: compressJpegLossly
+    };
+};
+
+module.exports = new ImageOptimizer();

lib/tools/weightChecker.js → lib/tools/weightChecker/weightChecker.js

@@ -5,13 +5,12 @@
  */
 
 
-var debug = require('debug')('ylt:weightChecker');
-
-var request = require('request');
-var http    = require('http');
-var async   = require('async');
+var debug   = require('debug')('ylt:weightChecker');
 var Q       = require('q');
+var http    = require('http');
 var zlib    = require('zlib');
+var async   = require('async');
+var request = require('request');
 
 var WeightChecker = function() {
 
@@ -28,7 +27,18 @@ var WeightChecker = function() {
         // Transform every request into a download function with a callback when done
         var redownloadList = requestsList.map(function(entry) {
             return function(callback) {
-                redownloadEntry(entry, callback);
+                
+                redownloadEntry(entry)
+
+                .then(recompressIfImage)
+
+                .then(function(entry) {
+                    callback(null, entry);
+                })
+
+                .fail(function(err) {
+                    callback(err);
+                });
             };
         });
 
@@ -118,26 +128,25 @@ var WeightChecker = function() {
     }
 
 
-    function redownloadEntry(entry, callback) {
+    function redownloadEntry(entry) {
+        var deferred = Q.defer();
         
         function onError(message) {
             debug('Could not download %s Error: %s', entry.url, message);
             entry.weightCheck = {
                 message: message
             };
-            setImmediate(function() {
-                callback(null, entry);
-            });
+            deferred.reject();
         }
 
         if (entry.method !== 'GET') {
             onError('only downloading GET');
-            return;
+            return deferred.promise;
         }
 
         if (entry.status !== 200) {
             onError('only downloading requests with status code 200');
-            return;
+            return deferred.promise;
         }
 
 
@@ -167,8 +176,10 @@ var WeightChecker = function() {
             debug('%s downloaded correctly', entry.url);
 
             entry.weightCheck = result;
-            callback(null, entry);
+            deferred.resolve(entry);
         });
+
+        return deferred.promise;
     }
 
     // Inspired by https://github.com/cvan/fastHAR-api/blob/10cec585/app.js
@@ -268,11 +279,22 @@ var WeightChecker = function() {
         });
     }
 
+
+    function recompressIfImage(entry) {
+        var deferred = Q.defer();
+
+        deferred.resolve(entry);
+
+        return deferred.promise;
+    }
+
+
     return {
         recheckAllFiles: recheckAllFiles,
         listRequestWeight: listRequestWeight,
         redownloadEntry: redownloadEntry,
-        download: download
+        download: download,
+        recompressIfImage: recompressIfImage
     };
 };
 

package.json

@@ -17,6 +17,8 @@
     "cors": "^2.6.0",
     "debug": "~2.2.0",
     "express": "~4.12.4",
+    "imagemin": "~3.2.0",
+    "imagemin-jpegoptim": "~4.0.0",
     "lwip": "0.0.6",
     "meow": "^3.1.0",
     "phantomas": "1.10.2",

test/core/imageOptimizerTest.js

@@ -0,0 +1,79 @@
+var should = require('chai').should();
+var imageOptimizer = require('../../lib/tools/weightChecker/imageOptimizer');
+var fs = require('fs');
+var path = require('path');
+
+describe('imageOptimizer', function() {
+    
+    it('should optimize a JPEG image losslessly', function(done) {
+        var fileContent = fs.readFileSync(path.resolve(__dirname, '../fixtures/jpeg-image.jpg'));
+
+        var fileSize = fileContent.length;
+
+        imageOptimizer.compressJpegLosslessly(fileContent).then(function(newFile) {
+            var newFileSize = newFile.contents.length;
+            newFileSize.should.be.below(fileSize);
+            done();
+        }).fail(function(err) {
+            done(err);
+        });
+    });
+
+    it('should optimize a JPEG image lossly', function(done) {
+        var fileContent = fs.readFileSync(path.resolve(__dirname, '../fixtures/jpeg-image.jpg'));
+
+        var fileSize = fileContent.length;
+
+        imageOptimizer.compressJpegLossly(fileContent).then(function(newFile) {
+            var newFileSize = newFile.contents.length;
+            newFileSize.should.be.below(fileSize);
+            done();
+        }).fail(function(err) {
+            done(err);
+        });
+    });
+
+    it('should find the best optimization for a jpeg', function(done) {
+        var fileContent = fs.readFileSync(path.resolve(__dirname, '../fixtures/jpeg-image.jpg'));
+        var fileSize = fileContent.length;
+
+        var entry = {
+            method: 'GET',
+            url: 'http://localhost:8388/an-image.jpg',
+            requestHeaders: {
+                'User-Agent': 'something',
+                Referer: 'http://www.google.fr/',
+                Accept: '*/*',
+                'Accept-Encoding': 'gzip, deflate'
+            },
+            status: 200,
+            isImage: true,
+            type: 'image',
+            contentType: 'image/jpeg',
+            contentLength: 999,
+            weightCheck: {
+                body: fileContent,
+                totalWeight: fileSize + 200,
+                headersSize: 200,
+                bodySize: fileSize,
+                isCompressed: false,
+                uncompressedSize: fileSize
+            }
+        };
+
+        imageOptimizer.optimizeImage(entry)
+
+        .then(function(newEntry) {
+            newEntry.weightCheck.should.have.a.property('isOptimized').that.equals(false);
+            newEntry.weightCheck.should.have.a.property('lossless').that.is.below(fileSize);
+            newEntry.weightCheck.should.have.a.property('lossy').that.is.below(newEntry.weightCheck.lossless);
+
+            done();
+        })
+
+        .fail(function(err) {
+            done(err);
+        });
+    });
+
+});

test/core/phantomasWrapperTest.js

@@ -48,7 +48,6 @@ describe('phantomasWrapper', function() {
             }
         }).then(function(data) {
 
-            console.log(data);
             done('Error: unwanted success');
 
         }).fail(function(err) {

test/core/weightCheckerTest.js

@@ -1,5 +1,5 @@
 var should = require('chai').should();
-var weightChecker = require('../../lib/tools/weightChecker');
+var weightChecker = require('../../lib/tools/weightChecker/weightChecker');
 
 describe('weightChecker', function() {
     
@@ -69,8 +69,9 @@ describe('weightChecker', function() {
             isJS: true
         };
 
-        weightChecker.redownloadEntry(entry, function(err, newEntry) {
-            should.not.exist(err);
+        weightChecker.redownloadEntry(entry)
+
+        .then(function(newEntry) {
 
             newEntry.weightCheck.bodySize.should.equal(93636);
             newEntry.weightCheck.uncompressedSize.should.equal(newEntry.weightCheck.bodySize);
@@ -79,6 +80,10 @@ describe('weightChecker', function() {
             newEntry.weightCheck.body.should.have.string('1.8.3');
 
             done();
+        })
+
+        .fail(function(err) {
+            done(err);
         });
     });
 
@@ -96,12 +101,16 @@ describe('weightChecker', function() {
             contentLength: 999
         };
 
-        weightChecker.redownloadEntry(entry, function(err, newEntry) {
-            should.not.exist(err);
+        weightChecker.redownloadEntry(entry)
 
+        .then(function(errnewEntry) {
             newEntry.weightCheck.should.have.a.property('message').that.equals('error while downloading: 404');
 
             done();
+        })
+
+        .fail(function(err) {
+            done(err);
         });
     });
 
@@ -119,12 +128,16 @@ describe('weightChecker', function() {
             contentLength: 999
         };
 
-        weightChecker.redownloadEntry(entry, function(err, newEntry) {
-            should.not.exist(err);
+        weightChecker.redownloadEntry(entry)
 
+        .then(function(newEntry) {
             newEntry.weightCheck.should.have.a.property('message').that.equals('only downloading requests with status code 200');
 
             done();
+        })
+
+        .fail(function(err) {
+            done(err);
         });
     });