Optimizing JS minification step that takes too long

lib/tools/weightChecker/fileMinifier.js

@@ -20,12 +20,12 @@ var FileMinifier = function() {
         var fileSize = entry.weightCheck.uncompressedSize;
         debug('Let\'s try to optimize %s', entry.url);
         debug('Current file size is %d', fileSize);
+        var startTime = Date.now();
 
-        if (entry.isJS) {
+        if (entry.isJS && !isKnownAsMinified(entry.url) && !looksAlreadyMinified(entry.weightCheck.body)) {
 
             debug('File is a JS');
 
-            // Starting softly with a lossless compression
             return minifyJs(entry.weightCheck.body)
 
             .then(function(newFile) {
@@ -34,6 +34,8 @@ var FileMinifier = function() {
                     return entry;
                 }
 
+                var endTime = Date.now();
+
                 var newFileSize = newFile.length;
 
                 debug('JS minification complete for %s', entry.url);
@@ -56,7 +58,6 @@ var FileMinifier = function() {
 
             debug('File is a CSS');
 
-            // Starting softly with a lossless compression
             return minifyCss(entry.weightCheck.body)
 
             .then(function(newFile) {
@@ -65,6 +66,9 @@ var FileMinifier = function() {
                     return entry;
                 }
 
+                var endTime = Date.now();
+                debug('CSS minification took %dms', endTime - startTime);
+
                 var newFileSize = newFile.length;
 
                 debug('CSS minification complete for %s', entry.url);
@@ -87,7 +91,6 @@ var FileMinifier = function() {
 
             debug('File is an HTML');
 
-            // Starting softly with a lossless compression
             return minifyHtml(entry.weightCheck.body)
 
             .then(function(newFile) {
@@ -96,6 +99,9 @@ var FileMinifier = function() {
                     return entry;
                 }
 
+                var endTime = Date.now();
+                debug('HTML minification took %dms', endTime - startTime);
+
                 var newFileSize = newFile.length;
 
                 debug('HTML minification complete for %s', entry.url);
@@ -132,18 +138,118 @@ var FileMinifier = function() {
 
     // Uglify
     function minifyJs(body) {
+        
+        // Splitting the Uglify function because it sometime takes too long (more than 10 seconds)
+        // I hope that, by splitting, it can be a little more asynchronous, so the application doesn't freeze.
+
+        return splittedUglifyStep1(body)
+        .delay(1)
+        .then(splittedUglifyStep2)
+        .delay(1)
+        .then(splittedUglifyStep3)
+        .delay(1)
+        .then(splittedUglifyStep4)
+        .delay(1)
+        .then(splittedUglifyStep5)
+        .delay(1)
+        .then(splittedUglifyStep6)
+        .delay(1)
+        .then(splittedUglifyStep7);
+
+    }
+
+    function splittedUglifyStep1(code) {
         var deferred = Q.defer();
+        var startTime = Date.now();
 
         try {
-            var result = UglifyJS.minify(body, {fromString: true});
-            deferred.resolve(result.code);
+            var toplevel_ast = UglifyJS.parse(code);
+
+            var endTime = Date.now();
+            debug('Uglify step 1 took %dms', endTime - startTime);
+            deferred.resolve(toplevel_ast);
+
         } catch(err) {
+            debug('JS syntax error, Uglify\'s parser failed (step 1)');
             deferred.reject(err);
         }
 
         return deferred.promise;
     }
 
+    function splittedUglifyStep2(toplevel) {
+        var deferred = Q.defer();
+        var startTime = Date.now();
+
+        toplevel.figure_out_scope();
+
+        var endTime = Date.now();
+        debug('Uglify step 2 took %dms', endTime - startTime);
+        deferred.resolve(toplevel);
+        return deferred.promise;
+    }
+
+    function splittedUglifyStep3(toplevel) {
+        var deferred = Q.defer();
+        var startTime = Date.now();
+
+        var compressor = UglifyJS.Compressor({warnings: false});
+        var compressed_ast = toplevel.transform(compressor);
+
+        var endTime = Date.now();
+        debug('Uglify step 3 took %dms', endTime - startTime);
+        deferred.resolve(compressed_ast);
+        return deferred.promise;
+    }
+
+    function splittedUglifyStep4(compressed_ast) {
+        var deferred = Q.defer();
+        var startTime = Date.now();
+
+        compressed_ast.figure_out_scope();
+
+        var endTime = Date.now();
+        debug('Uglify step 4 took %dms', endTime - startTime);
+        deferred.resolve(compressed_ast);
+        return deferred.promise;
+    }
+
+    function splittedUglifyStep5(compressed_ast) {
+        var deferred = Q.defer();
+        var startTime = Date.now();
+
+        compressed_ast.compute_char_frequency();
+
+        var endTime = Date.now();
+        debug('Uglify step 5 took %dms', endTime - startTime);
+        deferred.resolve(compressed_ast);
+        return deferred.promise;
+    }
+
+    function splittedUglifyStep6(compressed_ast) {
+        var deferred = Q.defer();
+        var startTime = Date.now();
+
+        compressed_ast.mangle_names();
+
+        var endTime = Date.now();
+        debug('Uglify step 6 took %dms', endTime - startTime);
+        deferred.resolve(compressed_ast);
+        return deferred.promise;
+    }
+
+    function splittedUglifyStep7(compressed_ast) {
+        var deferred = Q.defer();
+        var startTime = Date.now();
+
+        var code = compressed_ast.print_to_string();
+
+        var endTime = Date.now();
+        debug('Uglify step 7 took %dms', endTime - startTime);
+        deferred.resolve(code);
+        return deferred.promise;
+    }
+
     // Clear-css
     function minifyCss(body) {
         var deferred = Q.defer();
@@ -179,6 +285,44 @@ var FileMinifier = function() {
         return deferred.promise;
     }
 
+    // Avoid loosing time trying to compress some JS libraries known as already compressed
+    function isKnownAsMinified(url) {
+        var result = false;
+
+        // Twitter
+        result = result || /^https?:\/\/platform\.twitter\.com\/widgets\.js/.test(url);
+
+        // Facebook
+        result = result || /^https:\/\/connect\.facebook\.net\/[^\/]*\/(sdk|all)\.js/.test(url);
+
+        // Google +1
+        result = result || /^https:\/\/apis\.google\.com\/js\/plusone\.js/.test(url);
+
+        // jQuery CDN
+        result = result || /^https?:\/\/code\.jquery\.com\/.*\.min.js/.test(url);
+
+
+        if (result === true) {
+            debug('This file is known as already minified. Skipping minification: %s', url);
+        }
+
+        return result;
+    }
+
+    // Avoid loosing tome trying to compress JS files if they alreay look minified
+    // by counting the number of lines compared to the total size.
+    // Less than 1000kb per line is suspicious
+    function looksAlreadyMinified(code) {
+        var linesCount = code.split(/\r\n|\r|\n/).length;
+        var linesRatio = code.length / linesCount / 1024;
+        var looksMinified = linesRatio > 1;
+        
+        debug('Lines ratio is %d KB per line', linesRatio.toFixed(1));
+        debug(looksMinified ? 'It looks already minified' : 'It doesn\'t look minified');
+
+        return looksMinified;
+    }
+
     function entryTypeCanBeMinified(entry) {
         return entry.isJS || entry.isCSS || entry.isHTML;
     }
@@ -189,7 +333,8 @@ var FileMinifier = function() {
         minifyCss: minifyCss,
         minifyHtml: minifyHtml,
         gainIsEnough: gainIsEnough,
-        entryTypeCanBeMinified: entryTypeCanBeMinified
+        entryTypeCanBeMinified: entryTypeCanBeMinified,
+        isKnownAsMinified: isKnownAsMinified
     };
 };
 

lib/tools/weightChecker/gzipCompressor.js

@@ -15,7 +15,7 @@ var GzipCompressor = function() {
     function gzipUncompressedFile(entry) {
         var deferred = Q.defer();
 
-        if (entryTypeCanBeGzipped(entry) && entry.weightCheck && !entry.weightCheck.isCompressed) {
+        if (entryTypeCanBeGzipped(entry) && entry.weightCheck && !entry.weightCheck.isCompressed && entry.weightCheck.body) {
             debug('Compression missing, trying to gzip file %s', entry.url);
 
             var uncompressedSize = entry.weightCheck.uncompressedSize;
@@ -65,13 +65,8 @@ var GzipCompressor = function() {
                 } else {
                     var compressedSize = buffer.length;
 
-                    if (gainIsEnough(uncompressedSize, compressedSize)) {
-                        debug('Correctly gziped the minified file, was %d and is now %d bytes', uncompressedSize, compressedSize);
-
-                        entry.weightCheck.afterOptimizationAndCompression = compressedSize;
-                    } else {
-                        debug('Gzip gain is not enough on minified file, was %d and is now %d bytes', uncompressedSize, compressedSize);
-                    }
+                    debug('Correctly gziped the minified file, was %d and is now %d bytes', uncompressedSize, compressedSize);
+                    entry.weightCheck.afterOptimizationAndCompression = compressedSize;
 
                     deferred.resolve(entry);
                 }

lib/tools/weightChecker/weightChecker.js

@@ -20,7 +20,7 @@ var gzipCompressor  = require('./gzipCompressor');
 var WeightChecker = function() {
 
     var MAX_PARALLEL_DOWNLOADS = 10;
-    var REQUEST_TIMEOUT = 10000; // 10 seconds
+    var REQUEST_TIMEOUT = 15000; // 15 seconds
 
 
     // This function will re-download every asset and check if it could be optimized
@@ -195,17 +195,24 @@ var WeightChecker = function() {
         };
 
         requests.forEach(function(req) {
-            if (req.weightCheck.uncompressedSize && fileMinifier.entryTypeCanBeMinified(req) && req.weightCheck.isOptimized === false) {
-                var gain = req.weightCheck.uncompressedSize - req.weightCheck.optimized;
-
-                results.totalGain += gain;
-
-                results.files.push({
-                    url: req.url,
-                    original: req.weightCheck.uncompressedSize,
-                    optimized: req.weightCheck.optimized,
-                    gain: gain
-                });
+            if (req.weightCheck.bodySize > 0 && fileMinifier.entryTypeCanBeMinified(req) && req.weightCheck.isOptimized === false) {
+                var before = req.weightCheck.afterCompression || req.weightCheck.bodySize;
+                var after = req.weightCheck.afterOptimizationAndCompression || req.weightCheck.afterCompression;
+                var gain = before - after;
+
+                if (gain > 200) {
+                    results.totalGain += gain;
+
+                    results.files.push({
+                        url: req.url,
+                        original: req.weightCheck.bodySize,
+                        isCompressed: req.weightCheck.isCompressed,
+                        afterCompression: req.weightCheck.afterCompression,
+                        afterOptimizationAndCompression: req.weightCheck.afterOptimizationAndCompression,
+                        optimized: req.weightCheck.optimized,
+                        gain: gain
+                    });
+                }
             }
         });
 

test/core/fileMinifierTest.js

@@ -217,4 +217,23 @@ describe('fileMinifier', function() {
         });
     });
 
+    it('should avoid minifying some JS files known as minified', function() {
+        fileMinifier.isKnownAsMinified('https://platform.twitter.com/widgets.js').should.equal(true);
+        fileMinifier.isKnownAsMinified('http://platform.twitter.com/widgets.js').should.equal(true);
+
+        fileMinifier.isKnownAsMinified('https://connect.facebook.net/fr_FR/sdk.js').should.equal(true);
+        fileMinifier.isKnownAsMinified('https://connect.facebook.net/en_EN/sdk.js').should.equal(true);
+        fileMinifier.isKnownAsMinified('https://connect.facebook.net/fr_FR/all.js').should.equal(true);
+        fileMinifier.isKnownAsMinified('https://connect.facebook.net/en_EN/all.js').should.equal(true);
+
+        fileMinifier.isKnownAsMinified('https://apis.google.com/js/plusone.js').should.equal(true);
+
+        fileMinifier.isKnownAsMinified('https://code.jquery.com/jquery-2.1.4.min.js').should.equal(true);
+        fileMinifier.isKnownAsMinified('http://code.jquery.com/jquery-2.1.4.min.js').should.equal(true);
+        fileMinifier.isKnownAsMinified('https://code.jquery.com/jquery-2.1.4.js').should.equal(false);
+        fileMinifier.isKnownAsMinified('http://code.jquery.com/jquery-2.1.4.js').should.equal(false);
+
+        fileMinifier.isKnownAsMinified('http://anydomain.com/anyurl').should.equal(false);
+    });
+
 });