New rule called "Empty requests" (#180)

front/src/views/rule.html

@@ -141,7 +141,7 @@
                         <url-link url="offender" max-length="100"></url-link>
                     </div>
 
-                    <div ng-if="policyName === 'notFound' || policyName === 'closedConnections' || policyName === 'multipleRequests' || policyName === 'cachingDisabled' || policyName === 'cachingNotSpecified'">
+                    <div ng-if="policyName === 'notFound' || policyName === 'emptyRequests' || policyName === 'closedConnections' || policyName === 'multipleRequests' || policyName === 'cachingDisabled' || policyName === 'cachingNotSpecified'">
                         <url-link url="offender" max-length="100"></url-link>
                     </div>
 
@@ -158,7 +158,6 @@
                     <div ng-if="policyName === 'globalVariables' || policyName === 'jQueryVersionsLoaded' || policyName === 'synchronousXHR'">
                         {{offender}}
                     </div>
-
                 </div>
             </div>
         </div>

lib/metadata/policies.js

@@ -899,7 +899,7 @@ var policies = {
         }
     },
     "totalWeight": {
-        "tool": "weightChecker",
+        "tool": "redownload",
         "label": "Total weight",
         "message": "<p>The weight is of course very important if you want the page to load fast. Try to stay under 1MB, which is alreay very long to download over a slow connection.</p><p>Please note that Yellow Lab Tools' engine (PhantomJS) is not compatible with image srcset (unless you use a polyfill). This can lead to incorrect page weight.</p>",
         "isOkThreshold": 716800,
@@ -909,7 +909,7 @@ var policies = {
         "unit": 'bytes'
     },
     "imageOptimization": {
-        "tool": "weightChecker",
+        "tool": "redownload",
         "label": "Image optimization",
         "message": "<p>This metric measures the number of bytes that could be saved by optimizing images.</p><p>Image optimization is generally one of the easiest way to reduce a page weight, and as a result, the page load time. Don't use Photoshop or other image editing tools, they're not very good for optimization. Use specialized tools such as <a href=\"https://kraken.io/\" target=\"_blank\">Kraken.io</a> or the excellent <a href=\"https://imageoptim.com/\" target=\"_blank\">ImageOptim</a> on Mac. For SVG images, you can use <a href=\"https://jakearchibald.github.io/svgomg/\" target=\"_blank\">SVGOMG</a></p><p>The tools in use in YellowLabTools are not set to their maximum optimization power (JPEG quality 85), so you might be able to compress even more!</p><p>Please note that Yellow Lab Tools' engine (PhantomJS) is not compatible with image srcset (unless you use a polyfill). This can lead to incorrect page weight.</p>",
         "isOkThreshold": 10240,
@@ -919,7 +919,7 @@ var policies = {
         "unit": 'bytes'
     },
     "gzipCompression": {
-        "tool": "weightChecker",
+        "tool": "redownload",
         "label": "Gzip compression",
         "message": "<p>Measures the number of bytes that could be saved by compressing file transfers.</p><p>Gzip is a powerfull weight reducer and should be enabled on text-based assets in your server's configuration. Note that gzipping small files (< 1 KB) is arguable, and that some assets such as images should not be gzipped as they are already compressed. <a href=\"https://gist.github.com/gmetais/971ce13a1fbeebd88445\" target=\"_blank\">Here</a> is a list of Content-Types that should be gzipped.</p>",
         "isOkThreshold": 5125,
@@ -929,7 +929,7 @@ var policies = {
         "unit": 'bytes'
     },
     "fileMinification": {
-        "tool": "weightChecker",
+        "tool": "redownload",
         "label": "File minification",
         "message": "<p>This is the weight that could be saved if all text resources were correctly minified.</p><p>The tools in use here are <b>UglifyJS</b>, <b>clean-css</b> and <b>HTMLMinifier</b>. These tools are so good that some of your minified files can be marked as unminified. Change your tool it this happens :)</p><p>The gains of minification are generally small, but the impact can be high when these text files are loaded on the critical path.</p>",
         "isOkThreshold": 5125,
@@ -939,7 +939,7 @@ var policies = {
         "unit": 'bytes'
     },
     "totalRequests": {
-        "tool": "weightChecker",
+        "tool": "redownload",
         "label": "Requests number",
         "message": "<p>This is one of the most important performance rule. Every request is slowing down the page loading.</p><p>There are several technics to reduce their number:<ul><li>Concatenate JS files</li><li>Concatenate CSS files</li><li>Embed or inline small JS or CSS files in the HTML</li><li>Create sprites or icon fonts</li><li>Base64 encode small images in HTML or stylesheets</li><li>Use lazyloading for images</li></ul></p>",
         "isOkThreshold": 15,
@@ -1003,8 +1003,17 @@ var policies = {
         "isAbnormalThreshold": 10,
         "hasOffenders": true
     },
+    "emptyRequests": {
+        "tool": "redownload",
+        "label": "Empty requests",
+        "message": "<p>List of GET requests that respond with an empty body. These are probably the easiest requests to remove.</p>",
+        "isOkThreshold": 0,
+        "isBadThreshold": 1,
+        "isAbnormalThreshold": 5,
+        "hasOffenders": true
+    },
     "smallRequests": {
-        "tool": "weightChecker",
+        "tool": "redownload",
         "label": "Small requests",
         "message": "<p>List of all requests that are less than 2 KB. Try to merge them with other files.</p>",
         "isOkThreshold": 4,

lib/metadata/scoreProfileGeneric.json

@@ -16,6 +16,7 @@
                 "domains": 3,
                 "notFound": 3,
                 "multipleRequests": 2,
+                "emptyRequests": 3,
                 "smallRequests": 1,
                 "lazyLoadableImagesBelowTheFold": 2,
                 "hiddenImages": 1

lib/runner.js

@@ -6,7 +6,7 @@ var jsExecutionTransformer  = require('./tools/jsExecutionTransformer');
 var colorDiff               = require('./tools/colorDiff');
 var mediaQueriesChecker     = require('./tools/mediaQueriesChecker');
 var isHttp2                 = require('./tools/isHttp2');
-var weightChecker           = require('./tools/weightChecker/weightChecker');
+var redownload              = require('./tools/redownload/redownload');
 var rulesChecker            = require('./rulesChecker');
 var scoreCalculator         = require('./scoreCalculator');
 
@@ -38,7 +38,7 @@ var Runner = function(params) {
         data = mediaQueriesChecker.analyzeMediaQueries(data);
 
         // Redownload every file
-        return weightChecker.recheckAllFiles(data);
+        return redownload.recheckAllFiles(data);
 
     })
 

lib/tools/weightChecker/weightChecker.js → lib/tools/redownload/redownload.js

@@ -5,7 +5,7 @@
  */
 
 
-var debug               = require('debug')('ylt:weightChecker');
+var debug               = require('debug')('ylt:redownload');
 var Q                   = require('q');
 var http                = require('http');
 var zlib                = require('zlib');
@@ -18,7 +18,7 @@ var gzipCompressor      = require('./gzipCompressor');
 var contentTypeChecker  = require('./contentTypeChecker');
 
 
-var WeightChecker = function() {
+var Redownload = function() {
 
     var MAX_PARALLEL_DOWNLOADS = 10;
     var REQUEST_TIMEOUT = 15000; // 15 seconds
@@ -86,9 +86,10 @@ var WeightChecker = function() {
                 offenders.totalRequests = listRequestsByType(results);
                 metrics.totalRequests = offenders.totalRequests.total;
 
-                // Remove unwanted requests (redirections, about:blank)
+
+                // Remove unwanted responses (redownload failed, about:blank)
                 results = results.filter(function(result) {
-                    return (result !== null && result.weightCheck && result.weightCheck.bodySize > 0);
+                    return (result !== null && result.weightCheck && result.url.indexOf('about:blank') !== 0);
                 });
 
 
@@ -100,6 +101,17 @@ var WeightChecker = function() {
                 offenders.totalWeight = listRequestWeight(results);
                 metrics.totalWeight = offenders.totalWeight.totalWeight;
 
+                // Empty files
+                offenders.emptyRequests = listEmptyRequests(results);
+                metrics.emptyRequests = offenders.emptyRequests.length;
+
+
+                // Now emove unwanted responses (redirections)
+                results = results.filter(function(result) {
+                    return (result.status < 300 || result.status >= 400);
+                });
+
+
                 // Image compression
                 offenders.imageOptimization = listImageNotOptimized(results);
                 metrics.imageOptimization = offenders.imageOptimization.totalGain;
@@ -116,7 +128,7 @@ var WeightChecker = function() {
                 offenders.smallRequests = listSmallRequests(results);
                 metrics.smallRequests = offenders.smallRequests.total;
 
-                data.toolsResults.weightChecker = {
+                data.toolsResults.redownload = {
                     metrics: metrics,
                     offenders: offenders
                 };
@@ -144,6 +156,20 @@ var WeightChecker = function() {
         return results;
     }
 
+    function listEmptyRequests(requests) {
+        var results = [];
+        
+        requests.forEach(function(req) {
+            var weight = req.weightCheck.bodySize || req.contentLength;
+
+            if (weight === 0 && req.method === 'GET' && req.status === 200) {
+                results.push(req.url);
+            }
+        });
+
+        return results;
+    }
+
     function listRequestWeight(requests) {
         var results = {
             totalWeight: 0,
@@ -555,4 +581,4 @@ var WeightChecker = function() {
     };
 };
 
-module.exports = new WeightChecker();
+module.exports = new Redownload();

test/core/contentTypeCheckerTest.js

@@ -1,5 +1,5 @@
 var should = require('chai').should();
-var contentTypeChecker = require('../../lib/tools/weightChecker/contentTypeChecker');
+var contentTypeChecker = require('../../lib/tools/redownload/contentTypeChecker');
 var fs = require('fs');
 var path = require('path');
 

test/core/fileMinifierTest.js

@@ -1,5 +1,5 @@
 var should = require('chai').should();
-var fileMinifier = require('../../lib/tools/weightChecker/fileMinifier');
+var fileMinifier = require('../../lib/tools/redownload/fileMinifier');
 var fs = require('fs');
 var path = require('path');
 

test/core/gzipCompressorTest.js

@@ -1,6 +1,6 @@
 var should = require('chai').should();
-var gzipCompressor = require('../../lib/tools/weightChecker/gzipCompressor');
-var fileMinifier = require('../../lib/tools/weightChecker/fileMinifier');
+var gzipCompressor = require('../../lib/tools/redownload/gzipCompressor');
+var fileMinifier = require('../../lib/tools/redownload/fileMinifier');
 var fs = require('fs');
 var path = require('path');
 

test/core/imageOptimizerTest.js

@@ -1,5 +1,5 @@
 var should = require('chai').should();
-var imageOptimizer = require('../../lib/tools/weightChecker/imageOptimizer');
+var imageOptimizer = require('../../lib/tools/redownload/imageOptimizer');
 var fs = require('fs');
 var path = require('path');
 

test/core/weightCheckerTest.js → test/core/redownloadTest.js

@@ -1,9 +1,9 @@
 var should = require('chai').should();
-var weightChecker = require('../../lib/tools/weightChecker/weightChecker');
+var redownload = require('../../lib/tools/redownload/redownload');
 var fs = require('fs');
 var path = require('path');
 
-describe('weightChecker', function() {
+describe('redownload', function() {
     
     it('should download a list of files', function(done) {
         this.timeout(10000);
@@ -121,49 +121,49 @@ describe('weightChecker', function() {
             }
         };
 
-        weightChecker.recheckAllFiles(data)
+        redownload.recheckAllFiles(data)
 
         .then(function(data) {
-            data.toolsResults.should.have.a.property('weightChecker');
-            data.toolsResults.weightChecker.should.have.a.property('metrics');
-            data.toolsResults.weightChecker.should.have.a.property('offenders');
-
-            data.toolsResults.weightChecker.offenders.should.have.a.property('totalWeight');
-            data.toolsResults.weightChecker.offenders.totalWeight.totalWeight.should.be.above(0);
-            data.toolsResults.weightChecker.offenders.totalWeight.byType.html.requests.length.should.equal(1);
-            data.toolsResults.weightChecker.offenders.totalWeight.byType.js.requests.length.should.equal(2);
-            data.toolsResults.weightChecker.offenders.totalWeight.byType.css.requests.length.should.equal(1);
-            data.toolsResults.weightChecker.offenders.totalWeight.byType.image.requests.length.should.equal(2);
-            data.toolsResults.weightChecker.offenders.totalWeight.byType.other.requests.length.should.equal(1);
-
-            data.toolsResults.weightChecker.offenders.should.have.a.property('imageOptimization');
-            data.toolsResults.weightChecker.offenders.imageOptimization.totalGain.should.be.above(0);
-            data.toolsResults.weightChecker.offenders.imageOptimization.images.length.should.equal(2);
-
-            data.toolsResults.weightChecker.offenders.should.have.a.property('gzipCompression');
-            data.toolsResults.weightChecker.offenders.gzipCompression.totalGain.should.be.above(0);
-            data.toolsResults.weightChecker.offenders.gzipCompression.files.length.should.equal(5);
-
-            data.toolsResults.weightChecker.offenders.should.have.a.property('fileMinification');
-            data.toolsResults.weightChecker.offenders.fileMinification.totalGain.should.be.above(0);
-            data.toolsResults.weightChecker.offenders.fileMinification.files.length.should.equal(2);
-
-            data.toolsResults.weightChecker.metrics.should.have.a.property('totalRequests').that.equals(7);
-            data.toolsResults.weightChecker.offenders.should.have.a.property('totalRequests');
-            data.toolsResults.weightChecker.offenders.totalRequests.byType.html.length.should.equal(1);
-            data.toolsResults.weightChecker.offenders.totalRequests.byType.js.length.should.equal(2);
-            data.toolsResults.weightChecker.offenders.totalRequests.byType.css.length.should.equal(1);
-            data.toolsResults.weightChecker.offenders.totalRequests.byType.image.length.should.equal(2);
-            data.toolsResults.weightChecker.offenders.totalRequests.byType.json.length.should.equal(0);
-            data.toolsResults.weightChecker.offenders.totalRequests.byType.webfont.length.should.equal(0);
-            data.toolsResults.weightChecker.offenders.totalRequests.byType.video.length.should.equal(0);
-            data.toolsResults.weightChecker.offenders.totalRequests.byType.other.length.should.equal(1);
-
-            data.toolsResults.weightChecker.metrics.should.have.a.property('smallRequests').that.equals(0);
-            data.toolsResults.weightChecker.offenders.should.have.a.property('smallRequests');
-            data.toolsResults.weightChecker.offenders.smallRequests.byType.js.length.should.equal(0);
-            data.toolsResults.weightChecker.offenders.smallRequests.byType.css.length.should.equal(0);
-            data.toolsResults.weightChecker.offenders.smallRequests.byType.image.length.should.equal(0);
+            data.toolsResults.should.have.a.property('redownload');
+            data.toolsResults.redownload.should.have.a.property('metrics');
+            data.toolsResults.redownload.should.have.a.property('offenders');
+
+            data.toolsResults.redownload.offenders.should.have.a.property('totalWeight');
+            data.toolsResults.redownload.offenders.totalWeight.totalWeight.should.be.above(0);
+            data.toolsResults.redownload.offenders.totalWeight.byType.html.requests.length.should.equal(1);
+            data.toolsResults.redownload.offenders.totalWeight.byType.js.requests.length.should.equal(2);
+            data.toolsResults.redownload.offenders.totalWeight.byType.css.requests.length.should.equal(1);
+            data.toolsResults.redownload.offenders.totalWeight.byType.image.requests.length.should.equal(2);
+            data.toolsResults.redownload.offenders.totalWeight.byType.other.requests.length.should.equal(1);
+
+            data.toolsResults.redownload.offenders.should.have.a.property('imageOptimization');
+            data.toolsResults.redownload.offenders.imageOptimization.totalGain.should.be.above(0);
+            data.toolsResults.redownload.offenders.imageOptimization.images.length.should.equal(2);
+
+            data.toolsResults.redownload.offenders.should.have.a.property('gzipCompression');
+            data.toolsResults.redownload.offenders.gzipCompression.totalGain.should.be.above(0);
+            data.toolsResults.redownload.offenders.gzipCompression.files.length.should.equal(5);
+
+            data.toolsResults.redownload.offenders.should.have.a.property('fileMinification');
+            data.toolsResults.redownload.offenders.fileMinification.totalGain.should.be.above(0);
+            data.toolsResults.redownload.offenders.fileMinification.files.length.should.equal(2);
+
+            data.toolsResults.redownload.metrics.should.have.a.property('totalRequests').that.equals(7);
+            data.toolsResults.redownload.offenders.should.have.a.property('totalRequests');
+            data.toolsResults.redownload.offenders.totalRequests.byType.html.length.should.equal(1);
+            data.toolsResults.redownload.offenders.totalRequests.byType.js.length.should.equal(2);
+            data.toolsResults.redownload.offenders.totalRequests.byType.css.length.should.equal(1);
+            data.toolsResults.redownload.offenders.totalRequests.byType.image.length.should.equal(2);
+            data.toolsResults.redownload.offenders.totalRequests.byType.json.length.should.equal(0);
+            data.toolsResults.redownload.offenders.totalRequests.byType.webfont.length.should.equal(0);
+            data.toolsResults.redownload.offenders.totalRequests.byType.video.length.should.equal(0);
+            data.toolsResults.redownload.offenders.totalRequests.byType.other.length.should.equal(1);
+
+            data.toolsResults.redownload.metrics.should.have.a.property('smallRequests').that.equals(0);
+            data.toolsResults.redownload.offenders.should.have.a.property('smallRequests');
+            data.toolsResults.redownload.offenders.smallRequests.byType.js.length.should.equal(0);
+            data.toolsResults.redownload.offenders.smallRequests.byType.css.length.should.equal(0);
+            data.toolsResults.redownload.offenders.smallRequests.byType.image.length.should.equal(0);
 
             done();
         })
@@ -187,7 +187,7 @@ describe('weightChecker', function() {
             type: 'js'
         };
 
-        weightChecker.redownloadEntry(entry)
+        redownload.redownloadEntry(entry)
 
         .then(function(newEntry) {
 
@@ -221,7 +221,7 @@ describe('weightChecker', function() {
             contentType: 'image/png'
         };
 
-        weightChecker.redownloadEntry(entry)
+        redownload.redownloadEntry(entry)
 
         .then(function(newEntry) {
 
@@ -258,7 +258,7 @@ describe('weightChecker', function() {
             contentLength: 999
         };
 
-        weightChecker.redownloadEntry(entry)
+        redownload.redownloadEntry(entry)
 
         .then(function(newEntry) {
             newEntry.weightCheck.should.have.a.property('message').that.equals('error while downloading: 404');
@@ -285,7 +285,7 @@ describe('weightChecker', function() {
             contentLength: 999
         };
 
-        weightChecker.redownloadEntry(entry)
+        redownload.redownloadEntry(entry)
 
         .then(function(newEntry) {
             newEntry.should.not.have.a.property('weightCheck');
@@ -298,7 +298,7 @@ describe('weightChecker', function() {
     });
 
     it('should listRequestWeight', function() {
-        var totalWeightObj = weightChecker.listRequestWeight([{
+        var totalWeightObj = redownload.listRequestWeight([{
             method: 'GET',
             url: 'http://localhost:8388/jquery1.8.3.js',
             requestHeaders: {
@@ -328,7 +328,7 @@ describe('weightChecker', function() {
     });
 
     it('should listRequestWeight even if download failed', function() {
-        var totalWeightObj = weightChecker.listRequestWeight([{
+        var totalWeightObj = redownload.listRequestWeight([{
             method: 'GET',
             url: 'http://localhost:8388/jquery1.8.3.js',
             requestHeaders: {