Adding totalWeight metric

Gruntfile.js

@@ -143,7 +143,7 @@ module.exports = function(grunt) {
                 options: {
                     reporter: 'spec',
                 },
-                src: ['test/core/offendersHelpersTest.js']
+                src: ['test/core/phantomasWrapperTest.js']
             },
             coverage: {
                 options: {

lib/runner.js

@@ -3,6 +3,7 @@ var debug                   = require('debug')('ylt:runner');
 
 var phantomasWrapper        = require('./tools/phantomas/phantomasWrapper');
 var jsExecutionTransformer  = require('./tools/jsExecutionTransformer');
+var weightChecker           = require('./tools/weightChecker');
 var rulesChecker            = require('./rulesChecker');
 var scoreCalculator         = require('./scoreCalculator');
 
@@ -25,8 +26,10 @@ var Runner = function(params) {
         // Treat the JS Execution Tree from offenders
         data = jsExecutionTransformer.transform(data);
 
-        // Other tools go here
+        // Redownload every file
+        return weightChecker.recheckAllFiles(data);
 
+    }).then(function(data) {
 
         // Rules checker
         var policies = require('./metadata/policies');

lib/tools/phantomas/custom_modules/modules/requestsList/requestsList.js

@@ -0,0 +1,23 @@
+/**
+ * Retries download on every request to get the real file size
+ *
+ */
+
+exports.version = '0.1';
+
+exports.module = function(phantomas) {
+    'use strict';
+
+    phantomas.setMetric('requestsList');
+
+    var requests = [];
+
+    phantomas.on('recv', function(entry, res) {
+        requests.push(entry);
+    });
+
+    phantomas.on('report', function() {
+        phantomas.setMetric('requestsList', true, true);
+        phantomas.addOffender('requestsList', JSON.stringify(requests));
+    });
+};

lib/tools/phantomas/phantomasWrapper.js

@@ -67,7 +67,7 @@ var PhantomasWrapper = function() {
 
             if (value === true) {
                 optionsString += ' ' + '--' + opt;
-            } else if (value === false || value === null) {
+            } else if (value === false || value === null || value === undefined) {
                 // Nothing
             } else {
                 optionsString += ' ' + '--' + opt + '=' + value;
@@ -122,7 +122,7 @@ var PhantomasWrapper = function() {
                 }
 
 
-                debug('Returning from Phantomas');
+                debug('Returning from Phantomas with error %s', err);
 
                 // Adding some YellowLabTools errors here
                 if (json && json.metrics && (!json.metrics.javascriptExecutionTree || !json.offenders.javascriptExecutionTree)) {

lib/tools/weightChecker.js

@@ -0,0 +1,279 @@
+/*
+ * Redownloading every files after Phantomas has finished
+ * Checks weight and every kind of compression
+ *
+ */
+
+
+var debug = require('debug')('ylt:weightChecker');
+
+var request = require('request');
+var http    = require('http');
+var async   = require('async');
+var Q       = require('Q');
+var zlib    = require('zlib');
+
+var WeightChecker = function() {
+
+    var MAX_PARALLEL_DOWNLOADS = 10;
+    var REQUEST_TIMEOUT = 10000; // 10 seconds
+
+    function recheckAllFiles(data) {
+        var deferred = Q.defer();
+
+        var requestsList = JSON.parse(data.toolsResults.phantomas.offenders.requestsList);
+        delete data.toolsResults.phantomas.metrics.requestsList;
+        delete data.toolsResults.phantomas.offenders.requestsList;
+
+        // Transform every request into a download function with a callback when done
+        var redownloadList = requestsList.map(function(entry) {
+            return function(callback) {
+                redownloadEntry(entry, callback);
+            };
+        });
+
+        // Lanch all redownload functions and wait for completion
+        async.parallelLimit(redownloadList, MAX_PARALLEL_DOWNLOADS, function(err, results) {
+            if (err) {
+                debug(err);
+                deferred.reject(err);
+            } else {
+                debug('All files checked');
+                
+                var metrics = {};
+                var offenders = {};
+
+
+                // Total weight
+                offenders.totalWeight = listRequestWeight(results);
+                metrics.totalWeight = offenders.totalWeight.totalWeight;
+
+
+                data.toolsResults.weightChecker = {
+                    metrics: metrics,
+                    offenders: offenders
+                };
+
+                deferred.resolve(data);
+            }
+        });
+
+        return deferred.promise;
+    }
+
+
+    function listRequestWeight(requests) {
+        var results = {
+            totalWeight: 0,
+            byType: {
+                html: {
+                    totalWeight: 0,
+                    requests: []
+                },
+                css: {
+                    totalWeight: 0,
+                    requests: []
+                },
+                js: {
+                    totalWeight: 0,
+                    requests: []
+                },
+                json: {
+                    totalWeight: 0,
+                    requests: []
+                },
+                image: {
+                    totalWeight: 0,
+                    requests: []
+                },
+                video: {
+                    totalWeight: 0,
+                    requests: []
+                },
+                webfont: {
+                    totalWeight: 0,
+                    requests: []
+                },
+                other: {
+                    totalWeight: 0,
+                    requests: []
+                }
+            }
+        };
+
+        requests.forEach(function(req) {
+            var weight = (typeof req.weightCheck.bodySize === 'number') ? req.weightCheck.bodySize + req.weightCheck.headersSize : req.contentLength;
+            var type = req.type || 'other';
+
+            results.totalWeight += weight;
+            results.byType[type].totalWeight += weight;
+
+            results.byType[type].requests.push({
+                url: req.url,
+                weight: weight
+            });
+        });
+
+        return results;
+    }
+
+
+    function redownloadEntry(entry, callback) {
+        
+        function onError(message) {
+            debug('Could not download %s Error: %s', entry.url, message);
+            entry.weightCheck = {
+                message: message
+            };
+            setImmediate(function() {
+                callback(null, entry);
+            });
+        }
+
+        if (entry.method !== 'GET') {
+            onError('only downloading GET');
+            return;
+        }
+
+        if (entry.status !== 200) {
+            onError('only downloading requests with status code 200');
+            return;
+        }
+
+
+        debug('Downloading %s', entry.url);
+
+        // Always add a gzip header before sending, in case the server listens to it
+        var reqHeaders = entry.requestHeaders;
+        reqHeaders['Accept-Encoding'] = 'gzip, deflate';
+
+        var requestOptions = {
+            method: entry.method,
+            url: entry.url,
+            headers: reqHeaders,
+            timeout: REQUEST_TIMEOUT
+        };
+
+        download(requestOptions, function(error, result) {
+            if (error) {
+                if (error.code === 'ETIMEDOUT') {
+                    onError('timeout after ' + REQUEST_TIMEOUT + 'ms');
+                } else {
+                    onError('error while downloading: ' + error.code);
+                }
+                return;
+            }
+                
+            debug('%s downloaded correctly', entry.url);
+
+            entry.weightCheck = result;
+            callback(null, entry);
+        });
+    }
+
+    // Inspired by https://github.com/cvan/fastHAR-api/blob/10cec585/app.js
+    function download(requestOptions, callback) {
+
+        var statusCode;
+
+        request(requestOptions)
+
+        .on('response', function(res) {
+            
+            // Raw headers were added in NodeJS v0.12
+            // (https://github.com/joyent/node/issues/4844), but let's
+            // reconstruct them for backwards compatibility.
+            var rawHeaders = ('HTTP/' + res.httpVersion + ' ' + res.statusCode +
+                              ' ' + http.STATUS_CODES[res.statusCode] + '\r\n');
+            Object.keys(res.headers).forEach(function(headerKey) {
+                rawHeaders += headerKey + ': ' + res.headers[headerKey] + '\r\n';
+            });
+            rawHeaders += '\r\n';
+
+            var uncompressedSize = 0;  // size after uncompression
+            var bodySize = 0;  // bytes size over the wire
+            var body = '';  // plain text body (after uncompressing gzip/deflate)
+            var isCompressed = false;
+
+            function tally() {
+
+                if (statusCode !== 200) {
+                    callback({code: statusCode});
+                    return;
+                }
+
+                var result = {
+                    body: body,
+                    headersSize: Buffer.byteLength(rawHeaders, 'utf8'),
+                    bodySize: bodySize,
+                    isCompressed: isCompressed,
+                    uncompressedSize: uncompressedSize
+                };
+
+                callback(null, result);
+            }
+
+            switch (res.headers['content-encoding']) {
+                case 'gzip':
+                    var gzip = zlib.createGunzip();
+
+                    gzip.on('data', function (data) {
+                        body += data;
+                        uncompressedSize += data.length;
+                    }).on('end', function () {
+                        isCompressed = true;
+                        tally();
+                    });
+
+                    res.on('data', function (data) {
+                        bodySize += data.length;
+                    }).pipe(gzip);
+
+                    break;
+                case 'deflate':
+                    var deflate = zlib.createInflate();
+
+                    deflate.on('data', function (data) {
+                        body += data;
+                        uncompressedSize += data.length;
+                    }).on('end', function () {
+                        isCompressed = true;
+                        tally();
+                    });
+
+                    res.on('data', function (data) {
+                        bodySize += data.length;
+                    }).pipe(deflate);
+
+                    break;
+                default:
+                    res.on('data', function (data) {
+                        body += data;
+                        uncompressedSize += data.length;
+                        bodySize += data.length;
+                    }).on('end', function () {
+                        tally();
+                    });
+
+                    break;
+            }
+        })
+
+        .on('response', function(response) {
+            statusCode = response.statusCode;
+        })
+
+        .on('error', function(err) {
+            callback(err);
+        });
+    }
+
+    return {
+        recheckAllFiles: recheckAllFiles,
+        listRequestWeight: listRequestWeight,
+        redownloadEntry: redownloadEntry,
+        download: download
+    };
+};
+
+module.exports = new WeightChecker();

test/core/phantomasWrapperTest.js

@@ -42,10 +42,13 @@ describe('phantomasWrapper', function() {
         phantomasWrapper.execute({
             params: {
                 url: url,
-                options: {}
+                options: {
+                    device: 'desktop'
+                }
             }
         }).then(function(data) {
 
+            console.log(data);
             done('Error: unwanted success');
 
         }).fail(function(err) {

test/core/weightCheckerTest.js

@@ -0,0 +1,183 @@
+var should = require('chai').should();
+var weightChecker = require('../../lib/tools/weightChecker');
+
+describe('weightChecker', function() {
+    
+    it('should download a list of files', function(done) {
+        var requestsList = [
+            {
+                method: 'GET',
+                url: 'http://localhost:8388/simple-page.html',
+                requestHeaders: {
+                    'User-Agent': 'something',
+                   Referer: 'http://www.google.fr/',
+                   Accept: '*/*'
+                },
+                status: 200,
+                isHTML: true
+            },
+            {
+                method: 'GET',
+                url: 'http://localhost:8388/jquery1.8.3.js',
+                requestHeaders: {
+                    'User-Agent': 'something',
+                   Referer: 'http://www.google.fr/',
+                   Accept: '*/*'
+                },
+                status: 200,
+                isJS: true
+            }
+        ];
+
+        var data = {
+            toolsResults: {
+                phantomas: {
+                    metrics: {
+                        requestsList: true
+                    },
+                    offenders: {
+                        requestsList: JSON.stringify(requestsList)
+                    }
+                }
+            }
+        };
+
+        weightChecker.recheckAllFiles(data)
+
+        .then(function(data) {
+            data.toolsResults.should.have.a.property('weightChecker');
+            data.toolsResults.weightChecker.should.have.a.property('metrics');
+            data.toolsResults.weightChecker.should.have.a.property('offenders');
+            done();
+        })
+
+        .fail(function(err) {
+            done(err);
+        });
+    });
+
+    it('should download a file and measure its size', function(done) {
+        var entry = {
+            method: 'GET',
+            url: 'http://localhost:8388/jquery1.8.3.js',
+            requestHeaders: {
+                'User-Agent': 'something',
+               Referer: 'http://www.google.fr/',
+               Accept: '*/*'
+            },
+            status: 200,
+            isJS: true
+        };
+
+        weightChecker.redownloadEntry(entry, function(err, newEntry) {
+            should.not.exist(err);
+
+            newEntry.weightCheck.bodySize.should.equal(93636);
+            newEntry.weightCheck.uncompressedSize.should.equal(newEntry.weightCheck.bodySize);
+            newEntry.weightCheck.isCompressed.should.equal(false);
+            newEntry.weightCheck.headersSize.should.be.above(300).and.below(350);
+            newEntry.weightCheck.body.should.have.string('1.8.3');
+
+            done();
+        });
+    });
+
+    it('should fail downloading a file in error', function(done) {
+        var entry = {
+            method: 'GET',
+            url: 'http://localhost:8388/no-file',
+            requestHeaders: {
+                'User-Agent': 'something',
+               Referer: 'http://www.google.fr/',
+               Accept: '*/*'
+            },
+            status: 200,
+            isHTML: true,
+            contentLength: 999
+        };
+
+        weightChecker.redownloadEntry(entry, function(err, newEntry) {
+            should.not.exist(err);
+
+            newEntry.weightCheck.should.have.a.property('message').that.equals('error while downloading: 404');
+
+            done();
+        });
+    });
+
+    it('should not download a non 200 request', function(done) {
+        var entry = {
+            method: 'GET',
+            url: 'http://localhost:8388/no-file',
+            requestHeaders: {
+                'User-Agent': 'something',
+               Referer: 'http://www.google.fr/',
+               Accept: '*/*'
+            },
+            status: 302,
+            isHTML: true,
+            contentLength: 999
+        };
+
+        weightChecker.redownloadEntry(entry, function(err, newEntry) {
+            should.not.exist(err);
+
+            newEntry.weightCheck.should.have.a.property('message').that.equals('only downloading requests with status code 200');
+
+            done();
+        });
+    });
+
+    it('should listRequestWeight', function() {
+        var totalWeightObj = weightChecker.listRequestWeight([{
+            method: 'GET',
+            url: 'http://localhost:8388/jquery1.8.3.js',
+            requestHeaders: {
+                'User-Agent': 'something',
+                Referer: 'http://www.google.fr/',
+                Accept: '*/*',
+                'Accept-Encoding': 'gzip, deflate'
+            },
+            status: 200,
+            isHTML: true,
+            type: 'html',
+            contentLength: 999,
+            weightCheck: {
+                body: 'blabla',
+                headersSize: 200,
+                bodySize: 500,
+                isCompressed: false,
+                uncompressedSize: 500
+            }
+        }]);
+
+        totalWeightObj.totalWeight.should.equal(700);
+        totalWeightObj.byType.html.totalWeight.should.equal(700);
+        totalWeightObj.byType.html.requests.should.have.length(1);
+        totalWeightObj.byType.html.requests[0].should.have.a.property('url').that.equals('http://localhost:8388/jquery1.8.3.js');
+        totalWeightObj.byType.html.requests[0].should.have.a.property('weight').that.equals(700);
+    });
+
+    it('should listRequestWeight even if download failed', function() {
+        var totalWeightObj = weightChecker.listRequestWeight([{
+            method: 'GET',
+            url: 'http://localhost:8388/jquery1.8.3.js',
+            requestHeaders: {
+                'User-Agent': 'something',
+                Referer: 'http://www.google.fr/',
+                Accept: '*/*',
+                'Accept-Encoding': 'gzip, deflate'
+            },
+            status: 200,
+            isHTML: true,
+            type: 'html',
+            contentLength: 999,
+            weightCheck: {
+                message: 'error while downloading: 404'
+            }
+        }]);
+
+        totalWeightObj.totalWeight.should.equal(999);
+    });
+
+});