|
@@ -11,6 +11,7 @@ var http = require('http');
|
|
var zlib = require('zlib');
|
|
var zlib = require('zlib');
|
|
var async = require('async');
|
|
var async = require('async');
|
|
var request = require('request');
|
|
var request = require('request');
|
|
|
|
+var md5 = require('md5');
|
|
|
|
|
|
var imageOptimizer = require('./imageOptimizer');
|
|
var imageOptimizer = require('./imageOptimizer');
|
|
var fileMinifier = require('./fileMinifier');
|
|
var fileMinifier = require('./fileMinifier');
|
|
@@ -106,9 +107,9 @@ var Redownload = function() {
|
|
metrics.emptyRequests = offenders.emptyRequests.length;
|
|
metrics.emptyRequests = offenders.emptyRequests.length;
|
|
|
|
|
|
|
|
|
|
- // Now emove unwanted responses (redirections)
|
|
|
|
|
|
+ // Now remove unwanted responses (redirections and empty files)
|
|
results = results.filter(function(result) {
|
|
results = results.filter(function(result) {
|
|
- return (result.status < 300 || result.status >= 400);
|
|
|
|
|
|
+ return ((result.status < 300 || result.status >= 400) && result.weightCheck.bodySize > 0);
|
|
});
|
|
});
|
|
|
|
|
|
|
|
|
|
@@ -128,6 +129,11 @@ var Redownload = function() {
|
|
offenders.smallRequests = listSmallRequests(results);
|
|
offenders.smallRequests = listSmallRequests(results);
|
|
metrics.smallRequests = offenders.smallRequests.total;
|
|
metrics.smallRequests = offenders.smallRequests.total;
|
|
|
|
|
|
|
|
+ // Detect identical files
|
|
|
|
+ offenders.identicalFiles = listIdenticalFiles(results);
|
|
|
|
+ metrics.identicalFiles = offenders.identicalFiles.avoidableRequests;
|
|
|
|
+
|
|
|
|
+
|
|
data.toolsResults.redownload = {
|
|
data.toolsResults.redownload = {
|
|
metrics: metrics,
|
|
metrics: metrics,
|
|
offenders: offenders
|
|
offenders: offenders
|
|
@@ -368,6 +374,45 @@ var Redownload = function() {
|
|
return results;
|
|
return results;
|
|
}
|
|
}
|
|
|
|
|
|
|
|
+ function listIdenticalFiles(requests) {
|
|
|
|
+ var hashes = {};
|
|
|
|
+ var list = [];
|
|
|
|
+ var avoidableRequestsCount = 0;
|
|
|
|
+
|
|
|
|
+ requests.forEach(function(req) {
|
|
|
|
+ var requestHash = md5(req.weightCheck.body);
|
|
|
|
+
|
|
|
|
+ // Try to exclude tracking pixels
|
|
|
|
+ if (req.weightCheck.bodySize < 80 && req.type === 'image') {
|
|
|
|
+ return;
|
|
|
|
+ }
|
|
|
|
+
|
|
|
|
+ if (!hashes[requestHash]) {
|
|
|
|
+ hashes[requestHash] = {
|
|
|
|
+ weight: req.weightCheck.bodySize,
|
|
|
|
+ urls: []
|
|
|
|
+ };
|
|
|
|
+ }
|
|
|
|
+
|
|
|
|
+ if (hashes[requestHash].urls.indexOf(req.url) === -1) {
|
|
|
|
+ hashes[requestHash].urls.push(req.url);
|
|
|
|
+ }
|
|
|
|
+ });
|
|
|
|
+
|
|
|
|
+ for (var hash in hashes) {
|
|
|
|
+ if (hashes[hash].urls.length > 1) {
|
|
|
|
+ list.push(hashes[hash]);
|
|
|
|
+ avoidableRequestsCount += hashes[hash].urls.length - 1;
|
|
|
|
+ }
|
|
|
|
+ }
|
|
|
|
+
|
|
|
|
+ return {
|
|
|
|
+ avoidableRequests: avoidableRequestsCount,
|
|
|
|
+ count: list.length,
|
|
|
|
+ list: list
|
|
|
|
+ };
|
|
|
|
+ }
|
|
|
|
+
|
|
|
|
|
|
function redownloadEntry(entry, httpAuth) {
|
|
function redownloadEntry(entry, httpAuth) {
|
|
var deferred = Q.defer();
|
|
var deferred = Q.defer();
|