0ct0pu5/YellowLabTools
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge branch 'aws' into master

Browse source
This commit is contained in:
Gaël Métais 2021-03-15 23:19:26 +00:00 committed by GitHub
commit 5f12d1ce0e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
23 changed files with 302 additions and 8264 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.gitignore vendored
View file

@ -7,5 +7,6 @@ results/*
coverage
front/build
package-lock.json
yarn.lock
har.json
.idea/

2
bin/cli.js
View file

@ -15,7 +15,6 @@ var cli = meow({
'Options:',
' --device Simulates a device. Choose between phone (default), tablet, desktop and desktop-hd.',
' --screenshot Will take a screenshot and use this value as the output path. It needs to end with ".png".',
//' --wait-for-selector Once the page is loaded, Phantomas will wait until the given CSS selector matches some elements.',
' --proxy Sets an HTTP proxy to pass through. Syntax is "host:port".',
' --cookie Adds a cookie on the main domain.',
' --auth-user Basic HTTP authentication username.',
@ -98,7 +97,6 @@ if (cli.flags.reporter && cli.flags.reporter !== 'json' && cli.flags.reporter !=
// Remove some heavy parts of the results object
delete data.toolsResults;
delete data.javascriptExecutionTree;
var xmlOutput = serializer.render(data);

4
front/src/views/queue.html
View file

@ -17,11 +17,11 @@
</div>
<div ng-if="status.statusCode == 'running'">
<div class="status">Test is running...</div>
<div class="progress">
<!--<div class="progress">
<div class="progressBarEmpty">
<div class="progressBarFilled" ng-style="{'width': (progress.estimatedProgress*100) + '%'}"></div>
</div>
</div>
</div>-->
<p class="statusSubMessage" ng-if="!progress">(Phantomas launched)</p>
<p class="statusSubMessage" ng-if="progress.milestone == 'domReady'">(DOM Ready fired)</p>
<p class="statusSubMessage" ng-if="progress.milestone == 'domComplete'">(page loaded, waiting for late requests)</p>

10
front/src/views/rule.html
View file

@ -179,10 +179,18 @@
(<ng-pluralize count="offender.requests" when="{'one':'1 request','other':'{} requests'}"></ng-pluralize>)
</div>
<div ng-if="policyName === 'globalVariables' || policyName === 'jQueryVersionsLoaded' || policyName === 'synchronousXHR'">
<div ng-if="policyName === 'globalVariables'">
{{offender}}
</div>
<div ng-if="policyName === 'jQueryVersionsLoaded'">
{{offender.version}}
</div>
<div ng-if="policyName === 'synchronousXHR'">
{{offender.url}}
</div>
<div ng-if="policyName === 'fontsCount'">
<url-link url="offender.url" max-length="70"></url-link>
({{offender.size | bytes}})

10
lib/metadata/policies.js
View file

@ -538,7 +538,7 @@ var policies = {
"message": "<p>It can be useful, but only as a last resort. It is a bad practice because it overrides the normal cascading logic. The more you use !important, the more you need it again to over-override. This conducts to a poor maintainability.</p>",
"isOkThreshold": 0,
"isBadThreshold": 200,
"isAbnormalThreshold": 500,
"isAbnormalThreshold": 1000,
"hasOffenders": true,
"offendersTransformFn": function(offenders) {
var parsedOffenders = offenders.map(function(offender) {
@ -757,8 +757,8 @@ var policies = {
},
"compression": {
"tool": "redownload",
"label": "Brotli compression",
"message": "<p>Measures the number of bytes that could be saved by compressing textual files. Some files listed below might not be compressed at all, some might be already compressed with Gzip but would become even lighter with Brotli.</p><p>All major server systems are now compatible with Brotli.</p><p>Note that compressing small files (< 1 KB) is arguable, and that some assets such as images should not be compressed as it is already included in their format. <a href=\"https://gist.github.com/gmetais/971ce13a1fbeebd88445\" target=\"_blank\">Here</a> is a list of Content-Types that should be compressed.</p>",
"label": "Gzip/Brotli compression",
"message": "<p>Measures the number of bytes that could be saved by compressing textual files. Some files listed below might not be compressed at all, some might be already compressed with Gzip but would become even lighter with Brotli.</p><p>All major server systems are now compatible with Brotli.</p><p>Note that compressing small files (< 1 KB) is arguable, and that some assets such as images should not be compressed as it is already included in their format. <a href=\"https://letstalkaboutwebperf.com/en/gzip-brotli-server-config/\" target=\"_blank\">Here</a> is a list of MIME types that should be compressed.</p>",
"isOkThreshold": 20480,
"isBadThreshold": 204800,
"isAbnormalThreshold": 409600,
@ -875,7 +875,7 @@ var policies = {
return offenders;
}
},
/*"unusedUnicodeRanges": {
"unusedUnicodeRanges": {
"tool": "redownload",
"label": "Unused Unicode ranges",
"message": "<p>This metric counts the number of unused Unicode ranges inside each font. For example, one font could include Cyrillic glyphs but none of them are used on the page.</p><p>It also reveals the number of ligatures (letters that are represented differently when close to each other) and hidden chars (glyphs not linked to the unicode system that can't be displayed on the web).</p><p>Because of technical limitations, Yellow Lab Tools checks each font against the glyphs of the entire page. As a result, estimated use is >= to reality. For example, if you read that 10 glyphs are \"possibly used\", it means that these 10 glyphs are used on the page but nothing guaranties that they are displayed using this font.</p><p>Tools such as <a href=\"https://www.fontsquirrel.com/tools/webfont-generator\" target=\"_blank\">Font Squirrel</a> can remove some unicode ranges from a font.</p><p>In the case of an icon font, make sure you only keep the icons that are used on the website and to remove the others. Several tools are able to extract SVG images from a font, then some other tools can generate a font from the SVGs you want to keep.</p>",
@ -886,7 +886,7 @@ var policies = {
"offendersTransformFn": function(offenders) {
return offenders;
}
},*/
},
"nonWoff2Fonts": {
"tool": "redownload",
"label": "WOFF 2",

1
lib/metadata/scoreProfileGeneric.json
View file

@ -86,6 +86,7 @@
"policies": {
"fontsCount": 1,
"heavyFonts": 0.5,
"unusedUnicodeRanges": 0.5,
"nonWoff2Fonts": 0.5
}
},

21
lib/offendersHelpers.js
View file

@ -117,7 +117,7 @@ var OffendersHelpers = function() {
for (var i=0 ; i<traceArray.length ; i++) {
// Handle the new PhantomJS 2.x syntax
parts = /^(([\w$]+)@)?([^ ]+):(\d+):(\d+)$/.exec(traceArray[i]);
parts = /^\s*at( (\w+))? \(?([^ ]+):(\d+):(\d+)\)?$$/.exec(traceArray[i]);
if (parts) {
obj = {
@ -132,25 +132,6 @@ var OffendersHelpers = function() {
results.push(obj);
} else {
// Old syntax
parts = /^(([\w$]+) )?\(?([^ ]+):(\d+)\)?$/.exec(traceArray[i]);
if (parts) {
obj = {
file: parts[3],
line: parseInt(parts[4], 10)
};
if (parts[2]) {
obj.functionName = parts[2];
}
results.push(obj);
} else {
return null;
}
}
}
return results;

6
lib/runner.js
View file

@ -78,6 +78,12 @@ var Runner = function(params) {
milestone: 'redownload'
});
// Fix: don't display Unicode ranges if the module is not present in Phantomas
if (!data.toolsResults.phantomas.metrics.charactersCount) {
delete data.toolsResults.redownload.metrics.unusedUnicodeRanges;
delete data.toolsResults.redownload.offenders.unusedUnicodeRanges;
}
// Rules checker
var policies = require('./metadata/policies');
data.rules = rulesChecker.check(data, policies);

43
lib/screenshotHandler.js
View file

@ -7,21 +7,16 @@ var path = require('path');
var screenshotHandler = function() {
var tmpFolderPath = 'tmp';
var tmpFolderFullPath = path.join(__dirname, '..', tmpFolderPath);
var tmpFileName = 'temp-screenshot.png';
var tmpFileFullPath = path.join(tmpFolderFullPath, tmpFileName);
this.findAndOptimizeScreenshot = function(width) {
var that = this;
debug('Starting screenshot transformation');
return this.openImage(tmpFileFullPath)
return this.openImage(this.getTmpFileRelativePath())
.then(function(image) {
that.deleteTmpFile(tmpFileFullPath);
that.deleteTmpFile(that.getTmpFileRelativePath());
return that.resizeImage(image, width);
})
@ -97,7 +92,7 @@ var screenshotHandler = function() {
this.deleteTmpFile = function(tmpFilePath) {
var deferred = Q.defer();
fs.unlink(tmpFilePath, function (err) {
fs.unlink(this.getTmpFileRelativePath(), function (err) {
if (err) {
debug('Screenshot temporary file not found, could not be deleted. But it is not a problem.');
} else {
@ -110,31 +105,17 @@ var screenshotHandler = function() {
return deferred.promise;
};
// Create a /tmp folder on the project's root directory
this.createTmpScreenshotFolder = function() {
var deferred = Q.defer();
// Create the folder if it doesn't exist
fs.exists(tmpFolderFullPath, function(exists) {
if (exists) {
deferred.resolve();
} else {
debug('Creating the tmp image folder', tmpFolderFullPath);
fs.mkdir(tmpFolderFullPath, function(err) {
if (err) {
deferred.reject(err);
} else {
deferred.resolve();
}
});
}
});
return deferred.promise;
};
this.getTmpFileRelativePath = function() {
return tmpFolderPath + '/' + tmpFileName;
// Chrome saves a temporary file on the disk, which is then removed.
// Its default folder is /tmp, but it can be changed in server_config/settings.json
var serverSettings = require('../server_config/settings.json');
var tmpFolderPath = serverSettings.screenshotTempPath || '/tmp';
var tmpFileName = 'temp-chrome-screenshot.png';
var tmpFileFullPath = path.join(tmpFolderPath, tmpFileName);
return tmpFileFullPath;
};
};

194
lib/server/controllers/apiController.js
View file

@ -1,14 +1,16 @@
var debug = require('debug')('ylt:server');
var Q = require('q');
var AWS = require('aws-sdk');
var ylt = require('../../index');
var ScreenshotHandler = require('../../screenshotHandler');
var RunsQueue = require('../datastores/runsQueue');
var RunsDatastore = require('../datastores/runsDatastore');
var ResultsDatastore = require('../datastores/resultsDatastore');
var serverSettings = (process.env.IS_TEST) ? require('../../../test/fixtures/settings.json') : require('../../../server_config/settings.json');
var ResultsDatastore = (serverSettings.awsHosting) ? require('../datastores/awsResultsDatastore') : require('../datastores/resultsDatastore');
var ApiController = function(app) {
'use strict';
@ -51,21 +53,13 @@ var ApiController = function(app) {
}
};
// Create the tmp folder if it doesn't exist
ScreenshotHandler.createTmpScreenshotFolder(run.runId);
// Add test to the testQueue
debug('Adding test %s to the queue', run.runId);
var queuePromise = queue.push(run.runId);
// Save the run to the datastore
runsDatastore.add(run, queuePromise.startingPosition);
// Listening for position updates
queuePromise.progress(function(position) {
runsDatastore.updatePosition(run.runId, position);
});
//runsDatastore.add(run, queuePromise.startingPosition);
runsDatastore.add(run, 0);
// Let's start the run
queuePromise.then(function() {
@ -87,129 +81,82 @@ var ApiController = function(app) {
noExternals: run.params.noExternals
};
return ylt(run.params.url, runOptions)
// Update the progress bar on each progress
.progress(function(progress) {
runsDatastore.updateRunProgress(run.runId, progress);
});
})
// Phantomas completed
.then(function(data) {
debug('Success');
data.runId = run.runId;
const {region, arn} = chooseLambdaRegionByGeoIP(req.headers);
const lambda = new AWS.Lambda({region: region});
// Some conditional steps exist if there is a screenshot
var screenshotPromise = Q.resolve();
if (run.params.screenshot) {
var screenshotSize = serverSettings.screenshotWidth ? serverSettings.screenshotWidth[run.params.device] : 400;
// Replace the empty promise created earlier with Q.resolve()
screenshotPromise = ScreenshotHandler.findAndOptimizeScreenshot(screenshotSize)
// Read screenshot
.then(function(screenshotBuffer) {
if (screenshotBuffer) {
debug('Image optimized');
data.screenshotBuffer = screenshotBuffer;
data.screenshotUrl = '/api/results/' + data.runId + '/screenshot.jpg';
}
})
// Don't worry if there's an error
.fail(function(err) {
debug('An error occured while creating the screenshot\'s thumbnail. Ignoring and continuing...');
debug(err);
});
}
// Let's continue
return screenshotPromise
// Save results
.then(function() {
// Remove uneeded temp screenshot path
delete data.params.options.screenshot;
// Here we can remove tools results if not needed
delete data.toolsResults.phantomas.offenders.requests;
return resultsDatastore.saveResult(data);
})
// Mark as the run as complete and send the response if the request is still waiting
.then(function() {
debug('Result saved in datastore');
runsDatastore.markAsComplete(run.runId);
if (run.params.waitForResponse) {
// If the user only wants a portion of the result (partialResult option)
switch(run.params.partialResult) {
case 'generalScores':
res.redirect(302, '/api/results/' + run.runId + '/generalScores');
break;
case 'rules':
res.redirect(302, '/api/results/' + run.runId + '/rules');
break;
case 'javascriptExecutionTree':
res.redirect(302, '/api/results/' + run.runId + '/javascriptExecutionTree');
break;
case 'phantomas':
res.redirect(302, '/api/results/' + run.runId + '/toolsResults/phantomas');
break;
default:
res.redirect(302, '/api/results/' + run.runId);
}
}
})
.fail(function(err) {
console.error('Test failed for URL: %s', run.params.url);
console.error(err.toString());
runsDatastore.markAsFailed(run.runId, err.toString());
res.status(500).send('An error occured');
});
return lambda.invoke({
FunctionName: arn,
InvocationType: 'RequestResponse',
Payload: JSON.stringify({url: run.params.url, id: run.runId, options: runOptions})
}).promise();
})
.fail(function(err) {
console.error('Test failed for URL: %s', run.params.url);
console.error(err.toString());
.then(function(response) {
debug('We\'ve got a response from AWS Lambda');
debug('StatusCode = %d', response.StatusCode);
debug('Payload = %s', response.Payload);
if (response.StatusCode === 200 && response.Payload && response.Payload !== 'null') {
debug('Success!');
runsDatastore.markAsComplete(run.runId);
} else {
debug('Empty response from the lambda agent');
runsDatastore.markAsFailed(run.runId, "Empty response from the agent");
}
})
.catch(function(err) {
debug('Error from AWS Lambda:');
debug(err);
runsDatastore.markAsFailed(run.runId, err.toString());
res.status(400).send('Bad request');
})
.finally(function() {
queue.remove(run.runId);
});
// The user doesn't want to wait for the response, sending the run ID only
if (!run.params.waitForResponse) {
debug('Sending response without waiting.');
res.setHeader('Content-Type', 'application/json');
res.send(JSON.stringify({runId: run.runId}));
}
debug('Sending response without waiting.');
res.setHeader('Content-Type', 'application/json');
res.send(JSON.stringify({runId: run.runId}));
});
// Reads the Geoip_Continent_Code header and chooses the right region from the settings
function chooseLambdaRegionByGeoIP(headers) {
// The settings can be configured like this in server_config/settings.json:
//
// "awsHosting": {
// "lambda": {
// "regionByContinent": {
// "AF": "eu-west-3",
// "AS": "ap-southeast-1",
// "EU": "eu-west-3",
// "NA": "us-east-1",
// "OC": "ap-southeast-1",
// "SA": "us-east-1",
// "default": "eu-west-3"
// },
// "arnByRegion": {
// "us-east-1": "arn:aws:lambda:us-east-1:xxx:function:xxx",
// "eu-west-3": "arn:aws:lambda:eu-west-3:xxx:function:xxx",
// "ap-southeast-1": "arn:aws:lambda:ap-southeast-1:xxx:function:xxx"
// }
// }
// },
const header = headers.geoip_continent_code;
debug('Value of the Geoip_Continent_Code header: %s', header);
const continent = header || 'default';
const region = serverSettings.awsHosting.lambda.regionByContinent[continent];
const arn = serverSettings.awsHosting.lambda.arnByRegion[region];
debug('The chosen AWS Lambda is: %s', arn);
return {region, arn};
}
// Retrive one run by id
app.get('/api/runs/:id', function(req, res) {
var runId = req.params.id;
@ -227,7 +174,10 @@ var ApiController = function(app) {
// Counts all pending runs
app.get('/api/runs', function(req, res) {
res.setHeader('Content-Type', 'application/json');
res.send(JSON.stringify({pendingRuns: queue.length()}, null, 2));
res.send(JSON.stringify({
pendingRuns: queue.length(),
timeSinceLastTestStarted: queue.timeSinceLastTestStarted()
}, null, 2));
});
// Delete one run by id

119
lib/server/datastores/awsResultsDatastore.js Normal file
View file

@ -0,0 +1,119 @@
const Q = require('q');
const debug = require('debug')('ylt:resultsDatastore');
const path = require('path');
const AWS = require('aws-sdk');
function ResultsDatastore() {
'use strict';
const serverSettings = require('../../../server_config/settings.json');
const s3 = new AWS.S3();
const resultFileName = 'results.json';
const resultScreenshotName = 'screenshot.jpg';
const resultsFolderName = 'results';
this.saveResult = function(testResults) {
const resultFilePath = path.join(resultsFolderName, testResults.runId, resultFileName);
const screenshotFilePath = path.join(resultsFolderName, testResults.runId, resultScreenshotName);
debug('Starting to save screenshot then results.json file on s3...');
return saveScreenshotIfExists(testResults, screenshotFilePath)
.then(function() {
debug('Saving results file to s3, destination is %s', resultFilePath);
return s3PutObject(resultFilePath, JSON.stringify(testResults, null, 2));
});
};
this.getResult = function(runId) {
const resultFilePath = path.join(resultsFolderName, runId, resultFileName);
debug('Reading results (runID = %s) from AWS s3...', runId);
return s3GetObject(resultFilePath).then(function(bodyBuffer) {
return JSON.parse(bodyBuffer.toString('utf-8'));
});
};
// If there is a screenshot, save it as screenshot.jpg in the same folder as the results
function saveScreenshotIfExists(testResults, imagePath) {
var deferred = Q.defer();
if (testResults.screenshotBuffer) {
s3PutObject(imagePath, testResults.screenshotBuffer)
.fail(function() {
debug('Image %s could not be saved on s3. Ignoring.', imagePath);
})
.finally(function() {
delete testResults.screenshotBuffer;
deferred.resolve();
});
} else {
debug('Screenshot not found');
deferred.resolve();
}
return deferred.promise;
}
this.getScreenshot = function(runId) {
const screenshotFilePath = path.join(resultsFolderName, runId, resultScreenshotName);
debug('Retrieving screenshot (runID = %s) from s3...', runId);
return s3GetObject(screenshotFilePath);
};
function s3PutObject(path, body, ignoreError) {
var deferred = Q.defer();
s3.putObject({
Bucket: serverSettings.awsHosting.s3.bucket,
Key: path,
Body: body
}, function(err, data) {
if (err) {
debug('Could not save file %s on s3', path);
debug(err);
deferred.reject('File saving failed on s3');
} else {
debug('File %s saved on s3', path);
deferred.resolve();
}
});
return deferred.promise;
}
function s3GetObject(path) {
var deferred = Q.defer();
s3.getObject({
Bucket: serverSettings.awsHosting.s3.bucket,
Key: path
}, function(err, data) {
if (err) {
debug('Failed retrieving object %s from s3', path);
debug(err);
deferred.reject(err);
} else {
debug('Response for %s received from s3...', path);
deferred.resolve(data.Body);
}
});
return deferred.promise;
}
}
module.exports = ResultsDatastore;

8
lib/server/datastores/resultsDatastore.js
View file

@ -49,13 +49,13 @@ function ResultsDatastore() {
};
this.deleteResult = function(runId) {
/*this.deleteResult = function(runId) {
var folder = path.join(resultsDir, runId);
debug('Deleting results (runID = %s) from disk...', runId);
return Q.nfcall(rimraf, folder);
};
};*/
// The folder /results/folderName/
@ -92,10 +92,6 @@ function ResultsDatastore() {
return deferred.promise;
}
this.getResultFolder = function(runId) {
return path.join(resultsDir, runId);
};
// If there is a screenshot, save it as screenshot.jpg in the same folder as the results
function saveScreenshotIfExists(testResults, path) {
var deferred = Q.defer();

12
lib/server/datastores/runsQueue.js
View file

@ -6,11 +6,12 @@ function RunsQueue() {
'use strict';
var queue = [];
var lastTestTimestamp = 0;
this.push = function(runId) {
var deferred = Q.defer();
var startingPosition = queue.length;
//var startingPosition = queue.length;
var startingPosition = 0;
debug('Adding run %s to the queue, position is %d', runId, startingPosition);
@ -21,6 +22,7 @@ function RunsQueue() {
runId: runId
});
lastTestTimestamp = Date.now();
deferred.resolve();
} else {
@ -31,6 +33,7 @@ function RunsQueue() {
deferred.notify(position);
},
itIsTimeCallback: function() {
lastTestTimestamp = Date.now();
deferred.resolve();
}
});
@ -78,6 +81,11 @@ function RunsQueue() {
this.length = function() {
return queue.length;
};
// Returns the number of seconds since the last test was launched
this.timeSinceLastTestStarted = function() {
return Math.round((Date.now() - lastTestTimestamp) / 1000);
};
}
module.exports = RunsQueue;

10
lib/server/middlewares/apiLimitsMiddleware.js
View file

@ -13,7 +13,14 @@ var apiLimitsMiddleware = function(req, res, next) {
if (req.path.indexOf('/api/') === 0 && !res.locals.hasApiKey) {
if (req.path === '/api/runs') {
// Monitoring requests
if (req.path === '/api/runs' && req.method === 'GET') {
next();
return;
}
// New tests
if (req.path === '/api/runs' && req.method === 'POST') {
if (!runsTable.accepts(ipAddress)) {
// Sorry :/
@ -24,6 +31,7 @@ var apiLimitsMiddleware = function(req, res, next) {
}
// Every other calls
if (!callsTable.accepts(ipAddress)) {
// Sorry :/
debug('Too many API requests from IP address %s', ipAddress);

2
lib/tools/phantomas/phantomasWrapper.js
View file

@ -47,7 +47,7 @@ var PhantomasWrapper = function() {
// Mandatory
'analyze-css': true,
'ignore-ssl-errors': true
'ignoreSslErrors': true
};

18
lib/tools/redownload/contentTypeChecker.js
View file

@ -10,6 +10,7 @@ var isWoff2 = require('is-woff2');
var isOtf = require('is-otf');
var isTtf = require('is-ttf');
var isEot = require('is-eot');
var isJson = require('is-json');
var ContentTypeChecker = function() {
@ -57,7 +58,7 @@ var ContentTypeChecker = function() {
foundType = findContentType(entry.weightCheck.bodyBuffer);
// If it's an image or a font, then rewrite.
if (foundType !== null && (foundType.type === 'image' || foundType.type === 'webfont')) {
if (foundType !== null && (foundType.type === 'image' || foundType.type === 'webfont' || foundType.type === 'json')) {
if (foundType.type !== entry.type) {
debug('Content type %s is wrong for %s. It should be %s.', entry.type, entry.ulr, foundType.type);
}
@ -119,6 +120,10 @@ var ContentTypeChecker = function() {
return contentTypes.eot;
}
if (isJson(bodyStr)) {
return contentTypes.json;
}
return null;
}
@ -224,7 +229,16 @@ var ContentTypeChecker = function() {
entry.type = 'webfont';
entry.isWebFont = true;
}
}
},
json: {
type: 'json',
mimes: ['application/json'],
updateFn: function(entry) {
entry.type = 'json';
entry.isJSON = true;
}
},
};
return {

2
lib/tools/redownload/fileMinifier.js
View file

@ -224,7 +224,7 @@ var FileMinifier = function() {
return result;
}
// Avoid losing some trying to compress JS files if they already look minified
// Avoid losing time trying to compress JS files if they already look minified
// by counting the number of lines compared to the total size.
// Less than 2KB per line is suspicious
function looksAlreadyMinified(code) {

2
lib/tools/redownload/imageOptimizer.js
View file

@ -210,7 +210,7 @@ var ImageOptimizer = function() {
}
imagemin.buffer(imageBody, {use: engine})
imagemin.buffer(imageBody, {plugins: [engine]})
.then(function(file) {
var endTime = Date.now();

15
lib/tools/redownload/redownload.js
View file

@ -65,8 +65,8 @@ var Redownload = function() {
// Prevent a bug with the font analyzer on empty pages
var differentCharacters = '';
if (data.toolsResults.phantomas.offenders.differentCharacters && data.toolsResults.phantomas.offenders.differentCharacters.length > 0) {
differentCharacters = data.toolsResults.phantomas.offenders.differentCharacters[0];
if (data.toolsResults.phantomas.offenders.charactersCount && data.toolsResults.phantomas.offenders.charactersCount.length > 0) {
differentCharacters = data.toolsResults.phantomas.offenders.charactersCount[0];
}
// Transform every request into a download function with a callback when done
@ -195,6 +195,8 @@ var Redownload = function() {
offenders: offenders
};
cleanResults(results);
deferred.resolve(data);
}
});
@ -823,6 +825,7 @@ var Redownload = function() {
reqHeaders['Accept'] = '*/*,image/webp';
reqHeaders['Accept-Encoding'] = 'gzip, deflate, br';
reqHeaders['Connection'] = 'keep-alive';
reqHeaders['User-Agent'] = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.182 Safari/537.36';
var requestOptions = {
method: entry.method,
@ -899,6 +902,7 @@ var Redownload = function() {
var result = {
bodyBuffer: body,
headersSize: Buffer.byteLength(rawHeaders, 'utf8'),
headers: res.headers,
bodySize: bodySize,
isCompressed: isCompressed,
compressionTool: compressionTool,
@ -1008,6 +1012,13 @@ var Redownload = function() {
}
}
// Clean all the pollution this module added to the results
function cleanResults(requests) {
requests.forEach(function(req) {
delete req.weightCheck;
});
}
return {
recheckAllFiles: recheckAllFiles,
listRequestWeight: listRequestWeight,

16
package.json
View file

@ -1,6 +1,6 @@
{
"name": "yellowlabtools",
"version": "2.0.0-beta",
"version": "2.0.0",
"description": "Online tool to audit a webpage for performance and front-end quality issues",
"license": "GPL-2.0",
"author": {
@ -28,6 +28,7 @@
"angular-route": "1.7.7",
"angular-sanitize": "1.7.7",
"async": "2.6.1",
"aws-sdk": "2.862.0",
"body-parser": "1.18.3",
"chart.js": "2.7.3",
"clean-css": "4.2.1",
@ -42,14 +43,15 @@
"fontkit": "1.7.8",
"html-minifier": "4.0.0",
"image-size": "0.7.1",
"imagemin": "6.1.0",
"imagemin-jpegoptim": "6.0.0",
"imagemin-jpegtran": "6.0.0",
"imagemin-optipng": "6.0.0",
"imagemin-svgo": "7.0.0",
"imagemin": "7.0.1",
"imagemin-jpegoptim": "7.0.0",
"imagemin-jpegtran": "7.0.0",
"imagemin-optipng": "8.0.0",
"imagemin-svgo": "8.0.0",
"is-eot": "1.0.0",
"is-gif": "3.0.0",
"is-jpg": "2.0.0",
"is-json": "2.0.1",
"is-otf": "0.1.2",
"is-png": "1.1.0",
"is-svg": "3.0.0",
@ -67,7 +69,7 @@
"request": "2.88.0",
"rimraf": "2.6.3",
"temporary": "0.0.8",
"ttf2woff2": "3.0.0",
"ttf2woff2": "4.0.1",
"uglify-js": "3.4.9",
"woff-tools": "0.1.0"
},

10
server_config/settings-prod.json
View file

@ -1,16 +1,18 @@
{
"serverPort": 80,
"baseUrl": "/",
"googleAnalyticsId": "",
"screenshotWidth": {
"phone": 360,
"tablet": 420,
"desktop": 600,
"desktop-hd": 600
},
"baseUrl": "/",
"authorizedKeys": {
},
"screenshotTempPath": "/tmp/",
"authorizedKeys": {},
"maxAnonymousRunsPerDay": 1000,
"maxAnonymousCallsPerDay": 100000,
"blockedUrls": [],

10
server_config/settings.json
View file

@ -1,16 +1,18 @@
{
"serverPort": 8383,
"baseUrl": "/",
"googleAnalyticsId": "",
"screenshotWidth": {
"phone": 360,
"tablet": 420,
"desktop": 600,
"desktop-hd": 600
},
"baseUrl": "/",
"authorizedKeys": {
},
"screenshotTempPath": "/tmp/",
"authorizedKeys": {},
"maxAnonymousRunsPerDay": 99999999,
"maxAnonymousCallsPerDay": 99999999,
"blockedUrls": [],

8050
yarn.lock
View file

File diff suppressed because it is too large Load diff