Run YLT on AWS Lambda. TODO: should work both ways
This commit is contained in:
Gaël Métais
parent
9211074b32
commit
8558ea29df
4 changed files with 71 additions and 117 deletions
|
|
@ -17,11 +17,11 @@
|
|||
</div>
|
||||
<div ng-if="status.statusCode == 'running'">
|
||||
<div class="status">Test is running...</div>
|
||||
<div class="progress">
|
||||
<!--<div class="progress">
|
||||
<div class="progressBarEmpty">
|
||||
<div class="progressBarFilled" ng-style="{'width': (progress.estimatedProgress*100) + '%'}"></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>-->
|
||||
<p class="statusSubMessage" ng-if="!progress">(Phantomas launched)</p>
|
||||
<p class="statusSubMessage" ng-if="progress.milestone == 'domReady'">(DOM Ready fired)</p>
|
||||
<p class="statusSubMessage" ng-if="progress.milestone == 'domComplete'">(page loaded, waiting for late requests)</p>
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
var debug = require('debug')('ylt:server');
|
||||
var Q = require('q');
|
||||
var AWS = require('aws-sdk');
|
||||
|
||||
var ylt = require('../../index');
|
||||
var ScreenshotHandler = require('../../screenshotHandler');
|
||||
|
|
@ -57,13 +58,8 @@ var ApiController = function(app) {
|
|||
var queuePromise = queue.push(run.runId);
|
||||
|
||||
// Save the run to the datastore
|
||||
runsDatastore.add(run, queuePromise.startingPosition);
|
||||
|
||||
|
||||
// Listening for position updates
|
||||
queuePromise.progress(function(position) {
|
||||
runsDatastore.updatePosition(run.runId, position);
|
||||
});
|
||||
//runsDatastore.add(run, queuePromise.startingPosition);
|
||||
runsDatastore.add(run, 0);
|
||||
|
||||
// Let's start the run
|
||||
queuePromise.then(function() {
|
||||
|
|
@ -85,126 +81,82 @@ var ApiController = function(app) {
|
|||
noExternals: run.params.noExternals
|
||||
};
|
||||
|
||||
return ylt(run.params.url, runOptions)
|
||||
|
||||
// Update the progress bar on each progress
|
||||
.progress(function(progress) {
|
||||
runsDatastore.updateRunProgress(run.runId, progress);
|
||||
});
|
||||
|
||||
})
|
||||
|
||||
// Phantomas completed
|
||||
.then(function(data) {
|
||||
|
||||
debug('Success');
|
||||
data.runId = run.runId;
|
||||
|
||||
const {region, arn} = chooseLambdaRegionByGeoIP(req.headers);
|
||||
const lambda = new AWS.Lambda({region: region});
|
||||
|
||||
// Some conditional steps exist if there is a screenshot
|
||||
var screenshotPromise = Q.resolve();
|
||||
|
||||
if (run.params.screenshot) {
|
||||
|
||||
var screenshotSize = serverSettings.screenshotWidth ? serverSettings.screenshotWidth[run.params.device] : 400;
|
||||
|
||||
// Replace the empty promise created earlier with Q.resolve()
|
||||
screenshotPromise = ScreenshotHandler.findAndOptimizeScreenshot(screenshotSize)
|
||||
|
||||
// Read screenshot
|
||||
.then(function(screenshotBuffer) {
|
||||
if (screenshotBuffer) {
|
||||
debug('Image optimized');
|
||||
data.screenshotBuffer = screenshotBuffer;
|
||||
data.screenshotUrl = '/api/results/' + data.runId + '/screenshot.jpg';
|
||||
}
|
||||
})
|
||||
|
||||
// Don't worry if there's an error
|
||||
.fail(function(err) {
|
||||
debug('An error occured while creating the screenshot\'s thumbnail. Ignoring and continuing...');
|
||||
debug(err);
|
||||
});
|
||||
|
||||
}
|
||||
|
||||
// Let's continue
|
||||
return screenshotPromise
|
||||
|
||||
// Save results
|
||||
.then(function() {
|
||||
// Remove uneeded temp screenshot path
|
||||
delete data.params.options.screenshot;
|
||||
|
||||
return resultsDatastore.saveResult(data);
|
||||
})
|
||||
|
||||
// Mark as the run as complete and send the response if the request is still waiting
|
||||
.then(function() {
|
||||
|
||||
debug('Result saved in datastore');
|
||||
|
||||
runsDatastore.markAsComplete(run.runId);
|
||||
|
||||
if (run.params.waitForResponse) {
|
||||
|
||||
// If the user only wants a portion of the result (partialResult option)
|
||||
switch(run.params.partialResult) {
|
||||
case 'generalScores':
|
||||
res.redirect(302, '/api/results/' + run.runId + '/generalScores');
|
||||
break;
|
||||
case 'rules':
|
||||
res.redirect(302, '/api/results/' + run.runId + '/rules');
|
||||
break;
|
||||
case 'javascriptExecutionTree':
|
||||
res.redirect(302, '/api/results/' + run.runId + '/javascriptExecutionTree');
|
||||
break;
|
||||
case 'phantomas':
|
||||
res.redirect(302, '/api/results/' + run.runId + '/toolsResults/phantomas');
|
||||
break;
|
||||
default:
|
||||
res.redirect(302, '/api/results/' + run.runId);
|
||||
}
|
||||
}
|
||||
|
||||
})
|
||||
.fail(function(err) {
|
||||
console.error('Test failed for URL: %s', run.params.url);
|
||||
console.error(err.toString());
|
||||
|
||||
runsDatastore.markAsFailed(run.runId, err.toString());
|
||||
|
||||
res.status(500).send('An error occured');
|
||||
});
|
||||
return lambda.invoke({
|
||||
FunctionName: arn,
|
||||
InvocationType: 'RequestResponse',
|
||||
Payload: JSON.stringify({url: run.params.url, id: run.runId, options: runOptions})
|
||||
}).promise();
|
||||
|
||||
})
|
||||
|
||||
.fail(function(err) {
|
||||
|
||||
console.error('Test failed for URL: %s', run.params.url);
|
||||
console.error(err.toString());
|
||||
.then(function(response) {
|
||||
debug('We\'ve got a response from AWS Lambda');
|
||||
debug('StatusCode = %d', response.StatusCode);
|
||||
debug('Payload = %s', response.Payload);
|
||||
|
||||
if (response.StatusCode === 200 && response.Payload && response.Payload !== 'null') {
|
||||
debug('Success!');
|
||||
runsDatastore.markAsComplete(run.runId);
|
||||
} else {
|
||||
debug('Empty response from the lambda agent');
|
||||
runsDatastore.markAsFailed(run.runId, "Empty response from the agent");
|
||||
}
|
||||
})
|
||||
|
||||
.catch(function(err) {
|
||||
debug('Error from AWS Lambda:');
|
||||
debug(err);
|
||||
|
||||
runsDatastore.markAsFailed(run.runId, err.toString());
|
||||
|
||||
res.status(400).send('Bad request');
|
||||
|
||||
})
|
||||
|
||||
.finally(function() {
|
||||
queue.remove(run.runId);
|
||||
});
|
||||
|
||||
|
||||
// The user doesn't want to wait for the response, sending the run ID only
|
||||
if (!run.params.waitForResponse) {
|
||||
debug('Sending response without waiting.');
|
||||
res.setHeader('Content-Type', 'application/json');
|
||||
res.send(JSON.stringify({runId: run.runId}));
|
||||
}
|
||||
debug('Sending response without waiting.');
|
||||
res.setHeader('Content-Type', 'application/json');
|
||||
res.send(JSON.stringify({runId: run.runId}));
|
||||
|
||||
});
|
||||
|
||||
|
||||
// Reads the Geoip_Continent_Code header and chooses the right region from the settings
|
||||
function chooseLambdaRegionByGeoIP(headers) {
|
||||
|
||||
// The settings can be configured like this in server_config/settings.json:
|
||||
//
|
||||
// "awsHosting": {
|
||||
// "lambda": {
|
||||
// "regionByContinent": {
|
||||
// "AF": "eu-west-3",
|
||||
// "AS": "ap-southeast-1",
|
||||
// "EU": "eu-west-3",
|
||||
// "NA": "us-east-1",
|
||||
// "OC": "ap-southeast-1",
|
||||
// "SA": "us-east-1",
|
||||
// "default": "eu-west-3"
|
||||
// },
|
||||
// "arnByRegion": {
|
||||
// "us-east-1": "arn:aws:lambda:us-east-1:xxx:function:xxx",
|
||||
// "eu-west-3": "arn:aws:lambda:eu-west-3:xxx:function:xxx",
|
||||
// "ap-southeast-1": "arn:aws:lambda:ap-southeast-1:xxx:function:xxx"
|
||||
// }
|
||||
// }
|
||||
// },
|
||||
|
||||
const header = headers.geoip_continent_code;
|
||||
debug('Value of the Geoip_Continent_Code header: %s', header);
|
||||
|
||||
const continent = header || 'default';
|
||||
const region = serverSettings.awsHosting.lambda.regionByContinent[continent];
|
||||
const arn = serverSettings.awsHosting.lambda.arnByRegion[region];
|
||||
debug('The chosen AWS Lambda is: %s', arn);
|
||||
|
||||
return {region, arn};
|
||||
}
|
||||
|
||||
|
||||
// Retrive one run by id
|
||||
app.get('/api/runs/:id', function(req, res) {
|
||||
var runId = req.params.id;
|
||||
|
|
|
|||
|
|
@ -10,7 +10,8 @@ function RunsQueue() {
|
|||
|
||||
this.push = function(runId) {
|
||||
var deferred = Q.defer();
|
||||
var startingPosition = queue.length;
|
||||
//var startingPosition = queue.length;
|
||||
var startingPosition = 0;
|
||||
|
||||
debug('Adding run %s to the queue, position is %d', runId, startingPosition);
|
||||
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@
|
|||
"angular-route": "1.7.7",
|
||||
"angular-sanitize": "1.7.7",
|
||||
"async": "2.6.1",
|
||||
"aws-sdk": "2.862.0",
|
||||
"body-parser": "1.18.3",
|
||||
"chart.js": "2.7.3",
|
||||
"clean-css": "4.2.1",
|
||||
|
|
|
|||
Loading…
Reference in a new issue