Run YLT on AWS Lambda. TODO: should work both ways

front/src/views/queue.html

@@ -17,11 +17,11 @@
     </div>
     <div ng-if="status.statusCode == 'running'">
         <div class="status">Test is running...</div>
-        <div class="progress">
+        <!--<div class="progress">
             <div class="progressBarEmpty">
                 <div class="progressBarFilled" ng-style="{'width': (progress.estimatedProgress*100) + '%'}"></div>
             </div>
-        </div>
+        </div>-->
         <p class="statusSubMessage" ng-if="!progress">(Phantomas launched)</p>
         <p class="statusSubMessage" ng-if="progress.milestone == 'domReady'">(DOM Ready fired)</p>
         <p class="statusSubMessage" ng-if="progress.milestone == 'domComplete'">(page loaded, waiting for late requests)</p>

lib/server/controllers/apiController.js

@@ -1,5 +1,6 @@
 var debug               = require('debug')('ylt:server');
 var Q                   = require('q');
+var AWS                 = require('aws-sdk');
 
 var ylt                 = require('../../index');
 var ScreenshotHandler   = require('../../screenshotHandler');
@@ -57,13 +58,8 @@ var ApiController = function(app) {
         var queuePromise = queue.push(run.runId);
 
         // Save the run to the datastore
-        runsDatastore.add(run, queuePromise.startingPosition);
-
-
-        // Listening for position updates
-        queuePromise.progress(function(position) {
-            runsDatastore.updatePosition(run.runId, position);
-        });
+        //runsDatastore.add(run, queuePromise.startingPosition);
+        runsDatastore.add(run, 0);
 
         // Let's start the run
         queuePromise.then(function() {
@@ -85,126 +81,82 @@ var ApiController = function(app) {
                 noExternals: run.params.noExternals
             };
 
-            return ylt(run.params.url, runOptions)
-
-            // Update the progress bar on each progress
-            .progress(function(progress) {
-                runsDatastore.updateRunProgress(run.runId, progress);
-            });
-
-        })
-
-        // Phantomas completed
-        .then(function(data) {
-
-            debug('Success');
-            data.runId = run.runId;
-
+            const {region, arn} = chooseLambdaRegionByGeoIP(req.headers);
+            const lambda = new AWS.Lambda({region: region});
             
-            // Some conditional steps exist if there is a screenshot
-            var screenshotPromise = Q.resolve();
-
-            if (run.params.screenshot) {
-
-                var screenshotSize = serverSettings.screenshotWidth ? serverSettings.screenshotWidth[run.params.device] : 400;
+            return lambda.invoke({
+                FunctionName: arn,
+                InvocationType: 'RequestResponse',
+                Payload: JSON.stringify({url: run.params.url, id: run.runId, options: runOptions})
+            }).promise();
 
-                // Replace the empty promise created earlier with Q.resolve()
-                screenshotPromise = ScreenshotHandler.findAndOptimizeScreenshot(screenshotSize)
-                
-                    // Read screenshot
-                    .then(function(screenshotBuffer) {
-                        if (screenshotBuffer) {
-                            debug('Image optimized');
-                            data.screenshotBuffer = screenshotBuffer;
-                            data.screenshotUrl = '/api/results/' + data.runId + '/screenshot.jpg';
-                        }
-                    })
-
-                    // Don't worry if there's an error
-                    .fail(function(err) {
-                        debug('An error occured while creating the screenshot\'s thumbnail. Ignoring and continuing...');
-                        debug(err);
-                    });
+        })
 
+        .then(function(response) {
+            debug('We\'ve got a response from AWS Lambda');
+            debug('StatusCode = %d', response.StatusCode);
+            debug('Payload = %s', response.Payload);
+
+            if (response.StatusCode === 200 && response.Payload && response.Payload !== 'null') {
+                debug('Success!');
+                runsDatastore.markAsComplete(run.runId);
+            } else {
+                debug('Empty response from the lambda agent');
+                runsDatastore.markAsFailed(run.runId, "Empty response from the agent");
             }
-
-            // Let's continue
-            return screenshotPromise
-
-                // Save results
-                .then(function() {
-                    // Remove uneeded temp screenshot path
-                    delete data.params.options.screenshot;
-                    
-                    return resultsDatastore.saveResult(data);
-                })
-
-                // Mark as the run as complete and send the response if the request is still waiting
-                .then(function() {
-
-                    debug('Result saved in datastore');
-
-                    runsDatastore.markAsComplete(run.runId);
-
-                    if (run.params.waitForResponse) {
-
-                        // If the user only wants a portion of the result (partialResult option)
-                        switch(run.params.partialResult) {
-                            case 'generalScores': 
-                                res.redirect(302, '/api/results/' + run.runId + '/generalScores');
-                                break;
-                            case 'rules': 
-                                res.redirect(302, '/api/results/' + run.runId + '/rules');
-                                break;
-                            case 'javascriptExecutionTree':
-                                res.redirect(302, '/api/results/' + run.runId + '/javascriptExecutionTree');
-                                break;
-                            case 'phantomas':
-                                res.redirect(302, '/api/results/' + run.runId + '/toolsResults/phantomas');
-                                break;
-                            default:
-                                res.redirect(302, '/api/results/' + run.runId);
-                        }
-                    }
-                                    
-                })
-                .fail(function(err) {
-                    console.error('Test failed for URL: %s', run.params.url);
-                    console.error(err.toString());
-
-                    runsDatastore.markAsFailed(run.runId, err.toString());
-
-                    res.status(500).send('An error occured');
-                });
-
         })
 
-        .fail(function(err) {
-                    
-            console.error('Test failed for URL: %s', run.params.url);
-            console.error(err.toString());
+        .catch(function(err) {
+            debug('Error from AWS Lambda:');
+            debug(err);
 
             runsDatastore.markAsFailed(run.runId, err.toString());
-
-            res.status(400).send('Bad request');
-            
-        })
-
-        .finally(function() {
-            queue.remove(run.runId);
         });
 
-
         // The user doesn't want to wait for the response, sending the run ID only
-        if (!run.params.waitForResponse) {
-            debug('Sending response without waiting.');
-            res.setHeader('Content-Type', 'application/json');
-            res.send(JSON.stringify({runId: run.runId}));
-        }
+        debug('Sending response without waiting.');
+        res.setHeader('Content-Type', 'application/json');
+        res.send(JSON.stringify({runId: run.runId}));
 
     });
 
 
+    // Reads the Geoip_Continent_Code header and chooses the right region from the settings
+    function chooseLambdaRegionByGeoIP(headers) {
+
+        // The settings can be configured like this in server_config/settings.json:
+        //
+        // "awsHosting": {
+        //     "lambda": {
+        //         "regionByContinent": {
+        //             "AF": "eu-west-3",
+        //             "AS": "ap-southeast-1",
+        //             "EU": "eu-west-3",
+        //             "NA": "us-east-1",
+        //             "OC": "ap-southeast-1",
+        //             "SA": "us-east-1",
+        //             "default": "eu-west-3"
+        //         },
+        //         "arnByRegion": {
+        //             "us-east-1": "arn:aws:lambda:us-east-1:xxx:function:xxx",
+        //             "eu-west-3": "arn:aws:lambda:eu-west-3:xxx:function:xxx",
+        //             "ap-southeast-1": "arn:aws:lambda:ap-southeast-1:xxx:function:xxx"
+        //         }
+        //     }
+        // },
+
+        const header = headers.geoip_continent_code;
+        debug('Value of the Geoip_Continent_Code header: %s', header);
+
+        const continent = header || 'default';
+        const region = serverSettings.awsHosting.lambda.regionByContinent[continent];
+        const arn = serverSettings.awsHosting.lambda.arnByRegion[region];
+        debug('The chosen AWS Lambda is: %s', arn);
+
+        return {region, arn};
+    }
+
+
     // Retrive one run by id
     app.get('/api/runs/:id', function(req, res) {
         var runId = req.params.id;

lib/server/datastores/runsQueue.js

@@ -10,7 +10,8 @@ function RunsQueue() {
 
     this.push = function(runId) {
         var deferred = Q.defer();
-        var startingPosition = queue.length;
+        //var startingPosition = queue.length;
+        var startingPosition = 0;
 
         debug('Adding run %s to the queue, position is %d', runId, startingPosition);
 

package.json

@@ -28,6 +28,7 @@
     "angular-route": "1.7.7",
     "angular-sanitize": "1.7.7",
     "async": "2.6.1",
+    "aws-sdk": "2.862.0",
     "body-parser": "1.18.3",
     "chart.js": "2.7.3",
     "clean-css": "4.2.1",