API is starting to run

Gruntfile.js

@@ -71,13 +71,13 @@ module.exports = function(grunt) {
                 options: {
                     reporter: 'spec',
                 },
-                src: ['coverage/test/api/*.js', 'coverage/test/server/*.js']
+                src: ['coverage/test/core/*.js', 'coverage/test/api/*.js']
             },
             'test-current-work': {
                 options: {
                     reporter: 'spec',
                 },
-                src: ['coverage/test/server/runsQueueTest.js']
+                src: ['coverage/test/server/runsDatastoreTest.js']
             },
             coverage: {
                 options: {

bin/server.js

@@ -8,7 +8,7 @@ var bodyParser              = require('body-parser');
 var compress                = require('compression');
 
 app.use(compress());
-app.use(bodyParser.urlencoded({ extended: false }));
+app.use(bodyParser.json());
 
 
 // Initialize the controllers

lib/metadata/policies.json

@@ -365,6 +365,6 @@
         "message": "<p>For each domain met, the browser needs to make a DNS look-up, which is slow. Avoid having to many different domains and the page should render faster.</p><p>By the way, domain sharding is not a good practice anymore.</p>",
         "isOkThreshold": 10,
         "isBadThreshold": 25,
-        "isAbnormalThreshold": 40
+        "isAbnormalThreshold": 50
     }
 }

lib/server/controllers/apiController.js

@@ -1,86 +1,152 @@
-var debug           = require('debug')('ylt:server');
+var debug               = require('debug')('ylt:server');
 
-var runsQueue       = require('../datastores/runsQueue');
-var runsDatastore   = require('../datastores/runsDatastore');
+var YellowLabTools      = require('../../yellowlabtools');
+var RunsQueue           = require('../datastores/runsQueue');
+var RunsDatastore       = require('../datastores/runsDatastore');
+var ResultsDatastore    = require('../datastores/resultsDatastore');
 
 
-function ApiController(app) {
+var ApiController = function(app) {
     'use strict';
 
+    var queue = new RunsQueue();
+    var runsDatastore = new RunsDatastore();
+    var resultsDatastore = new ResultsDatastore();
+
     // Retrieve the list of all runs
-    /*app.get('/runs', function(req, res) {
+    /*app.get('/api/runs', function(req, res) {
         // NOT YET
     });*/
 
     // Create a new run
-    app.post('/runs', function(req, res) {
+    app.post('/api/runs', function(req, res) {
 
-        // Grab the test parameters
+        // Grab the test parameters and generate a random run ID
         var run = {
-            // Generate a random run ID
-            _id: (Date.now()*1000 + Math.round(Math.random()*1000)).toString(36),
+            runId: (Date.now()*1000 + Math.round(Math.random()*1000)).toString(36),
             params: {
                 url: req.body.url,
-                waitForResponse: req.body.waitForResponse || true
+                waitForResponse: req.body.waitForResponse !== false
             }
         };
 
         // Add test to the testQueue
-        debug('Adding test %s to the queue', run._id);
-        var queuing = runsQueue.push(run._id);
+        debug('Adding test %s to the queue', run.runId);
+        var queuePromise = queue.push(run.runId);
+
 
-        
         // Save the run to the datastore
-        var position = runsQueue.getPosition(run._id);
-        run.status = {
-            statusCode: (position === 0) ? STATUS_RUNNING : STATUS_AWAITING,
-            position: position
-        };
-        runsDatastore.add(run);
+        runsDatastore.add(run, queuePromise.startingPosition);
 
 
         // Listening for position updates
-        queuing.progress(function(position) {
-            var savedRun = runsDatastore.get(run._id);
-            savedRun.status = {
-                statusCode: STATUS_AWAITING,
-                position: position
-            };
-            runsDatastore.update(savedRun);
+        queuePromise.progress(function(position) {
+            runsDatastore.updatePosition(run.runId, position);
         });
 
+        // Let's start the run
+        queuePromise.then(function() {
 
-        queuing.then(function() {
-            
-        });
+            runsDatastore.updatePosition(run.runId, 0);
+
+            debug('Launching test %s on %s', run.runId, run.params.url);
+
+            new YellowLabTools(run.params.url)
+                .then(function(data) {
+
+                    debug('Success');
+                    runsDatastore.markAsComplete(run.runId);
+
+                    // Save result in datastore
+                    data.runId = run.runId;
+                    resultsDatastore.saveResult(data);
+
+                    // Send result if the user was waiting
+                    if (run.params.waitForResponse) {
+                        
+                        res.redirect(302, '/api/results/' + run.runId);
+                    }
+
+                }).fail(function(err) {
+                    
+                    console.error('Test failed for %s', run.params.url);
+                    console.error(err);
+                    console.error(err.stack);
 
-        // The user doesn't not want to wait for the response
-        if (!params.waitForResponse) {
+                    runsDatastore.markAsFailed(run.runId);
+                    
+                }).finally(function() {
+                    queue.remove(run.runId);
+                });
 
-            // Sending just the test id
-            res.setHeader('Content-Type', 'application/javascript');
-            res.send(JSON.stringify({
-                testId: testId
-            }));
+        }).fail(function(err) {
+            console.error('Error or YLT\'s core instanciation');
+            console.error(err);
+            console.error(err.stack);
+        });
+
+        // The user doesn't not want to wait for the response, sending the run ID only
+        if (!run.params.waitForResponse) {
+            console.log('Sending response without waiting.');
+            res.setHeader('Content-Type', 'application/json');
+            res.send(JSON.stringify({runId: run.runId}));
         }
     });
 
     // Retrive one run by id
-    app.get('/run/:id', function(req, res) {
+    app.get('/api/runs/:id', function(req, res) {
+        var runId = req.params.id;
+
+        var run = runsDatastore.get(runId);
 
+        if (run) {
+            res.setHeader('Content-Type', 'application/json');
+            res.send(JSON.stringify(run, null, 2));
+        } else {
+            res.status(404).send('Not found');
+        }
     });
 
     // Delete one run by id
-    /*app.delete('/run/:id', function(req, res) {
-        // NOT YET
+    /*app.delete('/api/runs/:id', function(req, res) {
+        deleteRun()
     });*/
 
+    // Delete all
+    /*app.delete('/api/runs', function(req, res) {
+        purgeRuns()
+    });
+
+    // List all
+    app.get('/api/runs', function(req, res) {
+        listRuns()
+    });
 
-    var STATUS_AWAITING     = 'awaiting';
-    var STATUS_RUNNING      = 'running';
-    var STATUS_DONE         = 'done';
-    var STATUS_FAILED       = 'failed';
+    // Exists
+    app.head('/api/runs/:id', function(req, res) {
+        existsX();
+        // Retourne 200 si existe ou 404 si n'existe pas
+    });
+    */
+
+    // Retrive one result by id
+    app.get('/api/results/:id', function(req, res) {
+        var runId = req.params.id;
+
+        resultsDatastore.getResult(runId)
+            .then(function(data) {
+                // This is the pivot format, we might need to clean it first?
+
+                // Hide phantomas results
+                data.toolsResults.phantomas = {};
+                
+                res.setHeader('Content-Type', 'application/json');
+                res.send(JSON.stringify(data, null, 2));
+            }).fail(function() {
+                res.status(404).send('Not found');
+            });
+    });
 
-}
+};
 
 module.exports = ApiController;

lib/server/controllers/uiController.js

@@ -0,0 +1,13 @@
+var UiController = function(app) {
+    'use strict';
+
+
+    // Create a new run
+    app.get('/', function(req, res) {
+
+        res.setHeader('Content-Type', 'text/html');
+        res.send('Test');
+    });
+};
+
+module.exports = UiController;

lib/server/datastores/resultsDatastore.js

@@ -1,9 +1,79 @@
+var fs          = require('fs');
+var rimraf      = require('rimraf');
+var path        = require('path');
+var Q           = require('q');
 
 
 function ResultsDatastore() {
     'use strict';
 
-    
+    var resultFileName = 'results.json';
+    var resultsFolderName = 'results';
+    var resultsDir = path.join(__dirname, '..', '..', '..', resultsFolderName);
+
+
+    this.saveResult = function(testResults) {
+
+        var promise = createResultFolder(testResults.runId);
+
+        promise.then(function() {
+
+            var resultFilePath = path.join(resultsDir, testResults.runId, resultFileName);
+            
+            return Q.nfcall(fs.writeFile, resultFilePath, JSON.stringify(testResults, null, 2));
+        });
+
+        return promise;
+    };
+
+
+    this.getResult = function(runId) {
+
+        var resultFilePath = path.join(resultsDir, runId, resultFileName);
+        
+        return Q.nfcall(fs.readFile, resultFilePath, {encoding: 'utf8'}).then(function(data) {
+            return JSON.parse(data);
+        });
+    };
+
+
+    this.deleteResult = function(runId) {
+        var folder = path.join(resultsDir, runId);
+
+        return Q.nfcall(rimraf, folder);
+    };
+
+
+    // The folder /results/folderName/
+    function createResultFolder(folderName) {
+        var folder = path.join(resultsDir, folderName);
+
+        return createGlobalFolder().then(function() {
+            return Q.nfcall(fs.mkdir, folder);
+        });
+    }
+
+    // The folder /results/
+    function createGlobalFolder() {
+        var deferred = Q.defer();
+
+        // Create the results folder if it doesn't exist
+        fs.exists(resultsDir, function(exists) {
+            if (exists) {
+                deferred.resolve();
+            } else {
+                fs.mkdir(resultsDir, function(err) {
+                    if (err) {
+                        deferred.reject(err);
+                    } else {
+                        deferred.resolve();
+                    }
+                });
+            }
+        });
+
+        return deferred.promise;
+    }
 }
 
 module.exports = ResultsDatastore;

lib/server/datastores/runsDatastore.js

@@ -7,23 +7,68 @@ function RunsDatastore() {
     // For the moment, maybe one day
     var runs = {};
 
+    var STATUS_AWAITING     = 'awaiting';
+    var STATUS_RUNNING      = 'running';
+    var STATUS_COMPLETE     = 'complete';
+    var STATUS_FAILED       = 'failed';
 
-    this.add = function(run) {
-        runs[run._id] = run;
+
+    this.add = function(run, position) {
+        runs[run.runId] = run;
+        this.updatePosition(run.runId, position);
     };
 
+
     this.get = function(runId) {
         return runs[runId];
     };
+
     
-    this.update = function(run) {
-        runs[run._id] = run;
+    this.updatePosition = function(runId, position) {
+        var run = runs[runId];
+        
+        if (position > 0) {
+            run.status = {
+                statusCode: STATUS_AWAITING,
+                position: position
+            };
+        } else {
+            run.status = {
+                statusCode: STATUS_RUNNING
+            };
+        }
+
+        runs[runId] = run;
+    };
+
+
+    this.markAsComplete = function(runId) {
+        var run = runs[runId];
+
+        run.status = {
+            statusCode: STATUS_COMPLETE
+        };
+
+        runs[runId] = run;
+    };
+
+
+    this.markAsFailed = function(runId) {
+        var run = runs[runId];
+
+        run.status = {
+            statusCode: STATUS_FAILED
+        };
+
+        runs[runId] = run;
     };
 
+
     this.delete = function(runId) {
         delete runs[runId];
     };
 
+
     this.list = function() {
         var runsArray = [];
         Object.keys(runs).forEach(function(key) {

lib/server/datastores/runsQueue.js

@@ -9,8 +9,9 @@ function RunsQueue() {
 
     this.push = function(runId) {
         var deferred = Q.defer();
+        var startingPosition = queue.length;
 
-        if (queue.length === 0) {
+        if (startingPosition === 0) {
             
             // The queue is empty, let's run immediatly
             queue.push({
@@ -32,7 +33,9 @@ function RunsQueue() {
             });
         }
 
-        return deferred.promise;
+        var promise = deferred.promise;
+        promise.startingPosition = startingPosition;
+        return promise;
     };
 
 
@@ -68,6 +71,10 @@ function RunsQueue() {
         });
 
     };
+
+    this.length = function() {
+        return queue.length;
+    };
 }
 
 module.exports = RunsQueue;

lib/tools/phantomasWrapper.js

@@ -64,7 +64,7 @@ var PhantomasWrapper = function() {
         }, 600000);
 
         // It's time to launch the test!!!
-        var triesNumber = 3;
+        var triesNumber = 2;
 
         async.retry(triesNumber, function(cb) {
             phantomas(task.url, options, function(err, json, results) {

lib/yellowlabtools.js

@@ -14,24 +14,19 @@ var YellowLabTools = function(url, options) {
 
     } else {
 
-        // Generate a test id
-        var testId = (Date.now()*1000 + Math.round(Math.random()*1000)).toString(36);
-
         if (url.indexOf('http://') !== 0 && url.indexOf('https://') !== 0) {
             url = 'http://' + url;
         }
 
         var params = {
-            testId: testId,
             url: url,
             options: options || {}
         };
 
-        var runner = new Runner(params);
-
-        runner.then(function(data) {
-            deferred.resolve(data);
-        });
+        var runner = new Runner(params)
+            .then(function(data) {
+                deferred.resolve(data);
+            });
     }
 
     return deferred.promise;

package.json

@@ -16,6 +16,7 @@
     "debug": "^2.1.0",
     "express": "~4.10.4",
     "phantomas": "1.7.0",
+    "rimraf": "^2.2.8",
     "socket.io": "~1.2.0"
   },
   "devDependencies": {

test/api/resultsDatastoreTest.js

@@ -0,0 +1,74 @@
+var should = require('chai').should();
+var resultsDatastore = require('../../lib/server/datastores/resultsDatastore');
+
+describe('resultsDatastore', function() {
+    
+    var datastore = new resultsDatastore();
+    
+    var testId1 = '123456789';
+    var testData1 = {
+        runId: testId1,
+        other: {
+            foo: 'foo',
+            bar: 1
+        }
+    };
+
+
+    it('should store a result', function(done) {
+        datastore.should.have.a.property('saveResult').that.is.a('function');
+
+        datastore.saveResult(testData1).then(function() {
+            done();
+        }).fail(function(err) {
+            done(err);
+        });
+    });
+
+    it('should store another result', function(done) {
+        var testData2 = {
+            runId: '987654321',
+            other: {
+                foo: 'foo',
+                bar: 2
+            }
+        };
+
+        datastore.saveResult(testData2).then(function() {
+            done();
+        }).fail(function(err) {
+            done(err);
+        });
+    });
+
+    it('should retrieve a result', function(done) {
+        datastore.getResult(testId1)
+            .then(function(results) {
+
+                // Compare results with testData
+                results.should.deep.equal(testData1);
+
+                done();
+            }).fail(function(err) {
+                done(err);
+            });
+    });
+
+    it('should delete a result', function(done) {
+        datastore.deleteResult(testId1)
+            .then(function() {
+                done();
+            }).fail(function(err) {
+                done(err);
+            });
+    });
+
+    it('should not find the result anymore', function(done) {
+        datastore.getResult(testId1)
+            .then(function(results) {
+                done('Error, the result is still in the datastore');
+            }).fail(function(err) {
+                done();
+            });
+    });
+});

test/api/runsDatastoreTest.js

@@ -0,0 +1,81 @@
+var should = require('chai').should();
+var runsDatastore = require('../../lib/server/datastores/runsDatastore');
+
+describe('runsDatastore', function() {
+    
+    var datastore = new runsDatastore();
+
+    var firstRunId = 333;
+    var secondRunId = 999;
+
+    it('should accept new runs', function() {
+        datastore.should.have.a.property('add').that.is.a('function');
+
+        datastore.add({
+            runId: firstRunId,
+            otherData: 123456789
+        }, 0);
+
+        datastore.add({
+            runId: secondRunId,
+            otherData: 'whatever'
+        }, 1);
+    });
+
+    it('should have stored the runs with a status "runnung"', function() {
+        datastore.should.have.a.property('get').that.is.a('function');
+
+        var firstRun = datastore.get(firstRunId);
+        firstRun.should.have.a.property('runId').that.equals(firstRunId);
+        firstRun.should.have.a.property('status').that.deep.equals({
+            statusCode: 'running'
+        });
+
+        var secondRun = datastore.get(secondRunId);
+        secondRun.should.have.a.property('runId').that.equals(secondRunId);
+        secondRun.should.have.a.property('status').that.deep.equals({
+            statusCode: 'awaiting',
+            position: 1
+        });
+
+    });
+
+    it('should have exactly 2 runs in the store', function() {
+        var runs = datastore.list();
+        runs.should.be.a('array');
+        runs.should.have.length(2);
+        runs[0].should.have.a.property('runId').that.equals(firstRunId);
+    });
+
+    it('shoud update statuses correctly', function() {
+        
+        datastore.markAsComplete(firstRunId);
+        var firstRun = datastore.get(firstRunId);
+        firstRun.should.have.a.property('status').that.deep.equals({
+            statusCode: 'complete'
+        });
+
+        datastore.updatePosition(secondRunId, 0);
+        var secondRun = datastore.get(secondRunId);
+        secondRun.should.have.a.property('status').that.deep.equals({
+            statusCode: 'running'
+        });
+
+        datastore.markAsFailed(secondRunId);
+        secondRun = datastore.get(secondRunId);
+        secondRun.should.have.a.property('status').that.deep.equals({
+            statusCode: 'failed'
+        });
+
+    });
+
+    it('should delete a run', function() {
+        datastore.delete(firstRunId);
+
+        var runs = datastore.list();
+        runs.should.be.a('array');
+        runs.should.have.length(1);
+
+        runs[0].should.have.a.property('runId').that.equals(secondRunId);
+    });
+});

test/server/runsQueueTest.js → test/api/runsQueueTest.js

@@ -4,13 +4,15 @@ var runsQueue = require('../../lib/server/datastores/runsQueue.js');
 describe('runsQueue', function() {
 
     var queue = new runsQueue();
+    var aaaRun = null;
+    var bbbRun = null;
     var cccRun = null;
 
     it('should accept a new runId', function(done) {
         queue.should.have.a.property('push').that.is.a('function');
 
-        var aaaRun = queue.push('aaa');
-        queue.push('bbb');
+        aaaRun = queue.push('aaa');
+        bbbRun = queue.push('bbb');
 
         aaaRun.then(function() {
             done();
@@ -20,9 +22,11 @@ describe('runsQueue', function() {
     it('should return the right positions', function() {
         var aaaPosition = queue.getPosition('aaa');
         aaaPosition.should.equal(0);
+        aaaRun.startingPosition.should.equal(0);
 
         var bbbPosition = queue.getPosition('bbb');
         bbbPosition.should.equal(1);
+        bbbRun.startingPosition.should.equal(1);
 
         var cccPosition = queue.getPosition('ccc');
         cccPosition.should.equal(-1);

test/server/runsDatastoreTest.js

@@ -1,41 +0,0 @@
-var should = require('chai').should();
-var runsDatastore = require('../../lib/server/datastores/runsDatastore');
-
-describe('runsDatastore', function() {
-    
-    var datastore = new runsDatastore();
-
-    var randomId = Math.round(Math.random() * 100000);
-
-    it('should accept a new run', function() {
-        datastore.should.have.a.property('add').that.is.a('function');
-
-        datastore.add({
-            _id: randomId,
-            otherData: 123456789
-        });
-    });
-
-    it('should have stored the run', function() {
-        datastore.should.have.a.property('get').that.is.a('function');
-
-        var run = datastore.get(randomId);
-
-        run.should.have.a.property('_id').that.equals(randomId);
-    });
-
-    it('should have exactly 1 run in the store', function() {
-        var runs = datastore.list();
-        runs.should.be.a('array');
-        runs.should.have.length(1);
-        runs[0].should.have.a.property('_id').that.equals(randomId);
-    });
-
-    it('should delete the run', function() {
-        datastore.delete(randomId);
-
-        var runs = datastore.list();
-        runs.should.be.a('array');
-        runs.should.have.length(0);
-    });
-});