Merge pull request #19 from gmetais/#4

Securize and add limits to API calls

Gruntfile.js

@@ -58,7 +58,8 @@ module.exports = function(grunt) {
             coverage: {
                 files: [
                     {src: ['test/**'], dest: 'coverage/'},
-                    {src: ['lib/metadata/**'], dest: 'coverage/'}
+                    {src: ['lib/metadata/**'], dest: 'coverage/'},
+                    {src: ['bin/**'], dest: 'coverage/'}
                 ]
             }
         },
@@ -70,6 +71,10 @@ module.exports = function(grunt) {
             coverageLib: {
                 src: ['lib/'],
                 dest: 'coverage/lib/'
+            },
+            coverageBin: {
+                src: ['bin/'],
+                dest: 'coverage/bin/'
             }
         },
         mochaTest: {
@@ -103,12 +108,6 @@ module.exports = function(grunt) {
                     showStack: true
                 }
             },
-            testServer: {
-                options: {
-                    port: 8387,
-                    server: './bin/server.js'
-                }
-            },
             testSuite: {
                 options: {
                     port: 8388,
@@ -118,6 +117,37 @@ module.exports = function(grunt) {
         }
     });
 
+
+    // Custom task: copies the test settings.json file to the coverage folder, and checks if there's no missing fields
+    grunt.registerTask('copy-test-server-settings', function() {
+        var mainSettingsFile = './server_config/settings.json';
+        var testSettingsFile = './test/fixtures/settings.json';
+
+        var mainSettings = grunt.file.readJSON(mainSettingsFile);
+        var testSettings = grunt.file.readJSON(testSettingsFile);
+
+        // Recursively compare keys of two objects (not the values)
+        function compareKeys(original, copy, context) {
+            for (var key in original) {
+                if (!copy[key] && copy[key] !== '' && copy[key] !== 0) {
+                    grunt.fail.warn('Settings file ' + testSettingsFile + ' doesn\'t contain key ' + context + '.' + key);
+                }
+                if (original[key] !== null && typeof original[key] === 'object') {
+                    compareKeys(original[key], copy[key], context + '.' + key);
+                }
+            }
+        }
+
+        compareKeys(mainSettings, testSettings, 'settings');
+
+        var outputFile = './coverage/server_config/settings.json';
+        grunt.file.write(outputFile, JSON.stringify(testSettings, null, 4));
+        grunt.log.ok('File ' + outputFile + ' created');
+    });
+
+
+
+
     require('matchdep').filterDev('grunt-*').forEach(grunt.loadNpmTasks);
 
     grunt.registerTask('icons', [
@@ -141,9 +171,9 @@ module.exports = function(grunt) {
     grunt.registerTask('test', [
         'build',
         'jshint',
-        'express:testServer',
         'express:testSuite',
         'clean:coverage',
+        'copy-test-server-settings',
         'blanket',
         'copy:coverage',
         'mochaTest:test',
@@ -153,9 +183,9 @@ module.exports = function(grunt) {
     grunt.registerTask('test-current-work', [
         'build',
         'jshint',
-        'express:testServer',
         'express:testSuite',
         'clean:coverage',
+        'copy-test-server-settings',
         'blanket',
         'copy:coverage',
         'mochaTest:test-current-work'

bin/cli.js

@@ -1,5 +1,3 @@
-#!/usr/bin/env node
-
 var debug = require('debug')('ylt:cli');
 
 var YellowLabTools = require('../lib/yellowlabtools');

bin/server.js

@@ -1,17 +1,17 @@
-// Config file
-var settings                = require('../server_config/settings.json');
-
 var express                 = require('express');
 var app                     = express();
 var server                  = require('http').createServer(app);
 var bodyParser              = require('body-parser');
 var compress                = require('compression');
 
-var authMiddleware          = require('../lib/server/authMiddleware');
+var authMiddleware          = require('../lib/server/middlewares/authMiddleware');
+var apiLimitsMiddleware     = require('../lib/server/middlewares/apiLimitsMiddleware');
+
 
 app.use(compress());
 app.use(bodyParser.json());
 app.use(authMiddleware);
+app.use(apiLimitsMiddleware);
 
 
 // Initialize the controllers
@@ -20,12 +20,14 @@ var uiController            = require('../lib/server/controllers/uiController')(
 
 
 // Let's start the server!
-if (!process.env.GRUNTED) {
-    // The server is not launched by Grunt
-    server.listen(settings.serverPort, function() {
-        console.log('Listening on port %d', server.address().port);
-    });
-}
-
-// For Grunt
-module.exports = app;
+var settings = require('../server_config/settings.json');
+server.listen(settings.serverPort, function() {
+    console.log('Listening on port %d', server.address().port);
+
+    // For the tests
+    if (server.startTests) {
+        server.startTests();
+    }
+});
+
+module.exports = server;

lib/server/authMiddleware.js

@@ -1,52 +0,0 @@
-var config      = require('../../server_config/settings.json');
-
-var jwt         = require('jwt-simple');
-var debug       = require('debug')('authMiddleware');
-
-
-var authMiddleware = function(req, res, next) {
-    'use strict';
-
-    if (req.path.indexOf('/api/') === 0) {
-        
-        // Test if it's an authorized key
-        if (req.headers && req.headers['x-api-key'] && isApiKeyValid(req.headers['x-api-key'])) {
-            next();
-            return;
-        }
-
-        // Test if it's an authorized token
-        if (req.headers && req.headers['x-api-token'] && isTokenValid(req.headers['x-api-token'])) {
-            next();
-            return;
-        }
-        
-        res.status(401).send('Unauthorized');
-    }
-};
-
-
-function isApiKeyValid(apiKey) {
-    return (config.authorizedKeys[apiKey]) ? true : false;
-}
-
-
-function isTokenValid(token) {
-
-    var data = null;
-
-    try {
-        data = jwt.decode(token, config.tokenSalt);
-    } catch(err) {
-        debug('Error while decoding token');
-        debug(err);
-        return false;
-    }
-
-    return data.expire &&
-        data.expire > Date.now() &&
-        data.application &&
-        config.authorizedApplications.indexOf(data.application) >= 0;
-}
-
-module.exports = authMiddleware;

lib/server/middlewares/apiLimitsMiddleware.js

@@ -0,0 +1,85 @@
+var config      = require('../../../server_config/settings.json');
+
+var debug       = require('debug')('apiLimitsMiddleware');
+
+
+var apiLimitsMiddleware = function(req, res, next) {
+    'use strict';
+
+    debug('Entering API Limits Middleware with IP address %s', req.connection.remoteAddress);
+
+    if (req.path.indexOf('/api/') === 0 && !res.locals.hasApiKey) {
+        
+        
+        if (req.path === '/api/runs') {
+            
+            if (!runsTable.accepts(req.connection.remoteAddress)) {
+                // Sorry :/
+                debug('Too many tests launched from IP address %s', req.connection.remoteAddress);
+                res.status(429).send('Too Many Requests');
+                return;
+            }
+
+        }
+
+        if (!callsTable.accepts(req.connection.remoteAddress)) {
+            // Sorry :/
+            debug('Too many API requests from IP address %s', req.connection.remoteAddress);
+            res.status(429).send('Too Many Requests');
+            return;
+        }
+
+        debug('Not blocked by the API limits');
+        // It's ok for the moment
+    }
+
+    next();
+};
+
+
+var RecordTable = function(maxPerDay) {
+    var table = {};
+
+    // Check if the user overpassed the limit and save its visit
+    this.accepts = function(ipAddress) {
+        if (table[ipAddress]) {
+            
+            this.cleanEntry(ipAddress);
+
+            debug('%d visits in the last 24 hours', table[ipAddress].length);
+
+            if (table[ipAddress].length >= maxPerDay) {
+                return false;
+            } else {
+                table[ipAddress].push(Date.now());
+            }
+
+        } else {
+            table[ipAddress] = [];
+            table[ipAddress].push(Date.now());
+        }
+
+        return true;
+    };
+
+    // Clean the table for this guy
+    this.cleanEntry = function(ipAddress) {
+        table[ipAddress] = table[ipAddress].filter(function(date) {
+            return date > Date.now() - 1000*60*60*24;
+        });
+    };
+
+    // Clean the entire table once in a while
+    this.removeOld = function() {
+        for (var ipAddress in table) {
+            this.cleanEntry(ipAddress);
+        }
+    };
+
+};
+
+// Init the records tables
+var runsTable = new RecordTable(config.maxAnonymousRunsPerDay);
+var callsTable = new RecordTable(config.maxAnonymousCallsPerDay);
+
+module.exports = apiLimitsMiddleware;

lib/server/middlewares/authMiddleware.js

@@ -0,0 +1,42 @@
+var config      = require('../../../server_config/settings.json');
+
+var debug       = require('debug')('authMiddleware');
+
+
+var authMiddleware = function(req, res, next) {
+    'use strict';
+
+    if (req.path.indexOf('/api/') === 0) {
+        
+        
+        if (req.headers && req.headers['x-api-key']) {
+
+            // Test if it's an authorized key
+            if (isApiKeyValid(req.headers['x-api-key'])) {
+                
+                // Come in!
+                debug('Authorized key: %s', req.headers['x-api-key']);
+                res.locals.hasApiKey = true;
+            
+            } else {
+                
+                // Sorry :/
+                debug('Unauthorized key %s', req.headers['x-api-key']);
+                res.status(401).send('Unauthorized');
+                return;
+            }
+        } else {
+            debug('No authorization key');
+            // It's ok for the moment but you might be blocked by the apiLimitsMiddleware, dude
+        }
+    }
+
+    next();
+};
+
+
+function isApiKeyValid(apiKey) {
+    return (config.authorizedKeys[apiKey]) ? true : false;
+}
+
+module.exports = authMiddleware;

package.json

@@ -15,7 +15,6 @@
     "compression": "~1.2.1",
     "debug": "^2.1.0",
     "express": "~4.10.4",
-    "jwt-simple": "^0.2.0",
     "phantomas": "1.7.0",
     "rimraf": "^2.2.8",
     "socket.io": "~1.2.0"

server_config/settings.json

@@ -5,8 +5,6 @@
     "authorizedKeys": {
         "1234567890": "contact@gaelmetais.com"
     },
-    "tokenSalt": "lake-city",
-    "authorizedApplications": [
-        "frontend"
-    ]
+    "maxAnonymousRunsPerDay": 50,
+    "maxAnonymousCallsPerDay": 1000
 }

test/api/apiTest.js

@@ -1,13 +1,11 @@
 var should      = require('chai').should();
 var request     = require('request');
-var jwt         = require('jwt-simple');
+var Q           = require('q');
 
 var config = {
     "authorizedKeys": {
         "1234567890": "contact@gaelmetais.com"
-    },
-    "tokenSalt": "lake-city",
-    "authorizedApplications": ["frontend"]
+    }
 };
 
 var apiUrl = 'http://localhost:8387/api';
@@ -16,25 +14,11 @@ var wwwUrl = 'http://localhost:8388';
 describe('api', function() {
 
     var runId;
-    
-    it('should not accept a query if there is no key in headers', function(done) {
-        this.timeout(5000);
+    var apiServer;
 
-        request({
-            method: 'POST',
-            url: apiUrl + '/runs',
-            body: {
-                url: wwwUrl + '/simple-page.html',
-                waitForResponse: false
-            },
-            json: true
-        }, function(error, response, body) {
-            if (!error && response.statusCode === 401) {
-                done();
-            } else {
-                done(error || response.statusCode);
-            }
-        });
+    before(function(done) {
+        apiServer = require('../../bin/server.js');
+        apiServer.startTests = done;
     });
 
     it('should refuse a query with an invalid key', function(done) {
@@ -87,85 +71,68 @@ describe('api', function() {
         });
     });
 
-    it('should refuse an expired token', function(done) {
+    it('should accept up to 10 anonymous runs to the API', function(done) {
         this.timeout(5000);
 
-        request({
-            method: 'POST',
-            url: apiUrl + '/runs',
-            body: {
-                url: wwwUrl + '/simple-page.html',
-                waitForResponse: false
-            },
-            json: true,
-            headers: {
-                'X-Api-Token': jwt.encode({
-                    application: config.authorizedApplications[0],
-                    expire: Date.now() - 60000
-                }, config.tokenSalt)
-            }
-        }, function(error, response, body) {
-            if (!error && response.statusCode === 401) {
+        function launchRun() {
+            var deferred = Q.defer();
+
+            request({
+                method: 'POST',
+                url: apiUrl + '/runs',
+                body: {
+                    url: wwwUrl + '/simple-page.html',
+                    waitForResponse: false
+                },
+                json: true
+            }, function(error, response, body) {
+                if (error) {
+                    deferred.reject(error);
+                } else {
+                    deferred.resolve(response, body);
+                }
+            });
+
+            return deferred.promise;
+        }
+
+        launchRun()
+        .then(launchRun)
+        .then(launchRun)
+        .then(launchRun)
+        .then(launchRun)
+
+        .then(function(response, body) {
+            
+            // Here should still be ok
+            response.statusCode.should.equal(200);
+
+            launchRun()
+            .then(launchRun)
+            .then(launchRun)
+            .then(launchRun)
+            .then(launchRun)
+            .then(launchRun)
+
+            .then(function(response, body) {
+
+                // It should fail now
+                response.statusCode.should.equal(429);
                 done();
-            } else {
-                done(error || response.statusCode);
-            }
-        });
-    });
 
-    it('should refuse a token from an unknown app', function(done) {
-        this.timeout(5000);
+            })
+            .fail(function(error) {
+                done(error);
+            });
 
-        request({
-            method: 'POST',
-            url: apiUrl + '/runs',
-            body: {
-                url: wwwUrl + '/simple-page.html',
-                waitForResponse: false
-            },
-            json: true,
-            headers: {
-                'X-Api-Token': jwt.encode({
-                    application: 'unknown-app',
-                    expire: Date.now() + 60000
-                }, config.tokenSalt)
-            }
-        }, function(error, response, body) {
-            if (!error && response.statusCode === 401) {
-                done();
-            } else {
-                done(error || response.statusCode);
-            }
+        }).fail(function(error) {
+            done(error);
         });
+        
     });
 
-    it('should accept a good token', function(done) {
-        this.timeout(5000);
-
-        request({
-            method: 'POST',
-            url: apiUrl + '/runs',
-            body: {
-                url: wwwUrl + '/simple-page.html',
-                waitForResponse: false
-            },
-            json: true,
-            headers: {
-                'X-Api-Token': jwt.encode({
-                    application: config.authorizedApplications[0],
-                    expire: Date.now() + 60000
-                }, config.tokenSalt)
-            }
-        }, function(error, response, body) {
-            if (!error && response.statusCode === 200) {
-
-                runId = body.runId;
-                runId.should.be.a('string');
-                done();
-
-            } else {
-                done(error || response.statusCode);
-            }
-        });
+    after(function() {
+        console.log('Closing the server');
+        apiServer.close();
     });
 });

test/fixtures/settings.json

@@ -0,0 +1,10 @@
+{
+    "serverPort": "8387",
+    "googleAnalyticsId": "",
+    
+    "authorizedKeys": {
+        "1234567890": "contact@gaelmetais.com"
+    },
+    "maxAnonymousRunsPerDay": 10,
+    "maxAnonymousCallsPerDay": 1000
+}