CLI is working, API not ready, Front broken

.travis.yml

@@ -5,4 +5,4 @@ before_install:
     - "npm install -g grunt-cli"
     - "npm install -g phantomjs"
 install: npm install
-before_script: grunt build
+before_script: grunt test

Gruntfile.js

@@ -31,7 +31,9 @@ module.exports = function(grunt) {
         jshint: {
             all: [
                 '*.js',
-                'app/lib/*',
+                'app/lib/*.js',
+                'bin/*.js',
+                'lib/**/*.js',
                 'app/nodeControllers/*.js',
                 'app/public/scripts/*.js',
                 'phantomas_custom/**/*.js'
@@ -52,9 +54,13 @@ module.exports = function(grunt) {
             }
         },
         blanket: {
-            coverage: {
+            coverageApp: {
                 src: ['app/'],
                 dest: 'coverage/app/'
+            },
+            coverageLib: {
+                src: ['lib/'],
+                dest: 'coverage/lib/'
             }
         },
         mochaTest: {
@@ -62,18 +68,29 @@ module.exports = function(grunt) {
                 options: {
                     reporter: 'spec',
                 },
-                src: ['coverage/test/server/*.js']
+                src: ['coverage/test/api/*.js']
+            },
+            'test-current-work': {
+                options: {
+                    reporter: 'spec',
+                },
+                src: ['coverage/test/api/rulesCheckerTest.js']
             },
             coverage: {
                 options: {
                     reporter: 'html-cov',
-                    // use the quiet flag to suppress the mocha console output
                     quiet: true,
-                    // specify a destination file to capture the mocha
-                    // output (the quiet option does not suppress this)
                     captureFile: 'coverage/coverage.html'
                 },
-                src: ['coverage/test/server/*.js']
+                src: ['coverage/test/api/*.js']
+            }
+        },
+        express: {
+            test: {
+                options: {
+                    port: 8388,
+                    bases: 'test/www'
+                }
             }
         }
     });
@@ -87,7 +104,6 @@ module.exports = function(grunt) {
     ]);
 
     grunt.registerTask('build', [
-        'jshint',
         'less'
     ]);
 
@@ -96,10 +112,24 @@ module.exports = function(grunt) {
     ]);
 
     grunt.registerTask('test', [
+        'build',
+        'jshint',
+        'express:test',
+        'clean:coverage',
+        'blanket',
+        'copy:coverage',
+        'mochaTest:test',
+        'mochaTest:coverage'
+    ]);
+
+    grunt.registerTask('test-current-work', [
+        'build',
+        'jshint',
+        'express:test',
         'clean:coverage',
         'blanket',
         'copy:coverage',
-        'mochaTest'
+        'mochaTest:test-current-work'
     ]);
 
 };

app/node_controllers/launchTestController.js

@@ -9,16 +9,10 @@ var strReplace      = require('../lib/strReplace');
 var launchTestController = function(req, res, testQueue, googleAnalyticsId) {
     'use strict';
 
-    // Generate test id
-    var testId = (Date.now()*1000 + Math.round(Math.random()*1000)).toString(36);
-
     var resultsPath = 'results/' + testId;
     var phantomasResultsPath = resultsPath + '/results.json';
     
     var url = req.body.url;
-    if (url.indexOf('http://') !== 0 && url.indexOf('https://') !== 0) {
-        url = 'http://' + url;
-    }
 
     var options = {};
     if (req.body.timeout) {

bin/cli.js

@@ -1,2 +1,35 @@
-'use strict';
+#!/usr/bin/env node
 
+//var color = require('colors');
+
+var YellowLabTools = require('../lib/yellowlabtools');
+
+// Check parameters
+if (process.argv.length !== 3) {
+    console.error('Incorrect parameters');
+    console.error('\nUsage: ylt <pageUrl>\n');
+    process.exit(1);
+}
+
+var url = process.argv[2];
+
+(function execute(url) {
+    'use strict';
+
+    var ylt = new YellowLabTools(url);
+    console.log('Test launched...');
+
+    ylt.
+        then(function(data) {
+
+            console.log('Success');
+            console.log(JSON.stringify(data.rules, null, 2));
+
+        }).fail(function(err) {
+            
+            console.error('Test failed for %s', url);
+            console.error(err);
+            
+        });
+
+})(url);

html/1416302062861.html

@@ -1 +0,0 @@
-<html><head></head><body><img src="http://karavel.112.2o7.net/b/ss/karavelpromovac2prod/1/H.25.2/s98901238560210?AQB=1&amp;ndh=1&amp;t=18%2F10%2F2014%2010%3A14%3A22%202%20-60&amp;ce=UTF-8&amp;ns=karavel&amp;pageName=Homepage%7CParis&amp;g=http%3A%2F%2Fwww.promovacances.com%2F&amp;cc=EUR&amp;ch=General&amp;server=www.promovacances.com&amp;events=event34&amp;c24=Homepage%7CParis&amp;c25=Homepage&amp;c26=Homepage%7CParis&amp;c27=Homepage%7CParis&amp;c28=Homepage&amp;c33=10%3A00AM&amp;v33=10%3A00AM-Tuesday&amp;c34=Tuesday&amp;c35=Weekday&amp;v35=General&amp;c39=Package&amp;v39=Package&amp;v41=Test-PersonnalisationHomepage-VersionB-Profil0&amp;c49=%2F&amp;j=1.6&amp;k=Y&amp;AQE=1" width="1" height="1" border="0" alt=""></body></html>

lib/metadata/policies.json

@@ -0,0 +1,370 @@
+{
+    "DOMelementsCount": {
+        "tool": "phantomas",
+        "label": "DOM elements count",
+        "message": "<p>A high number of DOM elements means a lot of work for the browser to render the page.</p><p>It also slows down Javascript DOM queries, as there are more elements to search through.</p>",
+        "isOkThreshold": 1000,
+        "isBadThreshold": 3000,
+        "isAbnormalThreshold": 5000
+    },
+    "DOMelementMaxDepth": {
+        "tool": "phantomas",
+        "label": "DOM max depth",
+        "message": "<p>A deep DOM makes the CSS matching with DOM elements difficult.</p><p>It also slows down Javascript modifications to the DOM because changing the dimensions of an element makes the browser re-calculate the dimensions of it's parents. Same thing for Javascript events, that bubble up to the document root.</p>",
+        "isOkThreshold": 10,
+        "isBadThreshold": 20,
+        "isAbnormalThreshold": 30
+    },
+    "iframesCount": {
+        "tool": "phantomas",
+        "label": "Number of iframes",
+        "message": "<p>iFrames are the most complex HTML elements. They are pages, just like the main page, and the browser needs to create a new page context, which has a cost.</p>",
+        "isOkThreshold": 2,
+        "isBadThreshold": 15,
+        "isAbnormalThreshold": 30
+    },
+    "DOMidDuplicated": {
+        "tool": "phantomas",
+        "label": "IDs duplicated",
+        "message": "<p>IDs of HTML elements must be document-wide unique. This can cause problems with getElementById returning the wrong element.</p>",
+        "isOkThreshold": 0,
+        "isBadThreshold": 5,
+        "isAbnormalThreshold": 10
+    },
+    "DOMinserts": {
+        "tool": "phantomas",
+        "label": "DOM inserts",
+        "message": "<p>Working with the DOM in Javascript triggers layout calculations and slows down the page.</p><p>Try, as much as possible, to have an HTML page fully generated by the server instead of making changes with JS.</p>",
+        "isOkThreshold": 10,
+        "isBadThreshold": 400,
+        "isAbnormalThreshold": 1000
+    },
+    "DOMqueries": {
+        "tool": "phantomas",
+        "label": "DOM queries",
+        "message": "<p>DOM queries are like looking in a large catalog of items. Even if the browsers made progress on the performances of queries, websites often make hundreds of them.</p><p>Try to reduce the number of queries by refactoring your Javascript code.</p><p>Avoid also to have a read query between two write queries. To be able to reduce the number repaints and optimize performances, browsers buffer the DOM writing operations and treat them in bulk. But each time a DOM reading is asked, the browser needs to empty the buffer. This can be particularly slow inside a loop.</p>",
+        "isOkThreshold": 50,
+        "isBadThreshold": 1000,
+        "isAbnormalThreshold": 2000
+    },
+    "DOMqueriesAvoidable": {
+        "tool": "phantomas",
+        "label": "Duplicated DOM queries",
+        "message": "<p>This is the number of queries that could be avoided by removing all duplicated queries.</p><p>Simply save the result of a query in a variable. Ok it is not always simple, especially with third-party scripts, but at least do it with your own code.</p>",
+        "isOkThreshold": 0,
+        "isBadThreshold": 200,
+        "isAbnormalThreshold": 500
+    },
+    "eventsBound": {
+        "tool": "phantomas",
+        "label": "Events bound",
+        "message": "<p>Binding too many events has a cost.</p><p>It can be avoided by using \"event delegation\". Instead of binding events on each element one by one, events delegation binds them on the top level document element and uses the bubbling principle. It will imperceptibly slow down the event when it occurs, but the loading of the page will speed-up.</p>",
+        "isOkThreshold": 100,
+        "isBadThreshold": 1500,
+        "isAbnormalThreshold": 2000
+    },
+    "jsErrors": {
+        "tool": "phantomas",
+        "label": "Javascript errors",
+        "message": "<p>Just to let you know there are some errors on the page.</p><p><b>Please note that some errors only occur in the PhantomJS browser, so you might need to double check on other browsers.</b></p>",
+        "isOkThreshold": 0,
+        "isBadThreshold": 1,
+        "isAbnormalThreshold": 1
+    },
+    "evalCalls": {
+        "tool": "phantomas",
+        "label": "eval calls",
+        "message": "<p>The 'eval' function is slow and is a bad coding practice. Try to get rid of it.</p>",
+        "isOkThreshold": 0,
+        "isBadThreshold": 10,
+        "isAbnormalThreshold": 20
+    },
+    "documentWriteCalls": {
+        "tool": "phantomas",
+        "label": "document.write calls",
+        "message": "<p>They slow down the page construction, especially if they are used to insert scripts in the page. Remove them ASAP.</p><p>If you cannot remove them because they come from a third-party script (such as ads), have a look at <a href=\"https://github.com/krux/postscribe\" target=\"_blank\">PostScribe</a>.</p>",
+        "isOkThreshold": 0,
+        "isBadThreshold": 10,
+        "isAbnormalThreshold": 20
+    },
+    "consoleMessages": {
+        "tool": "phantomas",
+        "label": "Console messages",
+        "message": "<p>Try to keep your console clean when in production. Debugging is good for development only.</p><p>Writing in the console has a cost, especially when dumping large object variables.</p><p>There is also a problem with Internet Explorer 8, not knowing the console object.</p>",
+        "isOkThreshold": 0,
+        "isBadThreshold": 10,
+        "isAbnormalThreshold": 25
+    },
+    "globalVariables": {
+        "tool": "phantomas",
+        "label": "Global variables",
+        "message": "<p>It is a bad practice because they clutter up the global namespace. If two scripts use the same variable name in the global scope, it can cause conflicts and it is generally hard to debug.</p><p>Global variables also take a (very) little bit longer to be accessed than variables in the local scope of a function.</p>",
+        "isOkThreshold": 10,
+        "isBadThreshold": 50,
+        "isAbnormalThreshold": 200
+    },
+    "inBodyDomManipulations": {
+        "tool": "ylt",
+        "label": "DOM manipulations in body",
+        "message": "<p>This metric counts the number of DOM queries, DOM inserts, binds, etc. made by the Javascript before the DOMContentLoaded event.</p><p>Wait for this event before manipulating the DOM. Do not execute Javascript in the middle of the BODY as it slows down the construction of the DOM and makes a poor maintainability. This is what i call spaghetti code.</p><p>The JS Timeline tab can help you identify what's happening.</p>",
+        "isOkThreshold": 10,
+        "isBadThreshold": 50,
+        "isAbnormalThreshold": 100
+    },
+    "jQueryVersion": {
+        "tool": "ylt",
+        "label": "jQuery version",
+        "message": "<p>Current latest versions of jQuery are 1.11 (with support for old IE versions) and 2.1 (without).</p><p>Each new version of jQuery optimizes performances. Do not keep an old version of jQuery. Updating can sometimes break a few things, but it is generally quite easy to fix them up. So don't hesitate.</p>",
+        "isOkThreshold": 1,
+        "isBadThreshold": 2,
+        "isAbnormalThreshold": 3
+    },
+    "jQueryDifferentVersions": {
+        "tool": "ylt",
+        "label": "Several versions loaded",
+        "message": "<p>jQuery is a heavy library. You should <b>never<b> load jQuery more than one on the same page.</p>",
+        "isOkThreshold": 1,
+        "isBadThreshold": 2,
+        "isAbnormalThreshold": 3
+    },
+    "cssRules": {
+        "tool": "phantomas",
+        "label": "Rules count",
+        "message": "<p>Having a huge number of CSS rules hurts performances. If the number of CSS rules is higher than the number of DOM elements, there is clearly a problem.</p><p>Huge stylesheets generally occur when the different pages of a website load all the CSS, concatenated in a single stylesheet, even if a large part of the rules are page-specific. Solution is to create one main CSS file with global rules and one custom files per page.</p>",
+        "isOkThreshold": 500,
+        "isBadThreshold": 2500,
+        "isAbnormalThreshold": 4000
+    },
+    "cssComplexSelectors": {
+        "tool": "phantomas",
+        "label": "Complex selectors",
+        "message": "<p>Complex selectors are CSS selectors with 4 or more expressions, like \"#header ul li .foo\".</p><p>They are adding more work for the browser, and this could be avoided by simplifying selectors.</p>",
+        "isOkThreshold": 0,
+        "isBadThreshold": 500,
+        "isAbnormalThreshold": 2000
+    },
+    "cssComplexSelectorsByAttribute": {
+        "tool": "phantomas",
+        "label": "Complex attributes selector",
+        "message": "<p>Complex attributes selectors are one of these:<ul><li>.foo[type*=bar] (contains bar)</li><li>.foo[type^=bar] (starts with bar)</li><li>.foo[type|=bar] (starts with bar or bar-)</li><li>.foo[type$=bar] (ends with bar)</li><li>.foo[type~=bar baz] (bar or baz)</li></ul></p><p>Their matching process needs more CPU and it has a cost on performances.</p>",
+        "isOkThreshold": 0,
+        "isBadThreshold": 50,
+        "isAbnormalThreshold": 100
+    },
+    "cssParsingErrors": {
+        "tool": "phantomas",
+        "label": "CSS syntax error",
+        "message": "<p>Yellow Lab Tools failed to parse a CSS file. I doubt the problem comes from the css parser.</p><p>Maybe a <a href=\"http://jigsaw.w3.org/css-validator\" target=\"_blank\">CSS validator</a> can help you.</p>",
+        "isOkThreshold": 1,
+        "isBadThreshold": 2,
+        "isAbnormalThreshold": 3
+    },
+    "cssImports": {
+        "tool": "phantomas",
+        "label": "Uses of @import",
+        "message": "<p>It’s bad for performance to use @import because CSS files don't get downloaded in parallel.</p><p>You should use &lt;link rel='stylesheet' href='a.css'&gt; instead.</p>",
+        "isOkThreshold": 0,
+        "isBadThreshold": 1,
+        "isAbnormalThreshold": 1
+    },
+    "cssDuplicatedSelectors": {
+        "tool": "phantomas",
+        "label": "Duplicated selectors",
+        "message": "<p>This is when two or more selectors are strictly identical and should be merged.</p>",
+        "isOkThreshold": 0,
+        "isBadThreshold": 40,
+        "isAbnormalThreshold": 80
+    },
+    "cssDuplicatedProperties": {
+        "tool": "phantomas",
+        "label": "Duplicated properties",
+        "message": "<p>This is the number of property definitions duplicated within a selector.</p>",
+        "isOkThreshold": 0,
+        "isBadThreshold": 50,
+        "isAbnormalThreshold": 100
+    },
+    "cssEmptyRules": {
+        "tool": "phantomas",
+        "label": "Empty rules",
+        "message": "<p>Very easy to fix: remove all empty rules.</p>",
+        "isOkThreshold": 0,
+        "isBadThreshold": 40,
+        "isAbnormalThreshold": 100
+    },
+    "cssExpressions": {
+        "tool": "phantomas",
+        "label": "CSS expressions",
+        "message": "<p>Such as: expression( document.body.clientWidth > 600 ? \"600px\" : \"auto\" )</p><p>This is a bad practice as it slows down browsers. There are some simpler CSS3 methods for doing this.</p>",
+        "isOkThreshold": 0,
+        "isBadThreshold": 1,
+        "isAbnormalThreshold": 20
+    },
+    "cssImportants": {
+        "tool": "phantomas",
+        "label": "Uses of !important",
+        "message": "<p>It can be useful, but only as a last resort. It is a bad practice because it overrides the normal cascading logic. The more you use !important, the more you need it again to over-override. This conducts to a poor maintainability.</p>",
+        "isOkThreshold": 0,
+        "isBadThreshold": 50,
+        "isAbnormalThreshold": 150
+    },
+    "cssOldIEFixes": {
+        "tool": "phantomas",
+        "label": "Old IE fixes",
+        "message": "<p>What browser do you need to support? Once you've got the answer, take a look at these old rules that pollute your CSS code and remove them.</p><p>IE6:<ul><li>* html</li><li>html > body (everything but IE6)</li></ul><p><p>IE7:<ul><li><b>*</b>height: 123px;</li><li>height: 123px <b>!ie</b>;</li></ul><p><p>IE9:<ul><li>-ms-filter</li><li>progid:DXImageTransform.Microsoft</li></ul></p>",
+        "isOkThreshold": 0,
+        "isBadThreshold": 50,
+        "isAbnormalThreshold": 300
+    },
+    "cssOldPropertyPrefixes": {
+        "tool": "phantomas",
+        "label": "Old prefixes",
+        "message": "<p>Many property prefixes such as -moz- or -webkit- are not needed anymore, or by very few people. You can remove them or replace them with the non-prefixed version. This will help reducing your stylesheets weight.</p>",
+        "isOkThreshold": 0,
+        "isBadThreshold": 50,
+        "isAbnormalThreshold": 300
+    },
+    "cssUniversalSelectors": {
+        "tool": "phantomas",
+        "label": "Universal selectors",
+        "message": "<p>Universal selectors are the most expensive CSS selectors.</p><p>More informations <a href=\"http://perfectionkills.com/profiling-css-for-fun-and-profit-optimization-notes/\" target=\"_blank\">here</a>.</p>",
+        "isOkThreshold": 0,
+        "isBadThreshold": 40,
+        "isAbnormalThreshold": 80
+    },
+    "cssRedundantBodySelectors": {
+        "tool": "phantomas",
+        "label": "Redundant body selectors",
+        "message": "<p>This is one way to remove complexity from a CSS rule. Generally, when \"body\" is specified in a rule it can be removed, because an element is necessarily inside the body.</p>",
+        "isOkThreshold": 0,
+        "isBadThreshold": 50,
+        "isAbnormalThreshold": 150
+    },
+    "cssRedundantChildNodesSelectors": {
+        "tool": "phantomas",
+        "label": "Redundant tags selectors",
+        "message": "<p>Some tags included inside other tags are obvious. For example, when \"ul li\" is specified in a rule, \"ul\" can be removed because the \"li\" element is <b>always</b> inside a \"ul\". Same thing for \"tr td\", \"select option\", ...</p><p>Lowering compexity in CSS selectors can make the page load a little faster.</p>",
+        "isOkThreshold": 0,
+        "isBadThreshold": 50,
+        "isAbnormalThreshold": 150
+    },
+    "requests": {
+        "tool": "phantomas",
+        "label": "Total requests number",
+        "message": "<p>This is one of the most important performance rule. Every request is slowing down the page loading.</p><p>There are several technics to reduce their number:<ul><li>Concatenate JS files</li><li>Concatenate CSS files</li><li>Embed or inline small JS or CSS files in the HTML</li><li>Create sprites or icon fonts</li><li>Base64 encode small images in HTML or stylesheets</li><li>Use lazyloading for images</li></ul></p>",
+        "isOkThreshold": 15,
+        "isBadThreshold": 100,
+        "isAbnormalThreshold": 200
+    },
+    "htmlCount": {
+        "tool": "phantomas",
+        "label": "Document count",
+        "message": "<p>The number of HTML pages requests, HTML fragments or iframes.</p>",
+        "isOkThreshold": 10,
+        "isBadThreshold": 20,
+        "isAbnormalThreshold": 30
+    },
+    "jsCount": {
+        "tool": "phantomas",
+        "label": "Script count",
+        "message": "<p>Reduce the number of scripts by concatenating them.</p>",
+        "isOkThreshold": 5,
+        "isBadThreshold": 15,
+        "isAbnormalThreshold": 30
+    },
+    "cssCount": {
+        "tool": "phantomas",
+        "label": "CSS count",
+        "message": "<p>Reduce the number of stylesheets by concatenating them.</p>",
+        "isOkThreshold": 3,
+        "isBadThreshold": 10,
+        "isAbnormalThreshold": 22
+    },
+    "imageCount": {
+        "tool": "phantomas",
+        "label": "Image count",
+        "message": "<p>Reduce the number of images by lazyloading them, by spriting them or by creating an icons font.</p>",
+        "isOkThreshold": 15,
+        "isBadThreshold": 40,
+        "isAbnormalThreshold": 70
+    },
+    "webfontCount": {
+        "tool": "phantomas",
+        "label": "Font count",
+        "message": "<p>Fonts are loaded on the critical path of the head. Load as many as possible.</p>",
+        "isOkThreshold": 0,
+        "isBadThreshold": 3,
+        "isAbnormalThreshold": 5
+    },
+    "videoCount": {
+        "tool": "phantomas",
+        "label": "Videos count",
+        "message": "<p>The number of videos loaded.</p>",
+        "isOkThreshold": 1,
+        "isBadThreshold": 5,
+        "isAbnormalThreshold": 15
+    },
+    "jsonCount": {
+        "tool": "phantomas",
+        "label": "JSON count",
+        "message": "<p>The number of AJAX requests to JSON files or webservices.</p>",
+        "isOkThreshold": 2,
+        "isBadThreshold": 10,
+        "isAbnormalThreshold": 25
+    },
+    "otherCount": {
+        "tool": "phantomas",
+        "label": "Other type of requests",
+        "message": "<p>They can be Flash, XML, music or any unknown format.</p>",
+        "isOkThreshold": 5,
+        "isBadThreshold": 20,
+        "isAbnormalThreshold": 40
+    },
+    "notFound": {
+        "tool": "phantomas",
+        "label": "404 not found",
+        "message": "<p>404 errors are never cached, so each time a page ask for it, it hits se server. Even if it is behind a CDN or a reverse-proxy cache.</p>",
+        "isOkThreshold": 0,
+        "isBadThreshold": 1,
+        "isAbnormalThreshold": 1
+    },
+    "closedConnections": {
+        "tool": "phantomas",
+        "label": "Connections closed",
+        "message": "<p>This counts the number of requests not keeping the connection alive (specifying \"Connection: close\" in the response headers). It is only counting a request if it is followed by another request on the same domain.</p><p>This is slowing down the next request, because the brower needs to open a new connection to the server, which means a additional round-trip.</p><p>Correct the problem by setting a Keep-Alive header on the guilty server.</p>",
+        "isOkThreshold": 0,
+        "isBadThreshold": 8,
+        "isAbnormalThreshold": 20
+    },
+    "multipleRequests": {
+        "tool": "phantomas",
+        "label": "Duplicated requests",
+        "message": "<p>This only happens when the asset has no cache and is requested more than once on the same page. Be very careful about it.</p>",
+        "isOkThreshold": 0,
+        "isBadThreshold": 5,
+        "isAbnormalThreshold": 10
+    },
+    "cachingDisabled": {
+        "tool": "phantomas",
+        "label": "Caching disabled",
+        "message": "<p>Counts responses with caching disabled (max-age=0)</p><p>Fix immediatly if on static assets.</p>",
+        "isOkThreshold": 0,
+        "isBadThreshold": 12,
+        "isAbnormalThreshold": 25
+    },
+    "cachingTooShort": {
+        "tool": "phantomas",
+        "label": "Caching too short",
+        "message": "<p>Responses with too short caching time (less than a week).</p><p>The longer you cache, the better. Add versionning to your static assets, if it's not already done, and set their cache time to one year.</p>",
+        "isOkThreshold": 5,
+        "isBadThreshold": 20,
+        "isAbnormalThreshold": 40
+    },
+    "domains": {
+        "tool": "phantomas",
+        "label": "Different domains",
+        "message": "<p>For each domain met, the browser needs to make a DNS look-up, which is slow. Avoid having to many different domains and the page should render faster.</p><p>By the way, domain sharding is not a good practice anymore.</p>",
+        "isOkThreshold": 10,
+        "isBadThreshold": 25,
+        "isAbnormalThreshold": 40
+    }
+}

lib/metadata/scoreProfileGeneric.json

@@ -0,0 +1,3 @@
+{
+    
+}

lib/rulesChecker.js

@@ -0,0 +1,52 @@
+var Q = require('q');
+
+var RulesChecker = function() {
+    'use strict';
+
+    this.check = function(data, policies) {
+        var deferred = Q.defer();
+
+        var results = {};
+        var err = null;
+
+        for (var metricName in policies) {
+            var policy = policies[metricName];
+
+            if (data.toolsResults[policy.tool] &&
+                data.toolsResults[policy.tool].metrics &&
+                (data.toolsResults[policy.tool].metrics[metricName] || data.toolsResults[policy.tool].metrics[metricName] === 0)) {
+                    
+                    var rule = {
+                        value: data.toolsResults[policy.tool].metrics[metricName],
+                        policy: policy
+                    };
+
+                    if (data.toolsResults[policy.tool].offenders &&
+                        data.toolsResults[policy.tool].offenders[metricName] &&
+                        data.toolsResults[policy.tool].offenders[metricName].length > 0) {
+                            //rule.offenders = data.toolsResults[policy.tool].offenders[metricName];
+                    }
+
+                    rule.bad = rule.value > policy.isOkThreshold;
+                    rule.abnormal = policy.isAbnormalThreshold && rule.value >= policy.isAbnormalThreshold;
+
+                    // A value between 0 (bad) and 100 (very good).
+                    var score = (policy.isBadThreshold - rule.value) * 100 / (policy.isBadThreshold - policy.isOkThreshold);
+                    rule.score = Math.min(Math.max(Math.round(score), 0), 100);
+
+                    // A value between 0 (abnormal) and negative-infinity (your website is a blackhole)
+                    var abnormalityScore = (policy.isAbnormalThreshold - rule.value) * 100 / (policy.isAbnormalThreshold - policy.isOkThreshold);
+                    rule.abnormalityScore = Math.min(Math.round(abnormalityScore), 0);
+
+                    results[metricName] = rule;
+            }
+        }
+
+        data.rules = results;
+        deferred.resolve(data);
+
+        return deferred.promise;
+    };
+};
+
+module.exports = new RulesChecker();

lib/runner.js

@@ -0,0 +1,38 @@
+var Q = require('q');
+
+var phantomasWrapper = require('./tools/phantomasWrapper');
+var rulesChecker = require('./rulesChecker');
+
+var Runner = function(params) {
+    'use strict';
+
+    // The pivot format
+    var data = {
+        params: params,
+        toolsResults: {}
+    };
+
+    // Execute Phantomas first
+    var run = phantomasWrapper.execute(data);
+
+    // Other tools go here
+
+
+    // Read each policy and save the results
+    run.then(function() {
+        var policies = require('./metadata/policies.json');
+        return rulesChecker.check(data, policies);
+    });
+
+    // TODO : error handler
+    /*run.catch(function(err) {
+        console.log('The run failed');
+        console.log(err);
+    });*/
+
+    run.done(data);
+
+    return run;
+};
+
+module.exports = Runner;

app/lib/phantomasWrapper.js → lib/tools/phantomasWrapper.js

@@ -1,8 +1,5 @@
-/**
- * Yellow Lab Tools main file
- */
-
 var async           = require('async');
+var Q               = require('q');
 var phantomas       = require('phantomas');
 
 var PhantomasWrapper = function() {
@@ -13,10 +10,14 @@ var PhantomasWrapper = function() {
      * Available options :
      *
      * - timeout : in seconds (default 60)
-     * - jsDeepAnalysis : should we inspect subrequests in the javascript execution tree (reported durations of main tasks will be slower than usual)
+     * - jsDeepAnalysis : should we inspect subrequests in the javascript execution tree?
      *
      */
-     this.execute = function(task, callback) {
+    this.execute = function(data) {
+
+        var deferred = Q.defer();
+
+        var task = data.params;
 
         var options = {
             // Cusomizable options
@@ -55,8 +56,8 @@ var PhantomasWrapper = function() {
 
         // Kill the application if nothing happens for 10 minutes
         var killer = setTimeout(function() {
-            console.log('Killing the server because the test ' + task.testId + ' on ' + task.url + ' was launched 10 minutes ago');
-            // Forever will restart the server
+            console.log('Killing the app because the test on ' + task.url + ' was launched 10 minutes ago');
+            // If in server mode, forever will restart the server
             process.exit(1);
         }, 600000);
 
@@ -83,21 +84,25 @@ var PhantomasWrapper = function() {
                 }
 
                 if (err) {
-                    console.log('Attempt failed for test id ' + task.testId + '. Error code ' + err);
+                    console.log('Attempt failed. Error code ' + err);
                 }
 
-                cb(err, {json: json, results: results});
+                cb(err, json);
             });
-        }, function(err, data) {
+        }, function(err, json) {
 
             clearTimeout(killer);
 
             if (err) {
-                console.log('All ' + triesNumber + ' attemps failed for test id ' + task.testId);
+                console.log('All ' + triesNumber + ' attemps failed for the test');
+                deferred.reject(err);
+            } else {
+                data.toolsResults.phantomas = json;
+                deferred.resolve(data);
             }
-            callback(err, data.json, data.results);
         });
 
+        return deferred.promise;
     };
 };
 

lib/yellowlabtools.js

@@ -1,10 +1,41 @@
+var Q = require('q');
 
+var Runner = require('./runner');
 
 
-var YellowLabTools = function() {
+var YellowLabTools = function(url, options) {
     'use strict';
 
-    
+    var deferred = Q.defer();
+
+    if (!url) {
+
+        deferred.reject('URL missing');
+
+    } else {
+
+        // Generate a test id
+        var testId = (Date.now()*1000 + Math.round(Math.random()*1000)).toString(36);
+
+        if (url.indexOf('http://') !== 0 && url.indexOf('https://') !== 0) {
+            url = 'http://' + url;
+        }
+
+        var params = {
+            testId: testId,
+            url: url,
+            options: options || {}
+        };
+
+        var runner = new Runner(params);
+
+        runner.then(function(data) {
+            console.log(data);
+            deferred.resolve(data);
+        });
+    }
+
+    return deferred.promise;
 };
 
 module.exports = YellowLabTools;

package.json

@@ -5,6 +5,10 @@
     "type": "git",
     "url": "git://github.com/gmetais/YellowLabTools.git"
   },
+  "bin": {
+    "ylt": "./bin/cli.js"
+  },
+  "main": "./lib/yellowlabtools.js",
   "dependencies": {
     "phantomas": "1.7.0",
     "express": "~4.10.1",
@@ -14,18 +18,20 @@
     "compression": "~1.2.0"
   },
   "devDependencies": {
+    "chai": "^1.9.2",
     "grunt": "^0.4.5",
-    "grunt-contrib-jshint": "^0.10.0",
-    "matchdep": "^0.3.0",
-    "grunt-mocha-test": "^0.12.2",
+    "grunt-blanket": "^0.0.8",
     "grunt-contrib-clean": "^0.6.0",
     "grunt-contrib-copy": "^0.7.0",
-    "grunt-blanket": "^0.0.8",
-    "chai": "^1.9.2",
+    "grunt-contrib-jshint": "^0.10.0",
+    "grunt-contrib-less": "^0.12.0",
+    "grunt-express": "^1.4.1",
+    "grunt-fontsmith": "^0.9.1",
+    "grunt-mocha-test": "^0.12.2",
+    "matchdep": "^0.3.0",
     "mocha": "^2.0.1",
     "phantomjs": "^1.9.10",
-    "grunt-fontsmith": "^0.9.1",
-    "grunt-contrib-less": "^0.12.0"
+    "q": "^1.1.2"
   },
   "scripts": {
     "test": "grunt test"

test/api/phantomasWrapperTest.js

@@ -0,0 +1,78 @@
+var should = require('chai').should();
+var phantomasWrapper = require('../../lib/tools/phantomasWrapper');
+
+describe('phantomasWrapper', function() {
+    
+    it('should have a method execute', function() {
+        phantomasWrapper.should.have.property('execute').that.is.a('function');
+    });
+    
+    it('should execute', function(done) {
+        var url = 'http://localhost:8388/simple-page.html';
+
+        this.timeout(15000);
+
+        phantomasWrapper.execute({
+            url: url,
+            options: {}
+        }).then(function(json) {
+            json.should.be.an('object');
+            json.should.have.a.property('generator');
+            json.generator.should.contain('phantomas');
+            json.should.have.a.property('url').that.equals(url);
+            json.should.have.a.property('metrics').that.is.an('object').not.empty;
+            json.should.have.a.property('offenders').that.is.an('object').not.empty;
+            json.offenders.should.have.a.property('javascriptExecutionTree').that.is.a('array').not.empty;
+
+            done();
+        }).fail(function(err) {
+            done(err);
+        });
+    });
+
+    it('should fail with error 254', function(done) {
+        var url = 'http://localhost:8389/not-existing.html';
+
+        this.timeout(15000);
+
+        phantomasWrapper.execute({
+            url: url,
+            options: {}
+        }).then(function(json) {
+
+            done('Error: unwanted success');
+
+        }).fail(function(err) {
+
+            should.exist(err);
+            err.should.equal(254);
+
+            done();
+        });
+    });
+
+    it('should timeout but return some results', function(done) {
+        var url = 'http://localhost:8388/simple-page.html';
+
+        this.timeout(5000);
+        phantomasWrapper.execute({
+            url: url,
+            options: {
+                timeout: 1
+            }
+        }).then(function(json) {
+            
+            json.should.be.an('object');
+            json.should.have.a.property('generator');
+            json.generator.should.contain('phantomas');
+            json.should.have.a.property('url').that.equals(url);
+            json.should.have.a.property('metrics').that.is.an('object').not.empty;
+            json.should.have.a.property('offenders').that.is.an('object').not.empty;
+            json.offenders.should.have.a.property('javascriptExecutionTree').that.is.a('array').not.empty;
+
+            done();
+        }).fail(function(err) {
+            done(err);
+        });
+    });
+});

test/api/rulesCheckerTest.js

@@ -0,0 +1,30 @@
+var should = require('chai').should();
+var rulesChecker = require('../../lib/rulesChecker');
+
+describe('rulesChecker', function() {
+    
+    it('should have a method check', function() {
+        rulesChecker.should.have.property('check').that.is.a('function');
+    });
+    
+    it('should produce a nice rules object', function(done) {
+        var data = require('../fixtures/rulesCheckerInput.json');
+        var policies = require('../fixtures/rulesCheckerPolicies.json');
+        var expected = require('../fixtures/rulesCheckerOutput.json');
+
+        var checker = rulesChecker.check(data, policies);
+
+        checker.then(function(results) {
+            try {
+                results.should.deep.equals(expected);
+                done();
+            } catch(e) {
+                done(e);
+            }
+        });
+
+        checker.fail(function(err) {
+            done(err);
+        });
+    });
+});

test/api/yellowlabtoolsTest.js

@@ -0,0 +1,52 @@
+var should              = require('chai').should();
+var YellowLabTools      = require('../../lib/yellowlabtools.js');
+
+
+describe('yellowlabtools', function() {
+
+    it('returns a promise', function() {
+        var ylt = new YellowLabTools();
+
+        ylt.should.have.property('then').that.is.a('function');
+        ylt.should.have.property('fail').that.is.a('function');
+    });
+
+    it('fails an undefined url', function(done) {
+        var ylt = new YellowLabTools().fail(function(err) {
+            err.should.be.a('string').that.equals('URL missing');
+            done();
+        });
+    });
+
+    it('fails with an empty url string', function(done) {
+        var ylt = new YellowLabTools('').fail(function(err) {
+            err.should.be.a('string').that.equals('URL missing');
+            done();
+        });
+    });
+
+    it('succeeds on simple-page.html', function(done) {
+        this.timeout(15000);
+
+        var url = 'http://localhost:8388/simple-page.html';
+
+        var ylt = new YellowLabTools(url)
+            .then(function(data) {
+                data.should.be.an('object');
+                data.should.have.a.property('url').that.equals(url);
+                
+                data.should.have.a.property('metrics').that.is.an('object');
+                data.metrics.should.have.a.property('requests').that.equals(1);
+
+                data.should.have.a.property('offenders').that.is.an('object');
+                data.offenders.should.have.a.property('DOMelementMaxDepth');
+                data.offenders.DOMelementMaxDepth.should.have.length(1);
+                data.offenders.DOMelementMaxDepth[0].should.equal('body > h1[1]');
+
+                done();
+            }).fail(function(err) {
+                done(err);
+            });
+    });
+
+});

test/fixtures/rulesCheckerInput.json

@@ -0,0 +1,25 @@
+{
+    "tool1": {
+        "metrics": {
+            "metric1": 1236,
+            "metric2": 222,
+            "metric3": 6666,
+            "metric4": 1000,
+            "metric5": 3000,
+            "metric6": 0,
+            "metric7": 5000
+        },
+        "offenders": {
+            "metric1": [],
+            "metric2": [],
+            "metric3": ["offender1", "offender2"],
+            "metric5": []
+        }
+    },
+    "tool2": {
+        "metrics": {
+            "metric1": 22,
+            "metric10": 22
+        }
+    }
+}

test/fixtures/rulesCheckerOutput.json

@@ -0,0 +1,124 @@
+{
+    "metric1": {
+        "policy": {
+            "tool": "tool1",
+            "label": "The metric 1",
+            "message": "A great message",
+            "isOkThreshold": 1000,
+            "isBadThreshold": 3000,
+            "isAbnormalThreshold": 5000
+        },
+        "value": 1236,
+        "bad": true,
+        "abnormal": false,
+        "score": 88,
+        "abnormalityScore": 0
+    },
+    "metric2": {
+        "policy": {
+            "tool": "tool1",
+            "label": "The metric 2",
+            "message": "A great message",
+            "isOkThreshold": 1000,
+            "isBadThreshold": 3000,
+            "isAbnormalThreshold": 5000
+        },
+        "value": 222,
+        "bad": false,
+        "abnormal": false,
+        "score": 100,
+        "abnormalityScore": 0
+    },
+    "metric3": {
+        "policy": {
+            "tool": "tool1",
+            "label": "The metric 3",
+            "message": "A great message",
+            "isOkThreshold": 1000,
+            "isBadThreshold": 3000,
+            "isAbnormalThreshold": 5000
+        },
+        "value": 6666,
+        "offenders": ["offender1", "offender2"],
+        "bad": true,
+        "abnormal": true,
+        "score": 0,
+        "abnormalityScore": -42
+    },
+    "metric4": {
+        "policy": {
+            "tool": "tool1",
+            "label": "The metric 4",
+            "message": "A great message",
+            "isOkThreshold": 1000,
+            "isBadThreshold": 3000,
+            "isAbnormalThreshold": 5000
+        },
+        "value": 1000,
+        "bad": false,
+        "abnormal": false,
+        "score": 100,
+        "abnormalityScore": 0
+    },
+    "metric5": {
+        "policy": {
+            "tool": "tool1",
+            "label": "The metric 5",
+            "message": "A great message",
+            "isOkThreshold": 1000,
+            "isBadThreshold": 3000,
+            "isAbnormalThreshold": 5000
+        },
+        "value": 3000,
+        "bad": true,
+        "abnormal": false,
+        "score": 0,
+        "abnormalityScore": 0
+    },
+    "metric6": {
+        "policy": {
+            "tool": "tool1",
+            "label": "The metric 6",
+            "message": "A great message",
+            "isOkThreshold": 1000,
+            "isBadThreshold": 3000,
+            "isAbnormalThreshold": 5000
+        },
+        "value": 0,
+        "bad": false,
+        "abnormal": false,
+        "score": 100,
+        "abnormalityScore": 0
+    },
+    "metric7": {
+        "policy": {
+            "tool": "tool1",
+            "label": "The metric 7",
+            "message": "A great message",
+            "isOkThreshold": 1000,
+            "isBadThreshold": 3000,
+            "isAbnormalThreshold": 5000
+        },
+        "value": 5000,
+        "bad": true,
+        "abnormal": true,
+        "score": 0,
+        "abnormalityScore": 0
+    },
+
+    "metric10": {
+        "policy": {
+            "tool": "tool2",
+            "label": "The metric 10",
+            "message": "<p>This is from another tool!</p>",
+            "isOkThreshold": 0,
+            "isBadThreshold": 3,
+            "isAbnormalThreshold": 11
+        },
+        "value": 22,
+        "bad": true,
+        "abnormal": true,
+        "score": 0,
+        "abnormalityScore": -100
+    }
+}

test/fixtures/rulesCheckerPolicies.json

@@ -0,0 +1,82 @@
+{
+    "metric1": {
+        "tool": "tool1",
+        "label": "The metric 1",
+        "message": "A great message",
+        "isOkThreshold": 1000,
+        "isBadThreshold": 3000,
+        "isAbnormalThreshold": 5000
+    },
+    "metric2": {
+        "tool": "tool1",
+        "label": "The metric 2",
+        "message": "A great message",
+        "isOkThreshold": 1000,
+        "isBadThreshold": 3000,
+        "isAbnormalThreshold": 5000
+    },
+    "metric3": {
+        "tool": "tool1",
+        "label": "The metric 3",
+        "message": "A great message",
+        "isOkThreshold": 1000,
+        "isBadThreshold": 3000,
+        "isAbnormalThreshold": 5000
+    },
+    "metric4": {
+        "tool": "tool1",
+        "label": "The metric 4",
+        "message": "A great message",
+        "isOkThreshold": 1000,
+        "isBadThreshold": 3000,
+        "isAbnormalThreshold": 5000
+    },
+    "metric5": {
+        "tool": "tool1",
+        "label": "The metric 5",
+        "message": "A great message",
+        "isOkThreshold": 1000,
+        "isBadThreshold": 3000,
+        "isAbnormalThreshold": 5000
+    },
+    "metric6": {
+        "tool": "tool1",
+        "label": "The metric 6",
+        "message": "A great message",
+        "isOkThreshold": 1000,
+        "isBadThreshold": 3000,
+        "isAbnormalThreshold": 5000
+    },
+    "metric7": {
+        "tool": "tool1",
+        "label": "The metric 7",
+        "message": "A great message",
+        "isOkThreshold": 1000,
+        "isBadThreshold": 3000,
+        "isAbnormalThreshold": 5000
+    },
+
+    "metric10": {
+        "tool": "tool2",
+        "label": "The metric 10",
+        "message": "<p>This is from another tool!</p>",
+        "isOkThreshold": 0,
+        "isBadThreshold": 3,
+        "isAbnormalThreshold": 11
+    },
+
+    "unexistantMetric": {
+        "tool": "tool1",
+        "label": "",
+        "message": "",
+        "isOkThreshold": 1000,
+        "isBadThreshold": 3000,
+        "isAbnormalThreshold": 5000
+    },
+    "unexistantTool": {
+        "tool": "unexistant",
+        "isOkThreshold": 1000,
+        "isBadThreshold": 3000,
+        "isAbnormalThreshold": 5000
+    }
+}

test/server/phantomasWrapperTest.js

@@ -1,53 +0,0 @@
-var should = require('chai').should();
-var phantomasWrapper = require('../../app/lib/phantomasWrapper.js');
-
-describe('phantomasWrapper', function() {
-    
-    it('should have a method execute', function() {
-        phantomasWrapper.should.have.property('execute').that.is.a('function');
-    });
-    
-    it('should execute', function(done) {
-        var url = 'http://www.google.fr/';
-
-        this.timeout(20000);
-        phantomasWrapper.execute({
-            url: url,
-            testId: '123abc',
-            options:{
-
-            }
-        }, function(err, json, results) {
-            should.not.exist(err);
-            
-            json.should.be.an('object');
-            json.should.have.a.property('generator');
-            json.generator.should.contain('phantomas');
-            json.should.have.a.property('url').that.equals(url);
-            json.should.have.a.property('metrics').that.is.an('object').not.empty;
-            json.should.have.a.property('offenders').that.is.an('object').not.empty;
-            json.offenders.should.have.a.property('javascriptExecutionTree').that.is.a('array').not.empty;
-
-            done();
-        });
-    });
-
-    it('should fail with error 254', function(done) {
-        var url = 'http://www.not.existing';
-
-        this.timeout(15000);
-        phantomasWrapper.execute({
-            url: url,
-            testId: '123abc',
-            options:{
-
-            }
-        }, function(err, json, results) {
-            should.exist(err);
-            err.should.equal(254);
-            should.not.exist(json);
-
-            done();
-        });
-    });
-});

test/www/simple-page.html

@@ -0,0 +1,8 @@
+<html>
+<head>
+    <title>Simple page</title>
+</head>
+<body>
+    <h1>Simple page</h1>
+</body>
+</html>