Merge branch 'develop' of https://github.com/gmetais/YellowLabTools into develop

bin/cli.js

@@ -111,6 +111,7 @@ if (cli.flags.reporter && cli.flags.reporter !== 'json' && cli.flags.reporter !=
                     xmlOutput = xmlOutput.replace(/(<[a-zA-Z]*>[^<]*)\n([^<]*<\/[a-zA-Z]*>)/g, '$1$2');
                     xmlOutput = xmlOutput.replace(/\0/g, '');
                     xmlOutput = xmlOutput.replace(/\uFFFF/g, '');
+                    xmlOutput = xmlOutput.replace(/\u0002/g, '');
 
                     console.log(xmlOutput);
                     break;

front/src/views/rule.html

@@ -117,10 +117,6 @@
                         <file-and-line-button file="offender.file" line="offender.line" column="offender.column"></file-and-line-button>
                     </div>
 
-                    <div ng-if="policyName === 'cssDuplicatedSelectors'">
-                        {{offender.rule}} (<b>x{{offender.occurrences}}</b>)
-                    </div>
-
                     <div ng-if="policyName === 'cssOldPropertyPrefixes'">
                         <b>{{offender.property}} {{offender.message}}</b>
                         <div ng-if="offender.rules.length" ng-click="offender.showMore = !offender.showMore" class="offenderButton">
@@ -188,6 +184,10 @@
                         <span ng-if="offender.line !== null && offender.column !== null"> @ {{offender.line}}:{{offender.column}}</span>
                     </div>
 
+                    <div ng-if="policyName === 'cssDuplicatedSelectors'">
+                        {{offender.rule}} (<b>x{{offender.occurrences}}</b>)
+                    </div>
+
                     <div ng-if="policyName === 'cssDuplicatedProperties'">
                         Property <b>{{offender.property}}</b> duplicated in <b>{{offender.rule}} { }</b>
                         <span ng-if="offender.line !== null && offender.column !== null"> @ {{offender.line}}:{{offender.column}}</span>
@@ -455,6 +455,11 @@
                         </div>
                     </div>
                 </div>
+                <div ng-if="font.ligaturesOrHiddenChars > 0">
+                    <div><b>Ligatures or hidden chars</b></div>
+                    <div ng-class="{offenderProblem: (font.ligaturesOrHiddenChars > 25)}">{{font.ligaturesOrHiddenChars}} glyphs</div>
+                    <div></div>
+                </div>
             </div>
         </div>
     </div>

lib/metadata/policies.js

@@ -596,24 +596,24 @@ var policies = {
         "isAbnormalThreshold": 100,
         "hasOffenders": true,
         "offendersTransformFn": function(offenders) {
-            return {
-                count: offenders.length,
-                list: offenders.map(function(offender) {
-                    var parts = /^(.*) \((\d+) times\)$/.exec(offender);
-
-                    if (!parts) {
-                        debug('cssDuplicatedSelectors offenders transform function error with "%s"', offender);
-                        return {
-                            parseError: offender
-                        };
-                    }
+            var parsedOffenders = offenders.map(function(offender) {
+                var parts = /^(.*) \((\d+) times\) ?<(.*)>$/.exec(offender);
 
+                if (!parts) {
+                    debug('cssDuplicatedSelectors offenders transform function error with "%s"', offender);
                     return {
-                        rule: parts[1],
-                        occurrences: parseInt(parts[2], 10)
+                        parseError: offender
                     };
-                })
-            };
+                }
+
+                return {
+                    rule: parts[1],
+                    occurrences: parseInt(parts[2], 10),
+                    file: parts[3]
+                };
+            });
+
+            return offendersHelpers.orderByFile(parsedOffenders);
         }
     },
     "cssDuplicatedProperties": {
@@ -1062,7 +1062,7 @@ var policies = {
     "unusedUnicodeRanges": {
         "tool": "redownload",
         "label": "Unused Unicode ranges",
-        "message": "<p>This metric counts the number of unused Unicode ranges inside each font. For example, one font could include Cyrillic glyphs but none of them are used on the page.</p><p>Because of technical limitations, Yellow Lab Tools checks each font against the glyphs of the entire page. As a result, estimated use is >= to reality. For example, if you read that 10 glyphs are \"possibly used\", it means that these 10 glyphs are used on the page but nothing guaranties that they are displayed using this font.</p><p>Tools such as <a href=\"https://www.fontsquirrel.com/tools/webfont-generator\" target=\"_blank\">Font Squirrel</a> can remove some unicode ranges from a font.</p><p>In the case of an icon font, make sure you only keep the icons that are used on the website and to remove the others. Several tools are able to extract SVG images from a font, then some other tools can generate a font from the SVGs you want to keep.</p>",
+        "message": "<p>This metric counts the number of unused Unicode ranges inside each font. For example, one font could include Cyrillic glyphs but none of them are used on the page.</p><p>It also reveals the number of ligatures (letters that are represented differently when close to each other) and hidden chars (glyphs not linked to the unicode system that can't be displayed on the web).</p><p>Because of technical limitations, Yellow Lab Tools checks each font against the glyphs of the entire page. As a result, estimated use is >= to reality. For example, if you read that 10 glyphs are \"possibly used\", it means that these 10 glyphs are used on the page but nothing guaranties that they are displayed using this font.</p><p>Tools such as <a href=\"https://www.fontsquirrel.com/tools/webfont-generator\" target=\"_blank\">Font Squirrel</a> can remove some unicode ranges from a font.</p><p>In the case of an icon font, make sure you only keep the icons that are used on the website and to remove the others. Several tools are able to extract SVG images from a font, then some other tools can generate a font from the SVGs you want to keep.</p>",
         "isOkThreshold": 0,
         "isBadThreshold": 8,
         "isAbnormalThreshold": 12,

lib/tools/redownload/fontAnalyzer.js

@@ -57,8 +57,17 @@ var FontAnalyzer = function() {
             var endTime = Date.now();
             debug('Font analysis took %dms', endTime - startTime);
 
-            deferred.resolve(result);
+            // Mark fonts that are not used on the page (#224)
+            var fontIsUsed = false;
+            for (var range in result.unicodeRanges) {
+                if (result.unicodeRanges[range].numGlyphsInCommonWithPageContent > 0) {
+                    fontIsUsed = true;
+                    break;
+                }
+            }
+            result.isUsed = fontIsUsed;
 
+            deferred.resolve(result);
         } catch(error) {
             deferred.reject(error);
         }

lib/tools/redownload/redownload.js

@@ -103,6 +103,14 @@ var Redownload = function() {
                 var metrics = {};
                 var offenders = {};
 
+                // Remove unused fonts that a normal browser would not download (fix #224)
+                results = results.filter(function(result) {
+                    if (result && result.fontMetrics) {
+                        return result.fontMetrics.isUsed !== false;
+                    }
+                    return true;
+                });
+
                 // Count requests
                 offenders.totalRequests = listRequestsByType(results);
                 metrics.totalRequests = offenders.totalRequests.total;
@@ -500,6 +508,8 @@ var Redownload = function() {
                 var ranges = [];
                 var others = null;
                 var rangeNames = Object.keys(req.fontMetrics.unicodeRanges);
+                var unicodePointsCount = 0;
+                var unusedRangesInFont = 0;
                 
                 rangeNames.forEach(function(rangeName) {
                     var range = req.fontMetrics.unicodeRanges[rangeName];
@@ -508,18 +518,20 @@ var Redownload = function() {
                     if (rangeName === 'Others') {
                         if (range.numGlyphsInCommonWithPageContent === 0 && range.charset.length > 50) {
                             range.underused = true;
-                            unusedUnicodeRanges ++;
+                            unusedRangesInFont ++;
                         }
 
+                        unicodePointsCount += range.charset.length;
                         others = range;
                     } else if (range.charset.length > 0) {
                         // Now lets detect if the current Unicode range is unused.
                         // Reminder: range.coverage = glyphs declared in this range, divided by the range size
                         if (range.coverage > 0.25 && range.numGlyphsInCommonWithPageContent === 0) {
                             range.underused = true;
-                            unusedUnicodeRanges ++;
+                            unusedRangesInFont ++;
                         }
 
+                        unicodePointsCount += range.charset.length;
                         ranges.push(range);
                     }
                 });
@@ -539,7 +551,7 @@ var Redownload = function() {
 
                     // And if less than 5% of the icons are used, let's report it as underused
                     if (others && others.numGlyphsInCommonWithPageContent / others.charset.length <= 0.05) {
-                        unusedUnicodeRanges ++;
+                        unusedRangesInFont = 1;
                     }
 
                 // Not an icons font
@@ -549,13 +561,21 @@ var Redownload = function() {
                         ranges.push(others);
                     }
 
+                    var ligaturesOrHiddenChars = req.fontMetrics.numGlyphs - unicodePointsCount;
+                    if (ligaturesOrHiddenChars > 25) {
+                        unusedUnicodeRanges ++;
+                    }
+
                     list.push({
                         url: req.url,
                         weight: req.weightCheck.bodySize,
                         isIconFont: false,
-                        unicodeRanges: ranges
+                        unicodeRanges: ranges,
+                        ligaturesOrHiddenChars: ligaturesOrHiddenChars 
                     });
                 }
+
+                unusedUnicodeRanges += unusedRangesInFont;
             }
         });
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "yellowlabtools",
-  "version": "1.12.1",
+  "version": "1.12.3",
   "description": "Online tool to audit a webpage for performance and front-end quality issues",
   "license": "GPL-2.0",
   "author": {
@@ -60,7 +60,7 @@
     "meow": "3.7.0",
     "minimize": "2.0.0",
     "parse-color": "1.0.0",
-    "phantomas": "1.18.0",
+    "phantomas": "1.19.0",
     "ps-node": "0.1.4",
     "q": "1.4.1",
     "request": "2.79.0",