Update test for extensions

package.json

@@ -206,7 +206,7 @@
     "test:eslint": "eslint src && eslint script",
     "test:jshint": "jshint src/main",
     "test:lockfile-lint": "npx lockfile-lint -p package-lock.json",
-    "test:mocha": "mocha -R list test/mocha*js",
+    "test:mocha": "mocha test/mocha*js",
     "test:playwright": "playwright test"
   },
   "devDependencies": {

src/extensions/ua-parser-extensions.js

@@ -21,6 +21,10 @@ const FETCHER   = 'fetcher';
 const INAPP     = 'inapp';
 const MODULE    = 'module';
 
+//////////////////////
+// COMMAND LINE APPS
+/////////////////////
+
 const CLIs = Object.freeze({
     browser : [
         // wget / curl / lynx
@@ -28,6 +32,10 @@ const CLIs = Object.freeze({
     ]
 });
 
+////////////////////////
+// CRAWLERS / SPIDERS
+///////////////////////
+
 const Crawlers = Object.freeze({
     browser : [
         // Amazonbot - https://developer.amazon.com/amazonbot
@@ -70,6 +78,10 @@ const Crawlers = Object.freeze({
     ]
 });
 
+//////////////////
+// EXTRA DEVICES
+/////////////////
+
 const ExtraDevices = Object.freeze({
     device : [[    
         /(nook)[\w ]+build\/(\w+)/i,                                        // Nook
@@ -146,6 +158,10 @@ const ExtraDevices = Object.freeze({
     ]
 });
 
+///////////////
+// EMAIL APPS
+//////////////
+
 const Emails = Object.freeze({
     browser : [
         // Microsoft Outlook / Thunderbird
@@ -153,6 +169,10 @@ const Emails = Object.freeze({
     ]
 });
 
+///////////////////////
+// ON-DEMAND SCRAPERS
+//////////////////////
+
 const Fetchers = Object.freeze({
     browser : [
         // BingPreview / Mastodon / Pinterestbot / Redditbot / Telegrambot / Twitterbot
@@ -181,12 +201,20 @@ const Fetchers = Object.freeze({
     ]
 });
 
+////////////////////
+// IN-APP BROWSERS
+///////////////////
+
 const InApps = Object.freeze({
     browser : [
         [/chatlyio\/([\d\.]+)/i], [VERSION, 'Slack', [TYPE, INAPP]]
     ]
 });
 
+//////////////////////
+// MEDIA PLAYER APPS
+/////////////////////
+
 const MediaPlayers = Object.freeze({
     browser : [[
 
@@ -293,6 +321,10 @@ const MediaPlayers = Object.freeze({
     ]
 });
 
+////////////////////////
+// MODULES / LIBRARIES
+///////////////////////
+
 const Modules = Object.freeze({
     browser : [
         // Axios/jsdom/Scrapy

test/mocha-test-extension.js

@@ -4,58 +4,40 @@ const parseJS = require('@babel/parser').parse;
 const traverse = require('@babel/traverse').default;
 const safe = require('safe-regex');
 const { UAParser } = require('../src/main/ua-parser');
-const { Bots, CLIs, Emails, Modules } = require('../src/extensions/ua-parser-extensions');
+const clis = require('./specs/browser-clis.json');
+const crawlers = require('./specs/browser-crawlers.json');
+const emails = require('./specs/browser-emails.json');
+const fetchers = require('./specs/browser-fetchers.json');
+const modules = require('./specs/browser-modules.json');
+const { CLIs, Crawlers, Emails, Fetchers, Modules } = require('../src/extensions/ua-parser-extensions');
+
+describe('Extensions', () => {
+    [   
+        ['CLIs', clis, CLIs], 
+        ['Crawlers', crawlers, Crawlers], 
+        ['Emails', emails, Emails], 
+        ['Fetchers', fetchers, Fetchers],
+        ['Modules', modules, Modules]
+    ]
+    .forEach((list) => {
+        describe(list[0], () => {
+            list[1].forEach((agent) => {
+                it(`Can detect ${agent.desc}`, () => {
+                    let browser = UAParser(agent.ua, list[2]).browser;
+                    assert.strictEqual(String(browser.name), agent.expect.name);
+                    assert.strictEqual(String(browser.version), agent.expect.version);
+                    assert.strictEqual(String(browser.type), agent.expect.type);
+                });
+            });
+        });
+    });
 
-describe('Bots', () => {
-    it('Can detect bots', () => {
-        const googleBot = 'Googlebot-Video/1.0';
-        const gptBot = 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.0; +https://openai.com/gptbot)';
-        const msnBot = 'msnbot-media/1.1 (+http://search.msn.com/msnbot.htm)';
-        const bingPreview = 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534+ (KHTML, like Gecko) BingPreview/1.0b';
-        const opera = 'Opera/8.5 (Macintosh; PPC Mac OS X; U; en)';
-        const wget = 'Wget/1.21.1';
-        const facebookBot = 'Mozilla/5.0 (compatible; FacebookBot/1.0; +https://developers.facebook.com/docs/sharing/webmasters/facebookbot/)';
-        const yandexBot = 'Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)';
-        const yandexMobileScreenShotBot ='Mozilla/5.0 (compatible; YandexMobileScreenShotBot/1.0; +http://yandex.com/bots)';
-        const appleBot = 'Mozilla/5.0 (iPhone; CPU iPhone OS 8_1 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12B410 Safari/600.1.4 (Applebot/0.1;+http://www.apple.com/go/applebot)';
-        const amazonBot = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML\, like Gecko) Version/8.0.2 Safari/600.2.5 (Amazonbot/0.1; +https://developer.amazon.com/support/amazonbot)';
-        const claudeBot = 'ClaudeBot';
-        const claudeBot2 = 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ClaudeBot/1.0; +claudebot@anthropic.com)';
-        const claudeWeb = 'Claude-Web/1.0 (web crawler; +https://www.anthropic.com/; bots@anthropic.com)';
-        const bytespider = 'Mozilla/5.0 (Linux; Android 8.0; Pixel 2 Build/OPD3.170816.012) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.1511.1269 Mobile Safari/537.36; Bytespider';
     const outlook = 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 10.0; WOW64; Trident/7.0; .NET4.0C; .NET4.0E; .NET CLR 2.0.50727; .NET CLR 3.0.30729; .NET CLR 3.5.30729; Microsoft Outlook 16.0.9126; Microsoft Outlook 16.0.9126; ms-office; MSOffice 16)';
     const thunderbird = 'Mozilla/5.0 (X11; Linux x86_64; rv:78.0) Gecko/20100101 Thunderbird/78.13.0';
     const axios = 'axios/1.3.5';
     const jsdom = 'Mozilla/5.0 (darwin) AppleWebKit/537.36 (KHTML, like Gecko) jsdom/20.0.3';
     const scrapy = 'Scrapy/1.5.0 (+https://scrapy.org)';
 
-        const botParser = new UAParser(Bots);
-        assert.deepEqual(botParser.setUA(googleBot).getBrowser(), {name: "Googlebot-Video", version: "1.0", major: "1", type: "bot"});
-        assert.deepEqual(botParser.setUA(gptBot).getBrowser(), {name: "GPTBot", version: "1.0", major: "1", type: "bot"});
-        assert.deepEqual(botParser.setUA(msnBot).getBrowser(), {name: "msnbot-media", version: "1.1", major: "1", type: "bot"});
-        assert.deepEqual(botParser.setUA(yandexBot).getBrowser(), {name: "YandexBot", version: "3.0", major: "3", type: "bot"});
-        assert.deepEqual(botParser.setUA(yandexMobileScreenShotBot).getBrowser(), {name: "yandex", version: undefined, major: undefined, type: "bot"});
-        assert.deepEqual(botParser.setUA(appleBot).getBrowser(), {name: "Applebot", version: "0.1", major: "0", type: "bot"});
-        assert.deepEqual(botParser.setUA(amazonBot).getBrowser(), {name: "Amazonbot", version: "0.1", major: "0", type: "bot"});
-        assert.deepEqual(botParser.setUA(claudeBot).getBrowser(), {name: "ClaudeBot", version: undefined, major: undefined, type: "bot"});
-        assert.deepEqual(botParser.setUA(claudeBot2).getBrowser(), {name: "ClaudeBot", version: "1.0", major: "1", type: "bot"});
-        assert.deepEqual(botParser.setUA(claudeWeb).getBrowser(), {name: "Claude-Web", version: "1.0", major: "1", type: "bot"});
-        assert.deepEqual(botParser.setUA(bytespider).getBrowser(), {name: "Bytespider", version: undefined, major: undefined, type: "bot"});
-        assert.deepEqual(botParser.setUA(bingPreview).getBrowser(), {name: "BingPreview", version: "1.0b", major: "1", type: "bot"});
-        assert.deepEqual(botParser.setUA(opera).getBrowser(), {name: "Opera", version: "8.5", major: "8", type: undefined});
-
-        // try merging Bots & CLIs
-        const botsAndCLIs = { browser : [...Bots.browser, ...CLIs.browser]};
-        const botsAndCLIsParser = new UAParser(botsAndCLIs);
-        assert.deepEqual(botsAndCLIsParser.setUA(wget).getBrowser(), {name: "Wget", version: "1.21.1", major: "1", type:"cli"});
-        assert.deepEqual(botsAndCLIsParser.setUA(facebookBot).getBrowser(), {name: "FacebookBot", version: "1.0", major: "1", type:"bot"});
-
-        // alternative merge options
-        const botsAndCLIsParser2 = new UAParser([Bots, CLIs]);
-        const botsAndCLIsParser3 = new UAParser(facebookBot, [Bots, CLIs]);
-        assert.deepEqual(botsAndCLIsParser2.setUA(wget).getBrowser(), {name: "Wget", version: "1.21.1", major: "1", type:"cli"});
-        assert.deepEqual(botsAndCLIsParser3.getBrowser(), {name: "FacebookBot", version: "1.0", major: "1", type:"bot"});
-
     const emailParser = new UAParser(Emails);
     assert.deepEqual(emailParser.setUA(outlook).getBrowser(), {name: "Microsoft Outlook", version: "16.0.9126", major: "16", type: "email"});
     assert.deepEqual(emailParser.setUA(thunderbird).getBrowser(), {name: "Thunderbird", version: "78.13.0", major: "78", type: "email"});
@@ -64,16 +46,30 @@ describe('Bots', () => {
     assert.deepEqual(moduleParser.setUA(axios).getBrowser(), {name: "axios", version: "1.3.5", major: "1", type: "module"});
     assert.deepEqual(moduleParser.setUA(jsdom).getBrowser(), {name: "jsdom", version: "20.0.3", major: "20", type: "module"});
     assert.deepEqual(moduleParser.setUA(scrapy).getBrowser(), {name: "Scrapy", version: "1.5.0", major: "1", type: "module"});
-    });
 });
 
-// TODO : move test spec to JSON file
+describe('Merge', () => {
+    it('Can merge multiple extensions', () => {
+        const wget = 'Wget/1.21.1';
+        const facebookBot = 'Mozilla/5.0 (compatible; FacebookBot/1.0; +https://developers.facebook.com/docs/sharing/webmasters/facebookbot/)';
+
+        // try merging crawlers & CLIs
+        const crawlersAndCLIs = { browser : [...Crawlers.browser, ...CLIs.browser]};
+        const crawlersAndCLIsParser = new UAParser(crawlersAndCLIs);
+        assert.deepEqual(crawlersAndCLIsParser.setUA(wget).getBrowser(), {name: "Wget", version: "1.21.1", major: "1", type:"cli"});
+        assert.deepEqual(crawlersAndCLIsParser.setUA(facebookBot).getBrowser(), {name: "FacebookBot", version: "1.0", major: "1", type:"crawler"});
+
+        // alternative merge options
+        const crawlersAndCLIsParser2 = new UAParser([Crawlers, CLIs]);
+        const crawlersAndCLIsParser3 = new UAParser(facebookBot, [Crawlers, CLIs]);
+        assert.deepEqual(crawlersAndCLIsParser2.setUA(wget).getBrowser(), {name: "Wget", version: "1.21.1", major: "1", type:"cli"});
+        assert.deepEqual(crawlersAndCLIsParser3.getBrowser(), {name: "FacebookBot", version: "1.0", major: "1", type:"crawler"});
+    });
+});
 
 describe('Testing regexes', () => {
 
     let regexes;
-
-    before('Read main js file', () => {
     let code = fs.readFileSync('src/extensions/ua-parser-extensions.js', 'utf8').toString();
     let ast = parseJS(code, { sourceType: 'script' });
     regexes = [];
@@ -82,20 +78,16 @@ describe('Testing regexes', () => {
             regexes.push(path.node.pattern);
         }
     });
+
     if (regexes.length === 0) {
         throw new Error('Regexes cannot be empty!');
     }
-    });
 
-    describe('Begin testing', () => {
+    describe('Checking for potentially vulnerable regex', () => {
-        it('all regexes in extension file', () => {
+        for (let regex of regexes) {
-            regexes.forEach(regex => {
+            it('Test against `safe-regex` : ' + regex, () => {
-                describe('Test against `safe-regex` : ' + regex, () => {
-                    it('should be safe from potentially vulnerable regex', () => {
                 assert.strictEqual(safe(regex), true);
             });
-                });
+        }
-            });
-        });
     });
 });

test/specs/browser-clis.json

@@ -0,0 +1,42 @@
+[
+    {
+        "desc"    : "curl",
+        "ua"      : "curl/7.38.0",
+        "expect"  :
+        {
+            "name"    : "curl",
+            "version" : "7.38.0",
+            "type"    : "cli"
+        }
+    },
+    {
+        "desc"    : "lynx",
+        "ua"      : "Lynx 2.8.8dev.3",
+        "expect"  :
+        {
+            "name"    : "Lynx",
+            "version" : "2.8.8dev.3",
+            "type"    : "cli"
+        }
+    },
+    {
+        "desc"    : "lynx",
+        "ua"      : "Lynx/2.6",
+        "expect"  :
+        {
+            "name"    : "Lynx",
+            "version" : "2.6",
+            "type"    : "cli"
+        }
+    },
+    {
+        "desc"    : "wget",
+        "ua"      : "Wget/1.21.1",
+        "expect"  :
+        {
+            "name"    : "Wget",
+            "version" : "1.21.1",
+            "type"    : "cli"
+        }
+    }
+]

test/specs/browser-crawlers.json

@@ -0,0 +1,92 @@
+[
+    {
+        "desc"    : "Applebot",
+        "ua"      : "Mozilla/5.0 (iPhone; CPU iPhone OS 8_1 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12B410 Safari/600.1.4 (Applebot/0.1;+http://www.apple.com/go/applebot)",
+        "expect"  :
+        {
+            "name"    : "Applebot",
+            "version" : "0.1",
+            "type"    : "crawler"
+        }
+    },
+    {
+        "desc"    : "Amazonbot",
+        "ua"      : "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML, like Gecko) Version/8.0.2 Safari/600.2.5 (Amazonbot/0.1; +https://developer.amazon.com/support/amazonbot)",
+        "expect"  :
+        {
+            "name"    : "Amazonbot",
+            "version" : "0.1",
+            "type"    : "crawler"
+        }
+    },
+    {
+        "desc"    : "Bytespider",
+        "ua"      : "Mozilla/5.0 (Linux; Android 8.0; Pixel 2 Build/OPD3.170816.012) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.1511.1269 Mobile Safari/537.36; Bytespider",
+        "expect"  :
+        {
+            "name"    : "Bytespider",
+            "version" : "undefined",
+            "type"    : "crawler"
+        }
+    },
+    {
+        "desc"    : "ClaudeBot",
+        "ua"      : "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ClaudeBot/1.0; +claudebot@anthropic.com)",
+        "expect"  :
+        {
+            "name"    : "ClaudeBot",
+            "version" : "1.0",
+            "type"    : "crawler"
+        }
+    },
+    {
+        "desc"    : "ClaudeWeb",
+        "ua"      : "Claude-Web/1.0 (web crawler; +https://www.anthropic.com/; bots@anthropic.com)",
+        "expect"  :
+        {
+            "name"    : "Claude-Web",
+            "version" : "1.0",
+            "type"    : "crawler"
+        }
+    },
+    {
+        "desc"    : "FacebookBot",
+        "ua"      : "Mozilla/5.0 (compatible; FacebookBot/1.0; +https://developers.facebook.com/docs/sharing/webmasters/facebookbot/",
+        "expect"  :
+        {
+            "name"    : "FacebookBot",
+            "version" : "1.0",
+            "type"    : "crawler"
+        }
+    },
+    {
+        "desc"    : "Googlebot-Video",
+        "ua"      : "Googlebot-Video/1.0",
+        "expect"  :
+        {
+            "name"    : "Googlebot-Video",
+            "version" : "1.0",
+            "type"    : "crawler"
+        }
+    },
+    {
+        "desc"    : "GPTBot",
+        "ua"      : "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; GPTBot/1.0; +https://openai.com/gptbot)",
+        "expect"  :
+        {
+            "name"    : "GPTBot",
+            "version" : "1.0",
+            "type"    : "crawler"
+        }
+    },
+    {
+        "desc"    : "YandexBot",
+        "ua"      : "Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)",
+        "expect"  :
+        {
+            "name"    : "YandexBot",
+            "version" : "3.0",
+            "type"    : "crawler"
+        }
+    }
+]

test/specs/browser-emails.json

@@ -0,0 +1,12 @@
+[
+    {
+        "desc"    : "Thunderbird",
+        "ua"      : "Mozilla/5.0 (X11; Linux x86_64; rv:78.0) Gecko/20100101 Thunderbird/78.13.0",
+        "expect"  :
+        {
+            "name"    : "Thunderbird",
+            "version" : "78.13.0",
+            "type"    : "email"
+        }
+    }
+]

test/specs/browser-fetchers.json

@@ -0,0 +1,12 @@
+[
+    {
+        "desc"    : "BingPreview",
+        "ua"      : "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534+ (KHTML, like Gecko) BingPreview/1.0b",
+        "expect"  :
+        {
+            "name"    : "BingPreview",
+            "version" : "1.0b",
+            "type"    : "fetcher"
+        }
+    }
+]

test/specs/browser-modules.json

@@ -0,0 +1,12 @@
+[
+    {
+        "desc"    : "Scrapy",
+        "ua"      : "Scrapy/1.5.0 (+https://scrapy.org)",
+        "expect"  :
+        {
+            "name"    : "Scrapy",
+            "version" : "1.5.0",
+            "type"    : "module"
+        }
+    }
+]