[extensions] Add bots: AdIdxBot, Linespider, LinkedInBot, MicrosoftPreview, OpenAI Image Downloader

src/extensions/ua-parser-extensions.js

@@ -42,24 +42,25 @@ const Crawlers = Object.freeze({
         [
             // AhrefsBot - https://ahrefs.com/robot
             // Amazonbot - https://developer.amazon.com/amazonbot
-            // Bingbot - http://www.bing.com/bingbot.htm
+            // Bingbot / AdIdxBot - https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0
             // CCBot - https://commoncrawl.org/faq
             // Dotbot - https://moz.com/help/moz-procedures/crawlers/dotbot
             // DuckDuckBot - http://duckduckgo.com/duckduckbot.html
             // FacebookBot - https://developers.facebook.com/docs/sharing/bot/
             // GPTBot - https://platform.openai.com/docs/gptbot
+            // LinkedInBot - http://www.linkedin.com
             // MJ12bot - https://mj12bot.com/
             // MojeekBot - https://www.mojeek.com/bot.html
             // OpenAI's SearchGPT - https://platform.openai.com/docs/bots
             // PerplexityBot - https://perplexity.ai/perplexitybot
             // SeznamBot - http://napoveda.seznam.cz/seznambot-intro
-            /((?:ahrefs|amazon|bing|cc|dot|duckduck|exa|facebook|gpt|mj12|mojeek|oai-search|perplexity|semrush|seznam)bot)\/([\w\.-]+)/i,
+            /((?:adidx|ahrefs|amazon|bing|cc|dot|duckduck|exa|facebook|gpt|linkedin|mj12|mojeek|oai-search|perplexity|semrush|seznam)bot)\/([\w\.-]+)/i,
 
             // Applebot - http://apple.com/go/applebot
-            /(applebot(?:-extended)?)\/([\w\.]+)/i,
+            /(applebot(?:-extended)?)\/?([\w\.]*)/i,
 
             // Baiduspider https://help.baidu.com/question?prod_id=99&class=0&id=3001
-            /(baiduspider)[-imagevdonsfcpr]{0,6}\/([\w\.]+)/i,
+            /(baiduspider[-imagevdonwsfcpr]{0,7})\/?([\w\.]*)/i,
 
             // ClaudeBot (Anthropic)
             /(claude(?:bot|-web)|anthropic-ai)\/?([\w\.]*)/i, 
@@ -92,8 +93,8 @@ const Crawlers = Object.freeze({
             // Yeti (Naver)
             /(yeti)\/([\w\.]+)/i,
 
-            // aiHitBot / Diffbot / Magpie-Crawler / Omgilibot / Webzio-Extended / Screaming Frog SEO Spider / Timpibot / VelenPublicWebCrawler / YisouSpider / YouBot
+            // aiHitBot / Diffbot / Linespider / Magpie-Crawler / Omgilibot / OpenAI Image Downloader / Webzio-Extended / Screaming Frog SEO Spider / Timpibot / VelenPublicWebCrawler / YisouSpider / YouBot
-            /((?:aihit|diff|timpi|you)bot|omgili(?:bot)?|(?:magpie-|velenpublicweb)crawler|webzio-extended|(?:screaming frog seo |yisou)spider)\/?([\w\.]*)/i
+            /((?:aihit|diff|timpi|you)bot|omgili(?:bot)?|openai image downloader|(?:magpie-|velenpublicweb)crawler|webzio-extended|(?:screaming frog seo |line|yisou)spider)\/?([\w\.]*)/i
         ],
 
         [NAME, VERSION, [TYPE, CRAWLER]],
@@ -219,10 +220,10 @@ const Fetchers = Object.freeze({
             // AhrefsSiteAudit - https://ahrefs.com/robot/site-audit
             // ChatGPT-User - https://platform.openai.com/docs/plugins/bot
             // DuckAssistBot - https://duckduckgo.com/duckassistbot/
-            // Better Uptime / BingPreview / Mastodon / Pinterestbot / Redditbot / Rogerbot / SiteAuditBot / Telegrambot / Twitterbot / UptimeRobot
+            // Better Uptime / BingPreview / Mastodon / MicrosoftPreview / Pinterestbot / Redditbot / Rogerbot / SiteAuditBot / Telegrambot / Twitterbot / UptimeRobot
             // Google Site Verifier / Meta / Yahoo! Japan
             // Yandex Bots - https://yandex.com/bots
-            /(ahrefssiteaudit|bingpreview|chatgpt-user|mastodon|(?:discord|duckassist|linkedin|pinterest|reddit|roger|siteaudit|twitter|uptimero)bot|google-site-verification|meta-externalfetcher|y!?j-dlc|yandex(?:calendar|direct(?:dyn)?|searchshop)|yadirectfetcher)\/([\w\.]+)/i,
+            /(ahrefssiteaudit|(?:bing|microsoft)preview|chatgpt-user|mastodon|(?:discord|duckassist|linkedin|pinterest|reddit|roger|siteaudit|twitter|uptimero)bot|google-site-verification|meta-externalfetcher|y!?j-dlc|yandex(?:calendar|direct(?:dyn)?|searchshop)|yadirectfetcher)\/([\w\.]+)/i,
 
             // Bluesky
             /(bluesky) cardyb\/([\w\.]+)/i,

test/data/ua/extension/crawler.json

@@ -9,6 +9,16 @@
             "type"    : "crawler"
         }
     },
+    {
+        "desc"    : "AdIdxBot",
+        "ua"      : "Mozilla/5.0 (compatible; adidxbot/2.0; +http://www.bing.com/bingbot.htm)",
+        "expect"  :
+        {
+            "name"    : "adidxbot",
+            "version" : "2.0",
+            "type"    : "crawler"
+        }
+    },
     {
         "desc"    : "AdsBot Mobile Web",
         "ua"      : "AdsBot-Google (+http://www.google.com/adsbot.html)",
@@ -79,6 +89,16 @@
             "type"    : "crawler"
         }
     },
+    {
+        "desc"    : "Applebot-Extended",
+        "ua"      : "Applebot-Extended",
+        "expect"  :
+        {
+            "name"    : "Applebot-Extended",
+            "version" : "undefined",
+            "type"    : "crawler"
+        }
+    },
     {
         "desc"    : "Amazonbot",
         "ua"      : "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML, like Gecko) Version/8.0.2 Safari/600.2.5 (Amazonbot/0.1; +https://developer.amazon.com/support/amazonbot)",
@@ -89,6 +109,126 @@
             "type"    : "crawler"
         }
     },
+    {
+        "desc"    : "Anthropic AI",
+        "ua"      : "anthropic-ai",
+        "expect"  :
+        {
+            "name"    : "anthropic-ai",
+            "version" : "undefined",
+            "type"    : "crawler"
+        }
+    },
+    {
+        "desc"    : "Archive.org Bot",
+        "ua"      : "ia_archiver/8.1 (Windows 2000 1.9; en-US;)",
+        "expect"  :
+        {
+            "name"    : "ia_archiver",
+            "version" : "8.1",
+            "type"    : "crawler"
+        }
+    },
+    {
+        "desc"    : "Archive.org Bot",
+        "ua"      : "Mozilla/5.0 (compatible; archive.org_bot/3.3.0 +https://archive.org/details/archive.org_bot)",
+        "expect"  :
+        {
+            "name"    : "archive.org_bot",
+            "version" : "3.3.0",
+            "type"    : "crawler"
+        }
+    },
+    {
+        "desc"    : "Baiduspider",
+        "ua"      : "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)",
+        "expect"  :
+        {
+            "name"    : "Baiduspider",
+            "version" : "2.0",
+            "type"    : "crawler"
+        }
+    },
+    {
+        "desc"    : "Baiduspider-ads",
+        "ua"      : "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:17.0; Baiduspider-ads) Gecko/17.0 Firefox/17.0",
+        "expect"  :
+        {
+            "name"    : "Baiduspider-ads",
+            "version" : "undefined",
+            "type"    : "crawler"
+        }
+    },
+    {
+        "desc"    : "Baiduspider-cpro",
+        "ua"      : "Mozilla/5.0 (compatible; Baiduspider-cpro; +http://www.baidu.com/search/spider.html)",
+        "expect"  :
+        {
+            "name"    : "Baiduspider-cpro",
+            "version" : "undefined",
+            "type"    : "crawler"
+        }
+    },
+    {
+        "desc"    : "Baiduspider-favo",
+        "ua"      : "Baiduspider-favo",
+        "expect"  :
+        {
+            "name"    : "Baiduspider-favo",
+            "version" : "undefined",
+            "type"    : "crawler"
+        }
+    },
+    {
+        "desc"    : "Baiduspider-image",
+        "ua"      : "Baiduspider-image+(+http://www.baidu.com/search/spider.htm)",
+        "expect"  :
+        {
+            "name"    : "Baiduspider-image",
+            "version" : "undefined",
+            "type"    : "crawler"
+        }
+    },
+    {
+        "desc"    : "Baiduspider-news",
+        "ua"      : "Baiduspider-news",
+        "expect"  :
+        {
+            "name"    : "Baiduspider-news",
+            "version" : "undefined",
+            "type"    : "crawler"
+        }
+    },
+    {
+        "desc"    : "Baiduspider-render",
+        "ua"      : "Mozilla/5.0 (compatible; Baiduspider-render/2.0; +http://www.baidu.com/search/spider.html)",
+        "expect"  :
+        {
+            "name"    : "Baiduspider-render",
+            "version" : "2.0",
+            "type"    : "crawler"
+        }
+    },
+    {
+        "desc"    : "Baiduspider-video",
+        "ua"      : "Baiduspider-video",
+        "expect"  :
+        {
+            "name"    : "Baiduspider-video",
+            "version" : "undefined",
+            "type"    : "crawler"
+        }
+    },
+    {
+        "desc"    : "Bingbot",
+        "ua"      : "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm) Chrome/",
+        "expect"  :
+        {
+            "name"    : "bingbot",
+            "version" : "2.0",
+            "type"    : "crawler"
+        }
+    },
     {
         "desc"    : "Bytespider",
         "ua"      : "Mozilla/5.0 (Linux; Android 8.0; Pixel 2 Build/OPD3.170816.012) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.1511.1269 Mobile Safari/537.36; Bytespider",
@@ -179,6 +319,16 @@
             "type"    : "crawler"
         }
     },
+    {
+        "desc"    : "DuckDuckBot",
+        "ua"      : "DuckDuckBot/1.1; ( http://duckduckgo.com/duckduckbot.html)",
+        "expect"  :
+        {
+            "name"    : "DuckDuckBot",
+            "version" : "1.1",
+            "type"    : "crawler"
+        }
+    },
     {
         "desc"    : "Exabot",
         "ua"      : "Mozilla/5.0 (compatible; Exabot/3.0; +http://www.exabot.com/go/robot)",
@@ -339,26 +489,6 @@
             "type"    : "crawler"
         }
     },
-    {
-        "desc"    : "Archive.org Bot",
-        "ua"      : "ia_archiver/8.1 (Windows 2000 1.9; en-US;)",
-        "expect"  :
-        {
-            "name"    : "ia_archiver",
-            "version" : "8.1",
-            "type"    : "crawler"
-        }
-    },
-    {
-        "desc"    : "Archive.org Bot",
-        "ua"      : "Mozilla/5.0 (compatible; archive.org_bot/3.3.0 +https://archive.org/details/archive.org_bot)",
-        "expect"  :
-        {
-            "name"    : "archive.org_bot",
-            "version" : "3.3.0",
-            "type"    : "crawler"
-        }
-    },
     {
         "desc"    : "ImagesiftBot",
         "ua"      : "Mozilla/5.0 (compatible; ImagesiftBot; +imagesift.com)",
@@ -369,6 +499,26 @@
             "type"    : "crawler"
         }
     },
+    {
+        "desc"    : "Linespider",
+        "ua"      : "Mozilla/5.0 (compatible; Linespider/1.1; +https://lin.ee/4dwXkTH)",
+        "expect"  :
+        {
+            "name"    : "Linespider",
+            "version" : "1.1",
+            "type"    : "crawler"
+        }
+    },
+    {
+        "desc"    : "LinkedInBot",
+        "ua"      : "LinkedInBot/1.0 (compatible; Mozilla/5.0; Apache-HttpClient +http://www.linkedin.com)",
+        "expect"  :
+        {
+            "name"    : "LinkedInBot",
+            "version" : "1.0",
+            "type"    : "crawler"
+        }
+    },
     {
         "desc"    : "magpie-crawler",
         "ua"      : "magpie-crawler/1.1 (robots-txt-checker; +http://www.brandwatch.net)",
@@ -462,7 +612,7 @@
     },
     {
         "desc"    : "PetalBot",
-        "ua"      : "Mozilla/5.0 (Linux; Android 7.0;) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; PetalBot;+https://webmaster.petalsearch.com/site/petalbot) ",
+        "ua"      : "Mozilla/5.0 (Linux; Android 7.0;) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; PetalBot;+https://webmaster.petalsearch.com/site/petalbot)",
         "expect"  :
         {
             "name"    : "PetalBot",
@@ -520,6 +670,16 @@
             "type"    : "crawler"
         }
     },
+    {
+        "desc"    : "Sogou",
+        "ua"      : "Sogou web spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm#07)",
+        "expect"  :
+        {
+            "name"    : "Sogou web spider",
+            "version" : "4.0",
+            "type"    : "crawler"
+        }
+    },
     {
         "desc"    : "Teoma",
         "ua"      : "Mozilla/2.0 (compatible; Ask Jeeves/Teoma; +http://sp.ask.com/docs/about/tech_crawling.html)",

test/data/ua/extension/fetcher.json

@@ -139,6 +139,26 @@
             "type"    : "fetcher"
         }
     },
+    {
+        "desc"    : "MicrosoftPreview",
+        "ua"      : "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; MicrosoftPreview/2.0; +https://aka.ms/MicrosoftPreview) Chrome/W.X.Y.Z Safari/537.36",
+        "expect"  :
+        {
+            "name"    : "MicrosoftPreview",
+            "version" : "2.0",
+            "type"    : "fetcher"
+        }
+    },
+    {
+        "desc"    : "Pinterestbot",
+        "ua"      : "Mozilla/5.0 (compatible; Pinterestbot/1.0; +http://www.pinterest.com/bot.html)",
+        "expect"  :
+        {
+            "name"    : "Pinterestbot",
+            "version" : "1.0",
+            "type"    : "fetcher"
+        }
+    },
     {
         "desc"    : "Rogerbot",
         "ua"      : "Mozilla/5.0 (compatible; rogerBot/1.0; UrlCrawler; http://www.seomoz.org/dp/rogerbot)",