diff --git a/src/extensions/ua-parser-extensions.js b/src/extensions/ua-parser-extensions.js index f1da2ac..761b3c8 100644 --- a/src/extensions/ua-parser-extensions.js +++ b/src/extensions/ua-parser-extensions.js @@ -42,24 +42,25 @@ const Crawlers = Object.freeze({ [ // AhrefsBot - https://ahrefs.com/robot // Amazonbot - https://developer.amazon.com/amazonbot - // Bingbot - http://www.bing.com/bingbot.htm + // Bingbot / AdIdxBot - https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0 // CCBot - https://commoncrawl.org/faq // Dotbot - https://moz.com/help/moz-procedures/crawlers/dotbot // DuckDuckBot - http://duckduckgo.com/duckduckbot.html // FacebookBot - https://developers.facebook.com/docs/sharing/bot/ // GPTBot - https://platform.openai.com/docs/gptbot + // LinkedInBot - http://www.linkedin.com // MJ12bot - https://mj12bot.com/ // MojeekBot - https://www.mojeek.com/bot.html // OpenAI's SearchGPT - https://platform.openai.com/docs/bots // PerplexityBot - https://perplexity.ai/perplexitybot // SeznamBot - http://napoveda.seznam.cz/seznambot-intro - /((?:ahrefs|amazon|bing|cc|dot|duckduck|exa|facebook|gpt|mj12|mojeek|oai-search|perplexity|semrush|seznam)bot)\/([\w\.-]+)/i, + /((?:adidx|ahrefs|amazon|bing|cc|dot|duckduck|exa|facebook|gpt|linkedin|mj12|mojeek|oai-search|perplexity|semrush|seznam)bot)\/([\w\.-]+)/i, // Applebot - http://apple.com/go/applebot - /(applebot(?:-extended)?)\/([\w\.]+)/i, + /(applebot(?:-extended)?)\/?([\w\.]*)/i, // Baiduspider https://help.baidu.com/question?prod_id=99&class=0&id=3001 - /(baiduspider)[-imagevdonsfcpr]{0,6}\/([\w\.]+)/i, + /(baiduspider[-imagevdonwsfcpr]{0,7})\/?([\w\.]*)/i, // ClaudeBot (Anthropic) /(claude(?:bot|-web)|anthropic-ai)\/?([\w\.]*)/i, @@ -92,8 +93,8 @@ const Crawlers = Object.freeze({ // Yeti (Naver) /(yeti)\/([\w\.]+)/i, - // aiHitBot / Diffbot / Magpie-Crawler / Omgilibot / Webzio-Extended / Screaming Frog SEO Spider / Timpibot / VelenPublicWebCrawler / YisouSpider / YouBot - /((?:aihit|diff|timpi|you)bot|omgili(?:bot)?|(?:magpie-|velenpublicweb)crawler|webzio-extended|(?:screaming frog seo |yisou)spider)\/?([\w\.]*)/i + // aiHitBot / Diffbot / Linespider / Magpie-Crawler / Omgilibot / OpenAI Image Downloader / Webzio-Extended / Screaming Frog SEO Spider / Timpibot / VelenPublicWebCrawler / YisouSpider / YouBot + /((?:aihit|diff|timpi|you)bot|omgili(?:bot)?|openai image downloader|(?:magpie-|velenpublicweb)crawler|webzio-extended|(?:screaming frog seo |line|yisou)spider)\/?([\w\.]*)/i ], [NAME, VERSION, [TYPE, CRAWLER]], @@ -219,10 +220,10 @@ const Fetchers = Object.freeze({ // AhrefsSiteAudit - https://ahrefs.com/robot/site-audit // ChatGPT-User - https://platform.openai.com/docs/plugins/bot // DuckAssistBot - https://duckduckgo.com/duckassistbot/ - // Better Uptime / BingPreview / Mastodon / Pinterestbot / Redditbot / Rogerbot / SiteAuditBot / Telegrambot / Twitterbot / UptimeRobot + // Better Uptime / BingPreview / Mastodon / MicrosoftPreview / Pinterestbot / Redditbot / Rogerbot / SiteAuditBot / Telegrambot / Twitterbot / UptimeRobot // Google Site Verifier / Meta / Yahoo! Japan // Yandex Bots - https://yandex.com/bots - /(ahrefssiteaudit|bingpreview|chatgpt-user|mastodon|(?:discord|duckassist|linkedin|pinterest|reddit|roger|siteaudit|twitter|uptimero)bot|google-site-verification|meta-externalfetcher|y!?j-dlc|yandex(?:calendar|direct(?:dyn)?|searchshop)|yadirectfetcher)\/([\w\.]+)/i, + /(ahrefssiteaudit|(?:bing|microsoft)preview|chatgpt-user|mastodon|(?:discord|duckassist|linkedin|pinterest|reddit|roger|siteaudit|twitter|uptimero)bot|google-site-verification|meta-externalfetcher|y!?j-dlc|yandex(?:calendar|direct(?:dyn)?|searchshop)|yadirectfetcher)\/([\w\.]+)/i, // Bluesky /(bluesky) cardyb\/([\w\.]+)/i, diff --git a/test/data/ua/extension/crawler.json b/test/data/ua/extension/crawler.json index 2a4d096..8ed2d00 100644 --- a/test/data/ua/extension/crawler.json +++ b/test/data/ua/extension/crawler.json @@ -9,6 +9,16 @@ "type" : "crawler" } }, + { + "desc" : "AdIdxBot", + "ua" : "Mozilla/5.0 (compatible; adidxbot/2.0; +http://www.bing.com/bingbot.htm)", + "expect" : + { + "name" : "adidxbot", + "version" : "2.0", + "type" : "crawler" + } + }, { "desc" : "AdsBot Mobile Web", "ua" : "AdsBot-Google (+http://www.google.com/adsbot.html)", @@ -79,6 +89,16 @@ "type" : "crawler" } }, + { + "desc" : "Applebot-Extended", + "ua" : "Applebot-Extended", + "expect" : + { + "name" : "Applebot-Extended", + "version" : "undefined", + "type" : "crawler" + } + }, { "desc" : "Amazonbot", "ua" : "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/600.2.5 (KHTML, like Gecko) Version/8.0.2 Safari/600.2.5 (Amazonbot/0.1; +https://developer.amazon.com/support/amazonbot)", @@ -89,6 +109,126 @@ "type" : "crawler" } }, + { + "desc" : "Anthropic AI", + "ua" : "anthropic-ai", + "expect" : + { + "name" : "anthropic-ai", + "version" : "undefined", + "type" : "crawler" + } + }, + { + "desc" : "Archive.org Bot", + "ua" : "ia_archiver/8.1 (Windows 2000 1.9; en-US;)", + "expect" : + { + "name" : "ia_archiver", + "version" : "8.1", + "type" : "crawler" + } + }, + { + "desc" : "Archive.org Bot", + "ua" : "Mozilla/5.0 (compatible; archive.org_bot/3.3.0 +https://archive.org/details/archive.org_bot)", + "expect" : + { + "name" : "archive.org_bot", + "version" : "3.3.0", + "type" : "crawler" + } + }, + { + "desc" : "Baiduspider", + "ua" : "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)", + "expect" : + { + "name" : "Baiduspider", + "version" : "2.0", + "type" : "crawler" + } + }, + { + "desc" : "Baiduspider-ads", + "ua" : "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:17.0; Baiduspider-ads) Gecko/17.0 Firefox/17.0", + "expect" : + { + "name" : "Baiduspider-ads", + "version" : "undefined", + "type" : "crawler" + } + }, + { + "desc" : "Baiduspider-cpro", + "ua" : "Mozilla/5.0 (compatible; Baiduspider-cpro; +http://www.baidu.com/search/spider.html)", + "expect" : + { + "name" : "Baiduspider-cpro", + "version" : "undefined", + "type" : "crawler" + } + }, + { + "desc" : "Baiduspider-favo", + "ua" : "Baiduspider-favo", + "expect" : + { + "name" : "Baiduspider-favo", + "version" : "undefined", + "type" : "crawler" + } + }, + { + "desc" : "Baiduspider-image", + "ua" : "Baiduspider-image+(+http://www.baidu.com/search/spider.htm)", + "expect" : + { + "name" : "Baiduspider-image", + "version" : "undefined", + "type" : "crawler" + } + }, + { + "desc" : "Baiduspider-news", + "ua" : "Baiduspider-news", + "expect" : + { + "name" : "Baiduspider-news", + "version" : "undefined", + "type" : "crawler" + } + }, + { + "desc" : "Baiduspider-render", + "ua" : "Mozilla/5.0 (compatible; Baiduspider-render/2.0; +http://www.baidu.com/search/spider.html)", + "expect" : + { + "name" : "Baiduspider-render", + "version" : "2.0", + "type" : "crawler" + } + }, + { + "desc" : "Baiduspider-video", + "ua" : "Baiduspider-video", + "expect" : + { + "name" : "Baiduspider-video", + "version" : "undefined", + "type" : "crawler" + } + }, + { + "desc" : "Bingbot", + "ua" : "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm) Chrome/", + "expect" : + { + "name" : "bingbot", + "version" : "2.0", + "type" : "crawler" + } + }, { "desc" : "Bytespider", "ua" : "Mozilla/5.0 (Linux; Android 8.0; Pixel 2 Build/OPD3.170816.012) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.1511.1269 Mobile Safari/537.36; Bytespider", @@ -179,6 +319,16 @@ "type" : "crawler" } }, + { + "desc" : "DuckDuckBot", + "ua" : "DuckDuckBot/1.1; ( http://duckduckgo.com/duckduckbot.html)", + "expect" : + { + "name" : "DuckDuckBot", + "version" : "1.1", + "type" : "crawler" + } + }, { "desc" : "Exabot", "ua" : "Mozilla/5.0 (compatible; Exabot/3.0; +http://www.exabot.com/go/robot)", @@ -339,26 +489,6 @@ "type" : "crawler" } }, - { - "desc" : "Archive.org Bot", - "ua" : "ia_archiver/8.1 (Windows 2000 1.9; en-US;)", - "expect" : - { - "name" : "ia_archiver", - "version" : "8.1", - "type" : "crawler" - } - }, - { - "desc" : "Archive.org Bot", - "ua" : "Mozilla/5.0 (compatible; archive.org_bot/3.3.0 +https://archive.org/details/archive.org_bot)", - "expect" : - { - "name" : "archive.org_bot", - "version" : "3.3.0", - "type" : "crawler" - } - }, { "desc" : "ImagesiftBot", "ua" : "Mozilla/5.0 (compatible; ImagesiftBot; +imagesift.com)", @@ -369,6 +499,26 @@ "type" : "crawler" } }, + { + "desc" : "Linespider", + "ua" : "Mozilla/5.0 (compatible; Linespider/1.1; +https://lin.ee/4dwXkTH)", + "expect" : + { + "name" : "Linespider", + "version" : "1.1", + "type" : "crawler" + } + }, + { + "desc" : "LinkedInBot", + "ua" : "LinkedInBot/1.0 (compatible; Mozilla/5.0; Apache-HttpClient +http://www.linkedin.com)", + "expect" : + { + "name" : "LinkedInBot", + "version" : "1.0", + "type" : "crawler" + } + }, { "desc" : "magpie-crawler", "ua" : "magpie-crawler/1.1 (robots-txt-checker; +http://www.brandwatch.net)", @@ -462,7 +612,7 @@ }, { "desc" : "PetalBot", - "ua" : "Mozilla/5.0 (Linux; Android 7.0;) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; PetalBot;+https://webmaster.petalsearch.com/site/petalbot) ", + "ua" : "Mozilla/5.0 (Linux; Android 7.0;) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; PetalBot;+https://webmaster.petalsearch.com/site/petalbot)", "expect" : { "name" : "PetalBot", @@ -520,6 +670,16 @@ "type" : "crawler" } }, + { + "desc" : "Sogou", + "ua" : "Sogou web spider/4.0(+http://www.sogou.com/docs/help/webmasters.htm#07)", + "expect" : + { + "name" : "Sogou web spider", + "version" : "4.0", + "type" : "crawler" + } + }, { "desc" : "Teoma", "ua" : "Mozilla/2.0 (compatible; Ask Jeeves/Teoma; +http://sp.ask.com/docs/about/tech_crawling.html)", diff --git a/test/data/ua/extension/fetcher.json b/test/data/ua/extension/fetcher.json index b385428..fed25bc 100644 --- a/test/data/ua/extension/fetcher.json +++ b/test/data/ua/extension/fetcher.json @@ -139,6 +139,26 @@ "type" : "fetcher" } }, + { + "desc" : "MicrosoftPreview", + "ua" : "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; MicrosoftPreview/2.0; +https://aka.ms/MicrosoftPreview) Chrome/W.X.Y.Z Safari/537.36", + "expect" : + { + "name" : "MicrosoftPreview", + "version" : "2.0", + "type" : "fetcher" + } + }, + { + "desc" : "Pinterestbot", + "ua" : "Mozilla/5.0 (compatible; Pinterestbot/1.0; +http://www.pinterest.com/bot.html)", + "expect" : + { + "name" : "Pinterestbot", + "version" : "1.0", + "type" : "fetcher" + } + }, { "desc" : "Rogerbot", "ua" : "Mozilla/5.0 (compatible; rogerBot/1.0; UrlCrawler; http://www.seomoz.org/dp/rogerbot)",