diff --git a/src/extensions/ua-parser-extensions.js b/src/extensions/ua-parser-extensions.js index 4cd644f..06ef90f 100644 --- a/src/extensions/ua-parser-extensions.js +++ b/src/extensions/ua-parser-extensions.js @@ -44,6 +44,8 @@ const Crawlers = Object.freeze({ // Amazonbot - https://developer.amazon.com/amazonbot // Bingbot / AdIdxBot - https://www.bing.com/webmasters/help/which-crawlers-does-bing-use-8c184ec0 // CCBot - https://commoncrawl.org/faq + // Coveobot - https://connect.coveo.com/s/article/19648 + // CriteoBot - https://www.criteo.com/criteo-crawler/ // Dotbot - https://moz.com/help/moz-procedures/crawlers/dotbot // DuckDuckBot - http://duckduckgo.com/duckduckbot.html // FacebookBot - https://developers.facebook.com/docs/sharing/bot/ @@ -56,7 +58,7 @@ const Crawlers = Object.freeze({ // OpenAI's SearchGPT - https://platform.openai.com/docs/bots // PerplexityBot - https://perplexity.ai/perplexitybot // SeznamBot - http://napoveda.seznam.cz/seznambot-intro - /((?:adidx|ahrefs|amazon|bing|cc|dot|duckduck|exa|facebook|gpt|iask|linkedin|mj12|mojeek|oai-search|onespot-scraper|perplexity|semrush|seznam)bot)\/([\w\.-]+)/i, + /((?:adidx|ahrefs|amazon|bing|cc|coveo|criteo|dot|duckduck|exa|facebook|gpt|iask|linkedin|mj12|mojeek|oai-search|onespot-scraper|perplexity|semrush|seznam)bot)\/([\w\.-]+)/i, // Applebot - http://apple.com/go/applebot /(applebot(?:-extended)?)\/?([\w\.]*)/i, @@ -65,7 +67,7 @@ const Crawlers = Object.freeze({ /(baiduspider[-imagevdonwsfcpr]{0,7})\/?([\w\.]*)/i, // ClaudeBot (Anthropic) - /(claude(?:bot|-web)|anthropic-ai)\/?([\w\.]*)/i, + /(claude(?:bot|-searchbot|-web)|anthropic-ai)\/?([\w\.]*)/i, // Coc Coc Bot - https://help.coccoc.com/en/search-engine /(coccocbot-(?:image|web))\/([\w\.]+)/i, @@ -235,15 +237,16 @@ const Fetchers = Object.freeze({ browser : [ [ // AhrefsSiteAudit - https://ahrefs.com/robot/site-audit + // Buffer Link Preview Bot - https://scraper.buffer.com/about/bots/link-preview-bot // ChatGPT-User - https://platform.openai.com/docs/plugins/bot // DuckAssistBot - https://duckduckgo.com/duckassistbot/ - // Better Uptime / BingPreview / Mastodon / MicrosoftPreview / Pinterestbot / Redditbot / Rogerbot / SiteAuditBot / Telegrambot / Twitterbot / UptimeRobot + // Better Uptime / BingPreview / Blueno / Mastodon / MicrosoftPreview / Pinterestbot / Redditbot / Rogerbot / SiteAuditBot / Telegrambot / Twitterbot / UptimeRobot // Google Site Verifier / Meta / Yahoo! Japan // Iframely - https://iframely.com/docs/about // Perplexity-User - https://docs.perplexity.ai/guides/bots // MistralAI-User - https://docs.mistral.ai/robots/ // Yandex Bots - https://yandex.com/bots - /(ahrefssiteaudit|(?:bing|microsoft)preview|(?:chatgpt|mistralai|perplexity)-user|mastodon|(?:discord|duckassist|linkedin|pinterest|reddit|roger|siteaudit|twitter|uptimero)bot|google-site-verification|iframely|meta-externalfetcher|y!?j-dlc|yandex(?:calendar|direct(?:dyn)?|searchshop)|yadirectfetcher)\/([\w\.]+)/i, + /(ahrefssiteaudit|(?:bing|microsoft)preview|blueno|(?:chatgpt|claude|mistralai|perplexity)-user|mastodon|(?:bufferlinkpreview|discord|duckassist|linkedin|pinterest|reddit|roger|siteaudit|twitter|uptimero)bot|google-site-verification|iframely|meta-externalfetcher|y!?j-dlc|yandex(?:calendar|direct(?:dyn)?|searchshop)|yadirectfetcher)\/([\w\.]+)/i, // Bluesky /(bluesky) cardyb\/([\w\.]+)/i, @@ -404,8 +407,8 @@ const Vehicles = Object.freeze({ const Bots = Object.freeze({ browser : [ ...CLIs.browser, - ...Crawlers.browser, ...Fetchers.browser, + ...Crawlers.browser, ...Libraries.browser ], os : [ diff --git a/src/helpers/ua-parser-helpers.js b/src/helpers/ua-parser-helpers.js index 8f79fe8..c00e49b 100644 --- a/src/helpers/ua-parser-helpers.js +++ b/src/helpers/ua-parser-helpers.js @@ -52,6 +52,7 @@ const isAIBot = (resultOrUA) => [ // Anthropic 'anthropic-ai', 'claude-web', + 'claude-searchbot', 'claudebot', // Apple @@ -63,6 +64,9 @@ const isAIBot = (resultOrUA) => [ // Common Crawl 'ccbot', + + // Coveo + 'coveobot', // DataForSeo 'dataforseobot', diff --git a/test/data/ua/extension/crawler.json b/test/data/ua/extension/crawler.json index 918ee98..ac580fc 100644 --- a/test/data/ua/extension/crawler.json +++ b/test/data/ua/extension/crawler.json @@ -269,6 +269,16 @@ "type" : "crawler" } }, + { + "desc" : "ClaudeWeb", + "ua" : "Claude-Web/1.0 (web crawler; +https://www.anthropic.com/; bots@anthropic.com)", + "expect" : + { + "name" : "Claude-Web", + "version" : "1.0", + "type" : "crawler" + } + }, { "desc" : "Coc Coc Bot (web)", "ua" : "Mozilla/5.0 (compatible; coccocbot-web/1.0; +http://help.coccoc.com/searchengine)", @@ -290,12 +300,22 @@ } }, { - "desc" : "ClaudeWeb", - "ua" : "Claude-Web/1.0 (web crawler; +https://www.anthropic.com/; bots@anthropic.com)", + "desc" : "Coveobot", + "ua" : "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko) (compatible; Coveobot/2.0;+http://www.coveo.com/bot.html)", "expect" : { - "name" : "Claude-Web", - "version" : "1.0", + "name" : "Coveobot", + "version" : "2.0", + "type" : "crawler" + } + }, + { + "desc" : "CriteoBot", + "ua" : "CriteoBot/0.1 (+https://www.criteo.com/criteo-crawler/)", + "expect" : + { + "name" : "CriteoBot", + "version" : "0.1", "type" : "crawler" } }, diff --git a/test/data/ua/extension/fetcher.json b/test/data/ua/extension/fetcher.json index 4a05393..19b05bb 100644 --- a/test/data/ua/extension/fetcher.json +++ b/test/data/ua/extension/fetcher.json @@ -29,6 +29,16 @@ "type" : "fetcher" } }, + { + "desc" : "Blueno", + "ua" : "acebookexternalhit/1.1 (compatible; Blueno/1.0; +http://naver.me/scrap)", + "expect" : + { + "name" : "Blueno", + "version" : "1.0", + "type" : "fetcher" + } + }, { "desc" : "Bluesky", "ua" : "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Bluesky Cardyb/1.1; +mailto:support@bsky.app) Chrome/100.0.0.0 Safari/537.36", @@ -39,6 +49,16 @@ "type" : "fetcher" } }, + { + "desc" : "BufferLinkPreviewBot", + "ua" : "BufferLinkPreviewBot/1.0 (+https://scraper.buffer.com/about/bots/link-preview-bot)", + "expect" : + { + "name" : "BufferLinkPreviewBot", + "version" : "1.0", + "type" : "fetcher" + } + }, { "desc" : "ChatGPT-User", "ua" : "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; ChatGPT-User/1.0; +https://openai.com/bot",