From 3d41647a55a71bc8755978b4a6a6c15302930f89 Mon Sep 17 00:00:00 2001 From: Faisal Salman Date: Tue, 4 Mar 2025 11:14:31 +0700 Subject: [PATCH] [extensions] Update some more bots #780: `Better Uptime Bot`, `SemrushBot`, `Yahoo! Slurp` --- src/extensions/ua-parser-extensions.js | 8 +++---- test/data/ua/extension/crawler.json | 30 ++++++++++++++++++++++++++ test/data/ua/extension/fetcher.json | 10 +++++++++ 3 files changed, 44 insertions(+), 4 deletions(-) diff --git a/src/extensions/ua-parser-extensions.js b/src/extensions/ua-parser-extensions.js index 46991cc..09feadf 100644 --- a/src/extensions/ua-parser-extensions.js +++ b/src/extensions/ua-parser-extensions.js @@ -78,7 +78,7 @@ const Crawlers = Object.freeze({ /(ia_archiver|archive\.org_bot)\/?([\w\.]*)/i, // SemrushBot - http://www.semrush.com/bot.html - /((?:semrush|splitsignal)bot[-abcfimostw]*)\/([\w\.-]+)/i, + /((?:semrush|splitsignal)bot[-abcfimostw]*)\/?([\w\.-]*)/i, // Sogou Spider /(sogou (?:pic|head|web|orion|news) spider)\/([\w\.]+)/i, @@ -110,7 +110,7 @@ const Crawlers = Object.freeze({ // Qihoo 360Spider // TurnitinBot - https://www.turnitin.com/robot/crawlerinfo.html // Yahoo! Slurp - http://help.yahoo.com/help/us/ysearch/slurp - /\b(360spider-?(?:image|video)?|bytespider|(?:ai2|aspiegel|dataforseo|imagesift|petal|turnitin)bot|teoma|(?=yahoo! )slurp)/i + /\b(360spider-?(?:image|video)?|bytespider|(?:ai2|aspiegel|dataforseo|imagesift|petal|turnitin)bot|teoma|yahoo! slurp)/i ], [NAME, [TYPE, CRAWLER]] ] @@ -219,7 +219,7 @@ const Fetchers = Object.freeze({ // AhrefsSiteAudit - https://ahrefs.com/robot/site-audit // ChatGPT-User - https://platform.openai.com/docs/plugins/bot // DuckAssistBot - https://duckduckgo.com/duckassistbot/ - // BingPreview / Mastodon / Pinterestbot / Redditbot / Rogerbot / SiteAuditBot / Telegrambot / Twitterbot / UptimeRobot + // Better Uptime / BingPreview / Mastodon / Pinterestbot / Redditbot / Rogerbot / SiteAuditBot / Telegrambot / Twitterbot / UptimeRobot // Google Site Verifier / Meta / Yahoo! Japan // Yandex Bots - https://yandex.com/bots /(ahrefssiteaudit|bingpreview|chatgpt-user|mastodon|(?:discord|duckassist|linkedin|pinterest|reddit|roger|siteaudit|twitter|uptimero)bot|google-site-verification|meta-externalfetcher|y!?j-dlc|yandex(?:calendar|direct(?:dyn)?|searchshop)|yadirectfetcher)\/([\w\.]+)/i, @@ -240,7 +240,7 @@ const Fetchers = Object.freeze({ [ // Google Bots / Cohere / Snapchat / Vercelbot / Yandex Bots - /(cohere-ai|vercelbot|feedfetcher-google|google(?:imageproxy|-read-aloud|-pagerenderer|producer)|snap url preview|telegrambot|yandex(?:sitelinks|userproxy))/i + /((?:better uptime |telegram|vercel)bot|cohere-ai|feedfetcher-google|google(?:imageproxy|-read-aloud|-pagerenderer|producer)|snap url preview|yandex(?:sitelinks|userproxy))/i ], [NAME, [TYPE, FETCHER]], ] diff --git a/test/data/ua/extension/crawler.json b/test/data/ua/extension/crawler.json index 48c5865..2a4d096 100644 --- a/test/data/ua/extension/crawler.json +++ b/test/data/ua/extension/crawler.json @@ -480,6 +480,26 @@ "type" : "crawler" } }, + { + "desc" : "SemrushBot for Backlink Audit tool", + "ua" : "Mozilla/5.0 (compatible; SemrushBot-BA; +http://www.semrush.com/bot.html)", + "expect" : + { + "name" : "SemrushBot-BA", + "version" : "undefined", + "type" : "crawler" + } + }, + { + "desc" : "SemrushBot for On Page SEO Checker tool", + "ua" : "Mozilla/5.0 (compatible; SemrushBot-SI/0.97; +http://www.semrush.com/bot.html)", + "expect" : + { + "name" : "SemrushBot-SI", + "version" : "0.97", + "type" : "crawler" + } + }, { "desc" : "SemrushBot for ContentShake AI tool", "ua" : "Mozilla/5.0 (compatible; SemrushBot-OCOB/1; +https://www.semrush.com/bot/)", @@ -550,6 +570,16 @@ "type" : "crawler" } }, + { + "desc" : "Yahoo! Slurp", + "ua" : "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp) ", + "expect" : + { + "name" : "Yahoo! Slurp", + "version" : "undefined", + "type" : "crawler" + } + }, { "desc" : "YandexBot", "ua" : "Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)", diff --git a/test/data/ua/extension/fetcher.json b/test/data/ua/extension/fetcher.json index 41d819e..b385428 100644 --- a/test/data/ua/extension/fetcher.json +++ b/test/data/ua/extension/fetcher.json @@ -9,6 +9,16 @@ "type" : "fetcher" } }, + { + "desc" : "Better Uptime Bot", + "ua" : "Better Uptime Bot Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36", + "expect" : + { + "name" : "Better Uptime Bot", + "version" : "undefined", + "type" : "fetcher" + } + }, { "desc" : "BingPreview", "ua" : "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/534+ (KHTML, like Gecko) BingPreview/1.0b",