[extensions][helpers] Update Semrush bot variants

This commit is contained in:
Faisal Salman
2025-02-20 20:34:45 +07:00
parent 6f0191c11b
commit 837d319630
6 changed files with 33 additions and 10 deletions

View File

@@ -52,7 +52,6 @@ const Crawlers = Object.freeze({
// MojeekBot - https://www.mojeek.com/bot.html
// OpenAI's SearchGPT - https://platform.openai.com/docs/bots
// PerplexityBot - https://perplexity.ai/perplexitybot
// SemrushBot - http://www.semrush.com/bot.html
// SeznamBot - http://napoveda.seznam.cz/seznambot-intro
/((?:ahrefs|amazon|bing|cc|dot|duckduck|exa|facebook|gpt|mj12|mojeek|oai-search|perplexity|semrush|seznam)bot)\/([\w\.-]+)/i,
@@ -78,6 +77,9 @@ const Crawlers = Object.freeze({
// Internet Archive (archive.org)
/(ia_archiver|archive\.org_bot)\/?([\w\.]*)/i,
// SemrushBot - http://www.semrush.com/bot.html
/((?:semrush|splitsignal)bot[-abcfimostw]*)\/([\w\.-]+)/i,
// Sogou Spider
/(sogou (?:pic|head|web|orion|news) spider)\/([\w\.]+)/i,
@@ -217,10 +219,10 @@ const Fetchers = Object.freeze({
// AhrefsSiteAudit - https://ahrefs.com/robot/site-audit
// ChatGPT-User - https://platform.openai.com/docs/plugins/bot
// DuckAssistBot - https://duckduckgo.com/duckassistbot/
// BingPreview / Mastodon / Pinterestbot / Redditbot / Rogerbot / Telegrambot / Twitterbot / UptimeRobot
// BingPreview / Mastodon / Pinterestbot / Redditbot / Rogerbot / SiteAuditBot / Telegrambot / Twitterbot / UptimeRobot
// Google Site Verifier / Meta / Yahoo! Japan
// Yandex Bots - https://yandex.com/bots
/(ahrefssiteaudit|bingpreview|chatgpt-user|mastodon|(?:discord|duckassist|linkedin|pinterest|reddit|roger|telegram|twitter|uptimero)bot|google-site-verification|meta-externalfetcher|y!?j-dlc|yandex(?:calendar|direct(?:dyn)?|searchshop)|yadirectfetcher)\/([\w\.]+)/i,
/(ahrefssiteaudit|bingpreview|chatgpt-user|mastodon|(?:discord|duckassist|linkedin|pinterest|reddit|roger|siteaudit|telegram|twitter|uptimero)bot|google-site-verification|meta-externalfetcher|y!?j-dlc|yandex(?:calendar|direct(?:dyn)?|searchshop)|yadirectfetcher)\/([\w\.]+)/i,
// Bluesky
/(bluesky) cardyb\/([\w\.]+)/i,