[extensions] Add new fetcher: Discordbot, KeybaseBot, Slackbot, Slackbot-LinkExpanding, Slack-ImgProxy, Twitterbot

This commit is contained in:
Faisal Salman
2025-10-01 11:25:26 +07:00
parent e7e7aaad4c
commit bd6bb216bd
3 changed files with 69 additions and 3 deletions

View File

@@ -628,6 +628,7 @@ const Extension = Object.freeze({
BLUESKY: 'Bluesky',
BUFFER_LINKPREVIEWBOT: 'BufferLinkPreviewBot',
COHERE_AI: 'Cohere-AI',
DISCORD_BOT: 'Discordbot',
DUCKDUCKGO_ASSISTBOT: 'DuckAssistBot',
GOOGLE_CHROME_LIGHTHOUSE: 'Chrome-Lighthouse',
GOOGLE_FEEDFETCHER: 'FeedFetcher-Google',
@@ -640,6 +641,7 @@ const Extension = Object.freeze({
HUBSPOT_PAGE_FETCHER: 'HubSpot Page Fetcher',
IFRAMELY: 'Iframely',
KAKAOTALK_SCRAP: 'kakaotalk-scrap',
KEYBASE_BOT: 'KeybaseBot',
META_EXTERNALFETCHER: 'meta-externalfetcher',
META_WHATSAPP: 'WhatsApp',
MICROSOFT_BINGPREVIEW: 'BingPreview',
@@ -651,6 +653,9 @@ const Extension = Object.freeze({
PERPLEXITY_USER: 'Perplexity-User',
PINTEREST_BOT: 'Pinterestbot',
SEMRUSH_SITEAUDITBOT: 'SiteAuditBot',
SLACK_BOT: 'Slackbot',
SLACK_BOT_LINKEXPANDING: 'Slackbot-LinkExpanding',
SLACK_IMGPROXY: 'Slack-ImgProxy',
SNAP_URL_PREVIEW: 'Snap URL Preview',
SKYPE_URIPREVIEW: 'SkypeUriPreview',
TELEGRAM_BOT: 'TelegramBot',
@@ -660,6 +665,7 @@ const Extension = Object.freeze({
VERCEL_BOT: 'Vercelbot',
VERCEL_FLAGS: 'vercelflags',
VERCEL_TRACING: 'verceltracing',
X_TWITTERBOT: 'Twitterbot',
YANDEX_CALENDAR: 'YandexCalendar',
YANDEX_DIRECT: 'YandexDirect',
YANDEX_DIRECTDYN: 'YandexDirectDyn',

View File

@@ -283,8 +283,8 @@ const Fetchers = Object.freeze({
[NAME, VERSION, [TYPE, FETCHER]],
[
// Google Bots / Chrome-Lighthouse / Gemini-Deep-Research / Snapchat / Vercelbot / Yandex Bots
/((?:better uptime |telegram|vercel)bot|chrome-lighthouse|feedfetcher-google|gemini-deep-research|google(?:imageproxy|-read-aloud|-pagerenderer|producer)|snap url preview|vercel(flags|tracing|-(favicon|screenshot)-bot)|yandex(?:sitelinks|userproxy))/i
// Google Bots / Chrome-Lighthouse / Gemini-Deep-Research / KeybaseBot / Snapchat / Vercelbot / Yandex Bots
/((?:better uptime |keybase|telegram|vercel)bot|chrome-lighthouse|feedfetcher-google|gemini-deep-research|google(?:imageproxy|-read-aloud|-pagerenderer|producer)|snap url preview|vercel(flags|tracing|-(favicon|screenshot)-bot)|yandex(?:sitelinks|userproxy))/i
],
[NAME, [TYPE, FETCHER]],
],

View File

@@ -51,7 +51,7 @@
},
{
"desc" : "Blueno",
"ua" : "acebookexternalhit/1.1 (compatible; Blueno/1.0; +http://naver.me/scrap)",
"ua" : "facebookexternalhit/1.1 (compatible; Blueno/1.0; +http://naver.me/scrap)",
"expect" :
{
"name" : "Blueno",
@@ -119,6 +119,16 @@
"type" : "fetcher"
}
},
{
"desc" : "Discordbot",
"ua" : "Mozilla/5.0 (compatible; Discordbot/2.0; +https://discordapp.com)",
"expect" :
{
"name" : "Discordbot",
"version" : "2.0",
"type" : "fetcher"
}
},
{
"desc" : "DuckAssistBot",
"ua" : "DuckAssistBot/1.2; (+http://duckduckgo.com/duckassistbot.html)",
@@ -239,6 +249,16 @@
"type" : "fetcher"
}
},
{
"desc" : "KeybaseBot",
"ua" : "Mozilla/5.0 (compatible; KeybaseBot; +https://keybase.io)",
"expect" :
{
"name" : "KeybaseBot",
"version" : "undefined",
"type" : "fetcher"
}
},
{
"desc" : "Meta-ExternalFetcher",
"ua" : "meta-externalfetcher/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/crawler)",
@@ -309,6 +329,36 @@
"type" : "fetcher"
}
},
{
"desc" : "Slack-ImgProxy",
"ua" : "Slack-ImgProxy 0.19 (+https://api.slack.com/robots)",
"expect" :
{
"name" : "Slack-ImgProxy",
"version" : "0.19",
"type" : "fetcher"
}
},
{
"desc" : "Slackbot",
"ua" : "Slackbot 1.0 (+https://api.slack.com/robots)",
"expect" :
{
"name" : "Slackbot",
"version" : "1.0",
"type" : "fetcher"
}
},
{
"desc" : "Slackbot-LinkExpanding",
"ua" : "Slackbot-LinkExpanding 1.0 (+https://api.slack.com/robots)",
"expect" :
{
"name" : "Slackbot-LinkExpanding",
"version" : "1.0",
"type" : "fetcher"
}
},
{
"desc" : "Snap URL Preview",
"ua" : "Snap URL Preview Service; bot; snapchat; https://developers.snap.com/robots ",
@@ -339,6 +389,16 @@
"type" : "fetcher"
}
},
{
"desc" : "Twitterbot",
"ua" : "Twitterbot/1.0",
"expect" :
{
"name" : "Twitterbot",
"version" : "1.0",
"type" : "fetcher"
}
},
{
"desc" : "UptimeRobot",
"ua" : "Mozilla/5.0 (compatible; UptimeRobot/2.0; http://www.uptimerobot.com/)",