[extensions][helpers] Add new bots: cohere-training-data-crawler, Gemini-Deep-Research, kakaotalk-scrap, TikTokSpider

This commit is contained in:
Faisal Salman
2025-08-11 13:12:28 +07:00
parent 74ef71cf63
commit 95485f7b5d
4 changed files with 50 additions and 6 deletions

View File

@@ -299,6 +299,16 @@
"type" : "crawler"
}
},
{
"desc" : "cohere-training-data-crawler",
"ua" : "cohere-training-data-crawler (+crawler@cohere.ai)",
"expect" :
{
"name" : "cohere-training-data-crawler",
"version" : "undefined",
"type" : "crawler"
}
},
{
"desc" : "Coveobot",
"ua" : "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko) (compatible; Coveobot/2.0;+http://www.coveo.com/bot.html)",

View File

@@ -109,6 +109,16 @@
"type" : "fetcher"
}
},
{
"desc" : "Gemini-Deep-Research",
"ua" : "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Gemini-Deep-Research; +https://gemini.google/overview/deep-research/) Chrome/135.0.0.0 Safari/537.36",
"expect" :
{
"name" : "Gemini-Deep-Research",
"version" : "undefined",
"type" : "fetcher"
}
},
{
"desc" : "Google FeedFetcher",
"ua" : "FeedFetcher-Google; (+http://www.google.com/feedfetcher.html)",
@@ -189,6 +199,16 @@
"type" : "fetcher"
}
},
{
"desc" : "kakaotalk-scrap",
"ua" : "facebookexternalhit/1.1; kakaotalk-scrap/1.0; +https://devtalk.kakao.com/t/scrap/33984",
"expect" :
{
"name" : "kakaotalk-scrap",
"version" : "1.0",
"type" : "fetcher"
}
},
{
"desc" : "Meta-ExternalFetcher",
"ua" : "meta-externalfetcher/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/crawler)",
@@ -289,6 +309,16 @@
"type" : "fetcher"
}
},
{
"desc" : "TikTokSpider",
"ua" : "Mozilla/5.0 (Linux; Android 5.0) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; TikTokSpider; ttspider-feedback@tiktok.com)",
"expect" :
{
"name" : "TikTokSpider",
"version" : "undefined",
"type" : "fetcher"
}
},
{
"desc" : "UptimeRobot",
"ua" : "Mozilla/5.0 (compatible; UptimeRobot/2.0; http://www.uptimerobot.com/)",