[extensions] Improve bot detection for ByteDance, Google, SB Intuitions, Webzio

This commit is contained in:
Faisal Salman
2025-08-31 20:04:49 +07:00
parent ce242a362f
commit 146f182533
4 changed files with 81 additions and 10 deletions

View File

@@ -579,6 +579,16 @@
"type" : "crawler"
}
},
{
"desc" : "APIs-Google",
"ua" : "APIs-Google (+https://developers.google.com/webmasters/APIs-Google.html)",
"expect" :
{
"name" : "APIs-Google",
"version" : "undefined",
"type" : "crawler"
}
},
{
"desc" : "Googlebot-Video",
"ua" : "Googlebot-Video/1.0",
@@ -679,6 +689,16 @@
"type" : "crawler"
}
},
{
"desc" : "Google-CloudVertexBot",
"ua" : "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.7204.183 Mobile Safari/537.36 (compatible; Google-CloudVertexBot; +https://cloud.google.com/enterprise-search)",
"expect" :
{
"name" : "Google-CloudVertexBot",
"version" : "undefined",
"type" : "crawler"
}
},
{
"desc" : "Google-Safety",
"ua" : "Google-Safety",
@@ -970,6 +990,16 @@
"type" : "crawler"
}
},
{
"desc" : "SBIntuitionsBot",
"ua" : "Mozilla/5.0 (compatible; SBIntuitionsBot/0.1;+https://www.sbintuitions.co.jp/bot/)",
"expect" :
{
"name" : "SBIntuitionsBot",
"version" : "0.1",
"type" : "crawler"
}
},
{
"desc" : "SeekportBot",
"ua" : "Mozilla/5.0 (compatible; SeekportBot; +https://bot.seekport.com)",
@@ -1080,6 +1110,16 @@
"type" : "crawler"
}
},
{
"desc" : "TikTokSpider",
"ua" : "Mozilla/5.0 (Linux; Android 5.0) AppleWebKit/537.36 (KHTML, like Gecko) Mobile Safari/537.36 (compatible; TikTokSpider; ttspider-feedback@tiktok.com)",
"expect" :
{
"name" : "TikTokSpider",
"version" : "undefined",
"type" : "crawler"
}
},
{
"desc" : "Timpibot",
"ua" : "Timpibot/0.8 (+http://www.timpi.io)",
@@ -1150,6 +1190,26 @@
"type" : "crawler"
}
},
{
"desc" : "webzio",
"ua" : "webzio (+https://webz.io/bot.html)",
"expect" :
{
"name" : "webzio",
"version" : "undefined",
"type" : "crawler"
}
},
{
"desc" : "Webzio-Extended",
"ua" : "Mozilla/5.0 (compatible; Webzio-Extended/1.0; +https://www.webzio.com/bot.html)",
"expect" :
{
"name" : "Webzio-Extended",
"version" : "1.0",
"type" : "crawler"
}
},
{
"desc" : "Yahoo! Japan",
"ua" : "Y!J-BRW/1.0 (https://www.yahoo-help.jp/app/answers/detail/p/595/a_id/42716)",