[extensions] Add new crawler bots: ChatGLM, Onespot, Startpage

This commit is contained in:
Faisal Salman
2025-06-03 11:03:45 +07:00
parent a3549efc22
commit 72d0c2acb3
3 changed files with 37 additions and 3 deletions

View File

@@ -259,6 +259,16 @@
"type" : "crawler"
}
},
{
"desc" : "ChatGLM-Spider",
"ua" : "Mozilla/5.0 (compatible; ChatGLM-Spider/1.0; +https://chatglm.cn/)",
"expect" :
{
"name" : "ChatGLM-Spider",
"version" : "1.0",
"type" : "crawler"
}
},
{
"desc" : "Coc Coc Bot (web)",
"ua" : "Mozilla/5.0 (compatible; coccocbot-web/1.0; +http://help.coccoc.com/searchengine)",
@@ -620,6 +630,16 @@
"type" : "crawler"
}
},
{
"desc" : "Onespot",
"ua" : "Mozilla/5.0 (compatible; Onespot-ScraperBot/1.0; +https://www.onespot.com/identifying-traffic.html)",
"expect" :
{
"name" : "Onespot-ScraperBot",
"version" : "1.0",
"type" : "crawler"
}
},
{
"desc" : "OpenAI Search",
"ua" : "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; OAI-SearchBot/1.0; +https://openai.com/searchbot",
@@ -750,6 +770,16 @@
"type" : "crawler"
}
},
{
"desc" : "Startpage",
"ua" : "StartpagePrivateImageProxy/3.0 (https://www.startpage.com/; support@startpage.com) aiohttp.client/3.11.11",
"expect" :
{
"name" : "StartpagePrivateImageProxy",
"version" : "3.0",
"type" : "crawler"
}
},
{
"desc" : "Teoma",
"ua" : "Mozilla/2.0 (compatible; Ask Jeeves/Teoma; +http://sp.ask.com/docs/about/tech_crawling.html)",