[extensions] Add new bots: Algolia Crawler, contxbot, HubSpot Page Fetcher, Kagibot

This commit is contained in:
Faisal Salman
2025-08-15 20:42:41 +07:00
parent 975c4860f4
commit 9003fe5724
3 changed files with 47 additions and 5 deletions

View File

@@ -79,6 +79,16 @@
"type" : "crawler"
}
},
{
"desc" : "Algolia Crawler",
"ua" : "Algolia Crawler/v2.183.0",
"expect" :
{
"name" : "Algolia Crawler",
"version" : "v2.183.0",
"type" : "crawler"
}
},
{
"desc" : "Applebot",
"ua" : "Mozilla/5.0 (iPhone; CPU iPhone OS 8_1 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12B410 Safari/600.1.4 (Applebot/0.1;+http://www.apple.com/go/applebot)",
@@ -319,6 +329,16 @@
"type" : "crawler"
}
},
{
"desc" : "contxbot",
"ua" : "Mozilla/5.0 (compatible;contxbot/1.0)",
"expect" :
{
"name" : "contxbot",
"version" : "1.0",
"type" : "crawler"
}
},
{
"desc" : "Cotoyogi",
"ua" : "Mozilla/5.0 (compatible; Cotoyogi/4.0; +https://ds.rois.ac.jp/center8/crawler/)",
@@ -649,6 +669,16 @@
"type" : "crawler"
}
},
{
"desc" : "Kagibot",
"ua" : "Mozilla/5.0 (compatible; Kagibot/1.0; +https://kagi.com/bot)",
"expect" :
{
"name" : "Kagibot",
"version" : "1.0",
"type" : "crawler"
}
},
{
"desc" : "Kangaroo Bot",
"ua" : "Mozilla/5.0 (compatible; Kangaroo Bot/1.0)",

View File

@@ -189,6 +189,16 @@
"type" : "fetcher"
}
},
{
"desc" : "HubSpot Page Fetcher",
"ua" : "HubSpot Page Fetcher/1.0 http://www.hubspot.com/ web-crawlers@hubspot.com",
"expect" :
{
"name" : "HubSpot Page Fetcher",
"version" : "1.0",
"type" : "fetcher"
}
},
{
"desc" : "Iframely",
"ua" : "Iframely/1.3.1 (+https://iframely.com/docs/about)",