[extensions][enums] Improve detection for Yandex bots

This commit is contained in:
Faisal Salman
2025-08-30 17:01:05 +07:00
parent 2078b1ec92
commit ce242a362f
4 changed files with 415 additions and 3 deletions

View File

@@ -1170,6 +1170,46 @@
"type" : "crawler"
}
},
{
"desc" : "YandexAccessibilityBot",
"ua" : "Mozilla/5.0 (compatible; YandexAccessibilityBot/3.0; +http://yandex.com/bots)",
"expect" :
{
"name" : "YandexAccessibilityBot",
"version" : "3.0",
"type" : "crawler"
}
},
{
"desc" : "YandexAdditionalBot",
"ua" : "Mozilla/5.0 (compatible; YandexAdditionalBot/3.0; +http://yandex.com/bots)",
"expect" :
{
"name" : "YandexAdditionalBot",
"version" : "3.0",
"type" : "crawler"
}
},
{
"desc" : "YandexAdNet",
"ua" : "Mozilla/5.0 (compatible; YandexAdNet/1.0; +http://yandex.com/bots)",
"expect" :
{
"name" : "YandexAdNet",
"version" : "1.0",
"type" : "crawler"
}
},
{
"desc" : "YandexBlogs",
"ua" : "Mozilla/5.0 (compatible; YandexBlogs/0.99; robot; +http://yandex.com/bots)",
"expect" :
{
"name" : "YandexBlogs",
"version" : "0.99",
"type" : "crawler"
}
},
{
"desc" : "YandexBot",
"ua" : "Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)",
@@ -1180,6 +1220,246 @@
"type" : "crawler"
}
},
{
"desc" : "YandexBot MirrorDetector",
"ua" : "Mozilla/5.0 (compatible; YandexBot/3.0; MirrorDetector; +http://yandex.com/bots)",
"expect" :
{
"name" : "YandexBot MirrorDetector",
"version" : "3.0",
"type" : "crawler"
}
},
{
"desc" : "YandexComBot",
"ua" : "Mozilla/5.0 (compatible; YandexComBot/3.0; +http://yandex.com/bots)",
"expect" :
{
"name" : "YandexComBot",
"version" : "3.0",
"type" : "crawler"
}
},
{
"desc" : "YandexFavicons",
"ua" : "Mozilla/5.0 (compatible; YandexFavicons/1.0; +http://yandex.com/bots)",
"expect" :
{
"name" : "YandexFavicons",
"version" : "1.0",
"type" : "crawler"
}
},
{
"desc" : "YandexImageResizer",
"ua" : "Mozilla/5.0 (compatible; YandexImageResizer/2.0; +http://yandex.com/bots)",
"expect" :
{
"name" : "YandexImageResizer",
"version" : "2.0",
"type" : "crawler"
}
},
{
"desc" : "YandexImages",
"ua" : "Mozilla/5.0 (compatible; YandexImages/3.0; +http://yandex.com/bots)",
"expect" :
{
"name" : "YandexImages",
"version" : "3.0",
"type" : "crawler"
}
},
{
"desc" : "YandexMarket",
"ua" : "Mozilla/5.0 (compatible; YandexMarket/1.0; +http://yandex.com/bots)",
"expect" :
{
"name" : "YandexMarket",
"version" : "1.0",
"type" : "crawler"
}
},
{
"desc" : "YandexMetrika",
"ua" : "Mozilla/5.0 (compatible; YandexMetrika/2.0; +http://yandex.com/bots)",
"expect" :
{
"name" : "YandexMetrika",
"version" : "2.0",
"type" : "crawler"
}
},
{
"desc" : "YandexMedia",
"ua" : "Mozilla/5.0 (compatible; YandexMedia/3.0; +http://yandex.com/bots)",
"expect" :
{
"name" : "YandexMedia",
"version" : "3.0",
"type" : "crawler"
}
},
{
"desc" : "YandexMobileBot",
"ua" : "Mozilla/5.0 (iPhone; CPU iPhone OS 8_1 like Mac OS X) AppleWebKit/600.1.4 (KHTML, like Gecko) Version/8.0 Mobile/12B411 Safari/600.1.4 (compatible; YandexMobileBot/3.0; +http://yandex.com/bots)",
"expect" :
{
"name" : "YandexMobileBot",
"version" : "3.0",
"type" : "crawler"
}
},
{
"desc" : "YandexMobileScreenShotBot",
"ua" : "Mozilla/5.0 (compatible; YandexMobileScreenShotBot/1.0; +http://yandex.com/bots)",
"expect" :
{
"name" : "YandexMobileScreenShotBot",
"version" : "1.0",
"type" : "crawler"
}
},
{
"desc" : "YandexNews",
"ua" : "Mozilla/5.0 (compatible; YandexNews/4.0; +http://yandex.com/bots)",
"expect" :
{
"name" : "YandexNews",
"version" : "4.0",
"type" : "crawler"
}
},
{
"desc" : "YandexOntoDB",
"ua" : "Mozilla/5.0 (compatible; YandexOntoDB/1.0; +http://yandex.com/bots)",
"expect" :
{
"name" : "YandexOntoDB",
"version" : "1.0",
"type" : "crawler"
}
},
{
"desc" : "YandexOntoDBAPI",
"ua" : "Mozilla/5.0 (compatible; YandexOntoDBAPI/1.0; +http://yandex.com/bots)",
"expect" :
{
"name" : "YandexOntoDBAPI",
"version" : "1.0",
"type" : "crawler"
}
},
{
"desc" : "YandexPartner",
"ua" : "Mozilla/5.0 (compatible; YandexPartner/3.0; +http://yandex.com/bots)",
"expect" :
{
"name" : "YandexPartner",
"version" : "3.0",
"type" : "crawler"
}
},
{
"desc" : "YandexRCA",
"ua" : "Mozilla/5.0 (compatible; YandexRCA/1.0; +http://yandex.com/bots)",
"expect" :
{
"name" : "YandexRCA",
"version" : "1.0",
"type" : "crawler"
}
},
{
"desc" : "YandexRenderResourcesBot",
"ua" : "Mozilla/5.0 (compatible; YandexRenderResourcesBot/1.0; +http://yandex.com/bots)",
"expect" :
{
"name" : "YandexRenderResourcesBot",
"version" : "1.0",
"type" : "crawler"
}
},
{
"desc" : "YandexScreenshotBot",
"ua" : "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.96 Safari/537.36 (compatible; YandexScreenshotBot/3.0; +http://yandex.com/bots)",
"expect" :
{
"name" : "YandexScreenshotBot",
"version" : "3.0",
"type" : "crawler"
}
},
{
"desc" : "YandexSpravBot",
"ua" : "Mozilla/5.0 (compatible; YandexSpravBot/1.0; +http://yandex.com/bots)",
"expect" :
{
"name" : "YandexSpravBot",
"version" : "1.0",
"type" : "crawler"
}
},
{
"desc" : "YandexTracker",
"ua" : "Mozilla/5.0 (compatible; YandexTracker/1.0; +http://yandex.com/bots)",
"expect" :
{
"name" : "YandexTracker",
"version" : "1.0",
"type" : "crawler"
}
},
{
"desc" : "YandexVertis",
"ua" : "Mozilla/5.0 (compatible; YandexVertis/3.0; +http://yandex.com/bots)",
"expect" :
{
"name" : "YandexVertis",
"version" : "3.0",
"type" : "crawler"
}
},
{
"desc" : "YandexVerticals",
"ua" : "Mozilla/5.0 (compatible; YandexVerticals/1.0; +http://yandex.com/bots)",
"expect" :
{
"name" : "YandexVerticals",
"version" : "1.0",
"type" : "crawler"
}
},
{
"desc" : "YandexVideo",
"ua" : "Mozilla/5.0 (compatible; YandexVideo/3.0; +http://yandex.com/bots)",
"expect" :
{
"name" : "YandexVideo",
"version" : "3.0",
"type" : "crawler"
}
},
{
"desc" : "YandexVideoParser",
"ua" : "Mozilla/5.0 (compatible; YandexVideoParser/1.0; +http://yandex.com/bots)",
"expect" :
{
"name" : "YandexVideoParser",
"version" : "1.0",
"type" : "crawler"
}
},
{
"desc" : "YandexWebmaster",
"ua" : "Mozilla/5.0 (compatible; YandexWebmaster/2.0; +http://yandex.com/bots)",
"expect" :
{
"name" : "YandexWebmaster",
"version" : "2.0",
"type" : "crawler"
}
},
{
"desc" : "YepBot",
"ua" : "Mozilla/5.0 (compatible; YepBot/1.0; +http://yep.com/yepbot/)",

View File

@@ -419,6 +419,96 @@
"type" : "fetcher"
}
},
{
"desc" : "YaDirectFetcher",
"ua" : "Mozilla/5.0 (compatible; YaDirectFetcher/1.0; Dyatel; +http://yandex.com/bots)",
"expect" :
{
"name" : "YaDirectFetcher",
"version" : "1.0",
"type" : "fetcher"
}
},
{
"desc" : "YandexCalendar",
"ua" : "Mozilla/5.0 (compatible; YandexCalendar/1.0; +http://yandex.com/bots)",
"expect" :
{
"name" : "YandexCalendar",
"version" : "1.0",
"type" : "fetcher"
}
},
{
"desc" : "YandexDirect",
"ua" : "Mozilla/5.0 (compatible; YandexDirect/3.0; +http://yandex.com/bots)",
"expect" :
{
"name" : "YandexDirect",
"version" : "3.0",
"type" : "fetcher"
}
},
{
"desc" : "YandexDirectDyn",
"ua" : "Mozilla/5.0 (compatible; YandexDirectDyn/1.0; +http://yandex.com/bots",
"expect" :
{
"name" : "YandexDirectDyn",
"version" : "1.0",
"type" : "fetcher"
}
},
{
"desc" : "YandexForDomain",
"ua" : "Mozilla/5.0 (compatible; YandexForDomain/1.0; +http://yandex.com/bots)",
"expect" :
{
"name" : "YandexForDomain",
"version" : "1.0",
"type" : "fetcher"
}
},
{
"desc" : "YandexPagechecker",
"ua" : "Mozilla/5.0 (compatible; YandexPagechecker/1.0; +http://yandex.com/bots)",
"expect" :
{
"name" : "YandexPagechecker",
"version" : "1.0",
"type" : "fetcher"
}
},
{
"desc" : "YandexSearchShop",
"ua" : "Mozilla/5.0 (compatible; YandexSearchShop/1.0; +http://yandex.com/bots)",
"expect" :
{
"name" : "YandexSearchShop",
"version" : "1.0",
"type" : "fetcher"
}
},
{
"desc" : "YandexSitelinks",
"ua" : "Mozilla/5.0 (compatible; YandexSitelinks; Dyatel; +http://yandex.com/bots)",
"expect" :
{
"name" : "YandexSitelinks",
"version" : "undefined",
"type" : "fetcher"
}
},
{
"desc" : "YandexUserproxy",
"ua" : "Mozilla/5.0 (compatible; YandexUserproxy; robot; +http://yandex.com/bots)",
"expect" :
{
"name" : "YandexUserproxy",
"version" : "undefined",
"type" : "fetcher"
}
},
{
"desc" : "Zoombot",
"ua" : "Mozilla/5.0 (compatible; Zoombot/1.0; +https://zoom.us; crawler@domain.com)",