Merge branch 'master' into master

This commit is contained in:
Suryaansh Chawla 2025-06-04 10:51:27 +05:30 committed by GitHub
commit 935f8e51e8
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
9 changed files with 178 additions and 8 deletions

View File

@ -38,6 +38,8 @@ export const Browser: Readonly<{
DUCKDUCKGO: "DuckDuckGo"; DUCKDUCKGO: "DuckDuckGo";
ECOSIA: "Ecosia"; ECOSIA: "Ecosia";
EDGE: "Edge"; EDGE: "Edge";
EDGE_WEBVIEW: "Edge WebView";
EDGE_WEBVIEW2: "Edge WebView2";
EPIPHANY: "Epiphany"; EPIPHANY: "Epiphany";
FACEBOOK: "Facebook"; FACEBOOK: "Facebook";
FALKON: "Falkon"; FALKON: "Falkon";

View File

@ -43,6 +43,8 @@ const Browser = Object.freeze({
DUCKDUCKGO: 'DuckDuckGo', DUCKDUCKGO: 'DuckDuckGo',
ECOSIA: 'Ecosia', ECOSIA: 'Ecosia',
EDGE: 'Edge', EDGE: 'Edge',
EDGE_WEBVIEW: 'Edge WebView',
EDGE_WEBVIEW2: 'Edge WebView2',
EPIPHANY: 'Epiphany', EPIPHANY: 'Epiphany',
FACEBOOK: 'Facebook', FACEBOOK: 'Facebook',
FALKON: 'Falkon', FALKON: 'Falkon',

View File

@ -48,13 +48,15 @@ const Crawlers = Object.freeze({
// DuckDuckBot - http://duckduckgo.com/duckduckbot.html // DuckDuckBot - http://duckduckgo.com/duckduckbot.html
// FacebookBot - https://developers.facebook.com/docs/sharing/bot/ // FacebookBot - https://developers.facebook.com/docs/sharing/bot/
// GPTBot - https://platform.openai.com/docs/gptbot // GPTBot - https://platform.openai.com/docs/gptbot
// iAskBot - https://iask.ai
// LinkedInBot - http://www.linkedin.com // LinkedInBot - http://www.linkedin.com
// MJ12bot - https://mj12bot.com/ // MJ12bot - https://mj12bot.com/
// MojeekBot - https://www.mojeek.com/bot.html // MojeekBot - https://www.mojeek.com/bot.html
// Onespot - https://www.onespot.com/identifying-traffic.html
// OpenAI's SearchGPT - https://platform.openai.com/docs/bots // OpenAI's SearchGPT - https://platform.openai.com/docs/bots
// PerplexityBot - https://perplexity.ai/perplexitybot // PerplexityBot - https://perplexity.ai/perplexitybot
// SeznamBot - http://napoveda.seznam.cz/seznambot-intro // SeznamBot - http://napoveda.seznam.cz/seznambot-intro
/((?:adidx|ahrefs|amazon|bing|cc|dot|duckduck|exa|facebook|gpt|linkedin|mj12|mojeek|oai-search|perplexity|semrush|seznam)bot)\/([\w\.-]+)/i, /((?:adidx|ahrefs|amazon|bing|cc|dot|duckduck|exa|facebook|gpt|iask|linkedin|mj12|mojeek|oai-search|onespot-scraper|perplexity|semrush|seznam)bot)\/([\w\.-]+)/i,
// Applebot - http://apple.com/go/applebot // Applebot - http://apple.com/go/applebot
/(applebot(?:-extended)?)\/?([\w\.]*)/i, /(applebot(?:-extended)?)\/?([\w\.]*)/i,
@ -68,6 +70,9 @@ const Crawlers = Object.freeze({
// Coc Coc Bot - https://help.coccoc.com/en/search-engine // Coc Coc Bot - https://help.coccoc.com/en/search-engine
/(coccocbot-(?:image|web))\/([\w\.]+)/i, /(coccocbot-(?:image|web))\/([\w\.]+)/i,
// Daum
/(daum(?:oa)?(?:-image)?)[ \/]([\w\.]+)/i,
// Facebook / Meta // Facebook / Meta
// https://developers.facebook.com/docs/sharing/webmasters/web-crawlers // https://developers.facebook.com/docs/sharing/webmasters/web-crawlers
/(facebook(?:externalhit|catalog)|meta-externalagent)\/([\w\.]+)/i, /(facebook(?:externalhit|catalog)|meta-externalagent)\/([\w\.]+)/i,
@ -78,6 +83,9 @@ const Crawlers = Object.freeze({
// Internet Archive (archive.org) // Internet Archive (archive.org)
/(ia_archiver|archive\.org_bot)\/?([\w\.]*)/i, /(ia_archiver|archive\.org_bot)\/?([\w\.]*)/i,
// Qwantbot - https://help.qwant.com/bot
/(qwantbot)[-\w]*\/?([\w\.]*)/i,
// SemrushBot - http://www.semrush.com/bot.html // SemrushBot - http://www.semrush.com/bot.html
/((?:semrush|splitsignal)bot[-abcfimostw]*)\/?([\w\.-]*)/i, /((?:semrush|splitsignal)bot[-abcfimostw]*)\/?([\w\.-]*)/i,
@ -93,8 +101,8 @@ const Crawlers = Object.freeze({
// Yeti (Naver) // Yeti (Naver)
/(yeti)\/([\w\.]+)/i, /(yeti)\/([\w\.]+)/i,
// aiHitBot / Diffbot / Linespider / Magpie-Crawler / Omgilibot / OpenAI Image Downloader / Webzio-Extended / Screaming Frog SEO Spider / Timpibot / VelenPublicWebCrawler / YisouSpider / YouBot // aiHitBot / Diffbot / Linespider / Magpie-Crawler / Omgilibot / OpenAI Image Downloader / Webzio-Extended / Screaming Frog SEO Spider / Startpage / Timpibot / VelenPublicWebCrawler / YisouSpider / YouBot
/((?:aihit|diff|timpi|you)bot|omgili(?:bot)?|openai image downloader|(?:magpie-|velenpublicweb)crawler|webzio-extended|(?:screaming frog seo |line|yisou)spider)\/?([\w\.]*)/i /((?:aihit|diff|timpi|you)bot|omgili(?:bot)?|openai image downloader|(?:magpie-|velenpublicweb)crawler|startpageprivateimageproxy|webzio-extended|(?:chatglm-|line|screaming frog seo |yisou)spider)\/?([\w\.]*)/i
], ],
[NAME, VERSION, [TYPE, CRAWLER]], [NAME, VERSION, [TYPE, CRAWLER]],
@ -222,10 +230,11 @@ const Fetchers = Object.freeze({
// DuckAssistBot - https://duckduckgo.com/duckassistbot/ // DuckAssistBot - https://duckduckgo.com/duckassistbot/
// Better Uptime / BingPreview / Mastodon / MicrosoftPreview / Pinterestbot / Redditbot / Rogerbot / SiteAuditBot / Telegrambot / Twitterbot / UptimeRobot // Better Uptime / BingPreview / Mastodon / MicrosoftPreview / Pinterestbot / Redditbot / Rogerbot / SiteAuditBot / Telegrambot / Twitterbot / UptimeRobot
// Google Site Verifier / Meta / Yahoo! Japan // Google Site Verifier / Meta / Yahoo! Japan
// Iframely - https://iframely.com/docs/about
// Perplexity-User - https://docs.perplexity.ai/guides/bots // Perplexity-User - https://docs.perplexity.ai/guides/bots
// MistralAI-User - https://docs.mistral.ai/robots/ // MistralAI-User - https://docs.mistral.ai/robots/
// Yandex Bots - https://yandex.com/bots // Yandex Bots - https://yandex.com/bots
/(ahrefssiteaudit|(?:bing|microsoft)preview|(?:chatgpt|mistralai|perplexity)-user|mastodon|(?:discord|duckassist|linkedin|pinterest|reddit|roger|siteaudit|twitter|uptimero)bot|google-site-verification|meta-externalfetcher|y!?j-dlc|yandex(?:calendar|direct(?:dyn)?|searchshop)|yadirectfetcher)\/([\w\.]+)/i, /(ahrefssiteaudit|(?:bing|microsoft)preview|(?:chatgpt|mistralai|perplexity)-user|mastodon|(?:discord|duckassist|linkedin|pinterest|reddit|roger|siteaudit|twitter|uptimero)bot|google-site-verification|iframely|meta-externalfetcher|y!?j-dlc|yandex(?:calendar|direct(?:dyn)?|searchshop)|yadirectfetcher)\/([\w\.]+)/i,
// Bluesky // Bluesky
/(bluesky) cardyb\/([\w\.]+)/i, /(bluesky) cardyb\/([\w\.]+)/i,

View File

@ -110,6 +110,9 @@ const isAIBot = (resultOrUA) => [
// You.com // You.com
'youbot', 'youbot',
// Zhipu AI
'chatglm-spider',
// Zyte // Zyte
'scrapy' 'scrapy'

View File

@ -152,11 +152,11 @@
has = function (str1, str2) { has = function (str1, str2) {
if (typeof str1 === OBJ_TYPE && str1.length > 0) { if (typeof str1 === OBJ_TYPE && str1.length > 0) {
for (var i in str1) { for (var i in str1) {
if (lowerize(str1[i]) == lowerize(str2)) return true; if (lowerize(str2) == lowerize(str1[i])) return true;
} }
return false; return false;
} }
return isString(str1) ? lowerize(str2).indexOf(lowerize(str1)) !== -1 : false; return isString(str1) ? lowerize(str2) == lowerize(str1) : false;
}, },
isExtensions = function (obj, deep) { isExtensions = function (obj, deep) {
for (var prop in obj) { for (var prop in obj) {
@ -326,7 +326,9 @@
// Most common regardless engine // Most common regardless engine
/\b(?:crmo|crios)\/([\w\.]+)/i // Chrome for Android/iOS /\b(?:crmo|crios)\/([\w\.]+)/i // Chrome for Android/iOS
], [VERSION, [NAME, PREFIX_MOBILE + 'Chrome']], [ ], [VERSION, [NAME, PREFIX_MOBILE + 'Chrome']], [
/edg(?:e|ios|a)?\/([\w\.]+)/i // Microsoft Edge /webview.+edge\/([\w\.]+)/i // Microsoft Edge
], [VERSION, [NAME, EDGE+' WebView']], [
/edg(?:e|ios|a)?\/([\w\.]+)/i
], [VERSION, [NAME, 'Edge']], [ ], [VERSION, [NAME, 'Edge']], [
// Presto based // Presto based
@ -443,6 +445,9 @@
/headlesschrome(?:\/([\w\.]+)| )/i // Chrome Headless /headlesschrome(?:\/([\w\.]+)| )/i // Chrome Headless
], [VERSION, [NAME, CHROME+' Headless']], [ ], [VERSION, [NAME, CHROME+' Headless']], [
/wv\).+chrome\/([\w\.]+).+edgw\//i // Edge WebView2
], [VERSION, [NAME, EDGE+' WebView2']], [
/ wv\).+(chrome)\/([\w\.]+)/i // Chrome WebView / wv\).+(chrome)\/([\w\.]+)/i // Chrome WebView
], [[NAME, CHROME+' WebView'], VERSION], [ ], [[NAME, CHROME+' WebView'], VERSION], [
@ -1234,10 +1239,16 @@
for (var i in brands) { for (var i in brands) {
var brandName = brands[i].brand || brands[i], var brandName = brands[i].brand || brands[i],
brandVersion = brands[i].version; brandVersion = brands[i].version;
if (this.itemType == UA_BROWSER && !/not.a.brand/i.test(brandName) && (!prevName || (/chrom/i.test(prevName) && brandName != CHROMIUM))) { if (this.itemType == UA_BROWSER &&
!/not.a.brand/i.test(brandName) &&
(!prevName ||
(/Chrom/.test(prevName) && brandName != CHROMIUM) ||
(prevName == EDGE && /WebView2/.test(brandName))
)) {
brandName = strMapper(brandName, { brandName = strMapper(brandName, {
'Chrome' : 'Google Chrome', 'Chrome' : 'Google Chrome',
'Edge' : 'Microsoft Edge', 'Edge' : 'Microsoft Edge',
'Edge WebView2' : 'Microsoft Edge WebView2',
'Chrome WebView' : 'Android WebView', 'Chrome WebView' : 'Android WebView',
'Chrome Headless' : 'HeadlessChrome', 'Chrome Headless' : 'HeadlessChrome',
'Huawei Browser' : 'HuaweiBrowser', 'Huawei Browser' : 'HuaweiBrowser',

View File

@ -2229,6 +2229,26 @@
"major" : "74" "major" : "74"
} }
}, },
{
"desc" : "Microsoft Edge WebView",
"ua" : "Mozilla/5.0 (Windows IoT 10.0; Android 6.0.1; WebView/3.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.140 Mobile Safari/537.36 Edge/18.17763",
"expect" :
{
"name" : "Edge WebView",
"version" : "18.17763",
"major" : "18"
}
},
{
"desc" : "Microsoft Edge WebView2",
"ua" : "Mozilla/5.0 (Linux; Android 11; SM-G991B Build/RP1A.200720.012; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/91.0.4472.120 Mobile Safari/537.36 EdgW/1.0",
"expect" :
{
"name" : "Edge WebView2",
"version" : "91.0.4472.120",
"major" : "91"
}
},
{ {
"desc" : "Iridium", "desc" : "Iridium",
"ua" : "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Iridium/43.8 Safari/537.36 Chrome/43.0.2357.132", "ua" : "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Iridium/43.8 Safari/537.36 Chrome/43.0.2357.132",

View File

@ -259,6 +259,16 @@
"type" : "crawler" "type" : "crawler"
} }
}, },
{
"desc" : "ChatGLM-Spider",
"ua" : "Mozilla/5.0 (compatible; ChatGLM-Spider/1.0; +https://chatglm.cn/)",
"expect" :
{
"name" : "ChatGLM-Spider",
"version" : "1.0",
"type" : "crawler"
}
},
{ {
"desc" : "Coc Coc Bot (web)", "desc" : "Coc Coc Bot (web)",
"ua" : "Mozilla/5.0 (compatible; coccocbot-web/1.0; +http://help.coccoc.com/searchengine)", "ua" : "Mozilla/5.0 (compatible; coccocbot-web/1.0; +http://help.coccoc.com/searchengine)",
@ -299,6 +309,36 @@
"type" : "crawler" "type" : "crawler"
} }
}, },
{
"desc" : "Daum",
"ua" : "Mozilla/5.0 (compatible; MSIE or Firefox mutant;) Daum 4.1",
"expect" :
{
"name" : "Daum",
"version" : "4.1",
"type" : "crawler"
}
},
{
"desc" : "Daumoa",
"ua" : "Mozilla/5.0 (compatible; MSIE or Firefox mutant; not on Windows server;) Daumoa 4.0",
"expect" :
{
"name" : "Daumoa",
"version" : "4.0",
"type" : "crawler"
}
},
{
"desc" : "Daumoa-image",
"ua" : "Mozilla/5.0 (compatible; MSIE or Firefox mutant; not on Windows server;) Daumoa-image/1.0",
"expect" :
{
"name" : "Daumoa-image",
"version" : "1.0",
"type" : "crawler"
}
},
{ {
"desc" : "Diffbot", "desc" : "Diffbot",
"ua" : "Diffbot/0.1", "ua" : "Diffbot/0.1",
@ -489,6 +529,16 @@
"type" : "crawler" "type" : "crawler"
} }
}, },
{
"desc" : "iAskBot",
"ua" : "Mozilla/5.0 AppleWebKit/605.1.15 (KHTML, like Gecko; compatible; iAskBot/1.0; +https://iask.ai/) Chrome/120.0.6099.119 Safari/605.1.15",
"expect" :
{
"name" : "iAskBot",
"version" : "1.0",
"type" : "crawler"
}
},
{ {
"desc" : "ImagesiftBot", "desc" : "ImagesiftBot",
"ua" : "Mozilla/5.0 (compatible; ImagesiftBot; +imagesift.com)", "ua" : "Mozilla/5.0 (compatible; ImagesiftBot; +imagesift.com)",
@ -580,6 +630,16 @@
"type" : "crawler" "type" : "crawler"
} }
}, },
{
"desc" : "Onespot",
"ua" : "Mozilla/5.0 (compatible; Onespot-ScraperBot/1.0; +https://www.onespot.com/identifying-traffic.html)",
"expect" :
{
"name" : "Onespot-ScraperBot",
"version" : "1.0",
"type" : "crawler"
}
},
{ {
"desc" : "OpenAI Search", "desc" : "OpenAI Search",
"ua" : "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; OAI-SearchBot/1.0; +https://openai.com/searchbot", "ua" : "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; OAI-SearchBot/1.0; +https://openai.com/searchbot",
@ -620,6 +680,36 @@
"type" : "crawler" "type" : "crawler"
} }
}, },
{
"desc" : "Qwantbot",
"ua" : "Mozilla/5.0 (compatible; Qwantbot/1.0_12345; +https://help.qwant.com/bot/)",
"expect" :
{
"name" : "Qwantbot",
"version" : "1.0_12345",
"type" : "crawler"
}
},
{
"desc" : "Qwantbot",
"ua" : "Mozilla/5.0 (compatible; Qwantbot-prod51071/1.0; +Qwantbot@qwant.com)",
"expect" :
{
"name" : "Qwantbot",
"version" : "1.0",
"type" : "crawler"
}
},
{
"desc" : "Qwantbot",
"ua" : "Mozilla/5.0 (compatible; Qwantbot-news/2.0; +https://help.qwant.com/bot/)",
"expect" :
{
"name" : "Qwantbot",
"version" : "2.0",
"type" : "crawler"
}
},
{ {
"desc" : "SemrushBot", "desc" : "SemrushBot",
"ua" : "Mozilla/5.0 (compatible; SemrushBot/7~bl; +http://www.semrush.com/bot.html)", "ua" : "Mozilla/5.0 (compatible; SemrushBot/7~bl; +http://www.semrush.com/bot.html)",
@ -680,6 +770,16 @@
"type" : "crawler" "type" : "crawler"
} }
}, },
{
"desc" : "Startpage",
"ua" : "StartpagePrivateImageProxy/3.0 (https://www.startpage.com/; support@startpage.com) aiohttp.client/3.11.11",
"expect" :
{
"name" : "StartpagePrivateImageProxy",
"version" : "3.0",
"type" : "crawler"
}
},
{ {
"desc" : "Teoma", "desc" : "Teoma",
"ua" : "Mozilla/2.0 (compatible; Ask Jeeves/Teoma; +http://sp.ask.com/docs/about/tech_crawling.html)", "ua" : "Mozilla/2.0 (compatible; Ask Jeeves/Teoma; +http://sp.ask.com/docs/about/tech_crawling.html)",

View File

@ -129,6 +129,16 @@
"type" : "fetcher" "type" : "fetcher"
} }
}, },
{
"desc" : "Iframely",
"ua" : "Iframely/1.3.1 (+https://iframely.com/docs/about)",
"expect" :
{
"name" : "Iframely",
"version" : "1.3.1",
"type" : "fetcher"
}
},
{ {
"desc" : "Meta-ExternalFetcher", "desc" : "Meta-ExternalFetcher",
"ua" : "meta-externalfetcher/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/crawler)", "ua" : "meta-externalfetcher/1.1 (+https://developers.facebook.com/docs/sharing/webmasters/crawler)",

View File

@ -313,6 +313,19 @@ describe('UA-CH Headers tests', () => {
} }
} }
}, },
{
headers : {
'sec-ch-ua': '" Not;A Brand";v="99", "Microsoft Edge";v="103", "Chromium";v="103", "Microsoft Edge WebView2";v="104"'
},
expect: {
browser : {
name : 'Edge WebView2',
version : '104',
major : '104',
type : undefined
}
}
},
{ {
headers : { headers : {
'sec-ch-ua': '"Not.A/Brand";v="8", "Chromium";v="114", "HuaweiBrowser";v="114"' 'sec-ch-ua': '"Not.A/Brand";v="8", "Chromium";v="114", "HuaweiBrowser";v="114"'