{"name":"Scout Radar","publisher":"AI Crawler Check","url":"https://aicrawlercheck.com/radar","view":"publisher view: what robots.txt files decide about each crawler. Not traffic.","list":"Tranco top 10,000","fetched":"2026-10-20","previous_fetch":"2026-10-17","domains":10000,"valid_files":5583,"parser":"RFC 9309","tranco_list":"Tranco top 10,000 (list GQG5K, generated 2026-10-19)","reproduce":"scripts/research/robots-study-fetch.py 10000, robots-study-retry.py, then build-scan.py","license":"CC BY 4.0, cite \"AI Crawler Check, Scout Radar\"","database":{"total":265,"official":193,"reported":72,"excluded":39},"decisions":{"anyTraining":1126,"anySearch":627,"anyAgent":613,"trainBlockSearchOpen":508,"trainAndSearch":618,"allAiOpen":4446,"gextSplit":575,"googlebotBlocked":154,"openaiSplit":427},"most_blocked":[{"token":"CCBot","operator":"Common Crawl","role":"training","files_blocking":899,"pct_of_valid":16.1,"pct_top1k":22.4,"pct_previous":15.9},{"token":"Bytespider","operator":"ByteDance","role":"training","files_blocking":887,"pct_of_valid":15.9,"pct_top1k":23.8,"pct_previous":15.8},{"token":"GPTBot","operator":"OpenAI","role":"training","files_blocking":859,"pct_of_valid":15.4,"pct_top1k":20.4,"pct_previous":15},{"token":"ClaudeBot","operator":"Anthropic","role":"training","files_blocking":805,"pct_of_valid":14.4,"pct_top1k":20.4,"pct_previous":14.2},{"token":"Google-Extended","operator":"Google","role":"training","files_blocking":723,"pct_of_valid":13,"pct_top1k":18.6,"pct_previous":12.7},{"token":"Meta-ExternalAgent","operator":"Meta","role":"training","files_blocking":706,"pct_of_valid":12.6,"pct_top1k":19.2,"pct_previous":12.5},{"token":"Applebot-Extended","operator":"Apple","role":"training","files_blocking":694,"pct_of_valid":12.4,"pct_top1k":17.6,"pct_previous":12.2},{"token":"Amazonbot","operator":"Amazon","role":"training","files_blocking":680,"pct_of_valid":12.2,"pct_top1k":15.6,"pct_previous":11.9},{"token":"PerplexityBot","operator":"Perplexity","role":"search","files_blocking":609,"pct_of_valid":10.9,"pct_top1k":17,"pct_previous":11.1},{"token":"ChatGPT-User","operator":"OpenAI","role":"agent","files_blocking":530,"pct_of_valid":9.5,"pct_top1k":13.4,"pct_previous":9.6},{"token":"DuckAssistBot","operator":"DuckDuckGo","role":"search","files_blocking":461,"pct_of_valid":8.3,"pct_top1k":14.2,"pct_previous":8.4},{"token":"AhrefsBot","operator":"Ahrefs","role":"seo","files_blocking":456,"pct_of_valid":8.2,"pct_top1k":11.6,"pct_previous":8.3},{"token":"MJ12bot","operator":"Majestic","role":"seo","files_blocking":455,"pct_of_valid":8.1,"pct_top1k":11.6,"pct_previous":8.3},{"token":"Perplexity-User","operator":"Perplexity","role":"agent","files_blocking":449,"pct_of_valid":8,"pct_top1k":15,"pct_previous":8.1},{"token":"OAI-SearchBot","operator":"OpenAI","role":"search","files_blocking":434,"pct_of_valid":7.8,"pct_top1k":12.2,"pct_previous":7.9},{"token":"Claude-User","operator":"Anthropic","role":"agent","files_blocking":430,"pct_of_valid":7.7,"pct_top1k":14,"pct_previous":7.8},{"token":"Claude-SearchBot","operator":"Anthropic","role":"search","files_blocking":424,"pct_of_valid":7.6,"pct_top1k":13.6,"pct_previous":7.7},{"token":"DotBot","operator":"Moz","role":"seo","files_blocking":419,"pct_of_valid":7.5,"pct_top1k":10.6,"pct_previous":7.5},{"token":"SemrushBot","operator":"Semrush","role":"seo","files_blocking":409,"pct_of_valid":7.3,"pct_top1k":10.2,"pct_previous":7.3},{"token":"GrokBot","operator":"xAI","role":"training","files_blocking":289,"pct_of_valid":5.2,"pct_top1k":11.4,"pct_previous":5.2},{"token":"Applebot","operator":"Apple","role":"search engine","files_blocking":283,"pct_of_valid":5.1,"pct_top1k":6.4,"pct_previous":5},{"token":"xAI-SearchBot","operator":"xAI","role":"search","files_blocking":246,"pct_of_valid":4.4,"pct_top1k":8.8,"pct_previous":4.4},{"token":"Bingbot","operator":"Microsoft","role":"search engine","files_blocking":182,"pct_of_valid":3.3,"pct_top1k":3.8,"pct_previous":3.2},{"token":"Googlebot","operator":"Google","role":"search engine","files_blocking":154,"pct_of_valid":2.8,"pct_top1k":3.6,"pct_previous":2.7}],"mistakes":[{"mistake":"Name a retired token","files":683,"denominator":5583,"pct":12.2},{"mistake":"Name only the retired token, not its replacement","files":151,"denominator":5583,"pct":2.7},{"mistake":"Block CSS or JavaScript paths","files":483,"denominator":5583,"pct":8.7},{"mistake":"Serve an HTML page instead of rules","files":501,"denominator":10000,"pct":5},{"mistake":"Answer robots.txt with HTTP 403","files":1117,"denominator":10000,"pct":11.2},{"mistake":"Block everyone with Disallow: /","files":240,"denominator":5583,"pct":4.3},{"mistake":"Block Googlebot itself","files":154,"denominator":5583,"pct":2.8},{"mistake":"Have no User-agent group at all","files":217,"denominator":5583,"pct":3.9}],"signals":{"contentSignal":188,"aiTrainNo":212,"aiTrainYes":139,"llmsTxt":118,"contentUsage":4,"sitemap":3764,"crawlDelay":651},"changes":[{"label":"Valid robots.txt files","a":5254,"b":5583,"unit":"files","aPct":100,"bPct":100},{"label":"Files with a sitemap line","a":3564,"b":3764,"unit":"files","aPct":67.8,"bPct":67.4},{"label":"Files mentioning llms.txt","a":115,"b":118,"unit":"files","aPct":2.2,"bPct":2.1},{"label":"Files with a Content-Signal line","a":164,"b":188,"unit":"files","aPct":3.1,"bPct":3.4},{"label":"Files that block GPTBot","a":789,"b":859,"unit":"files","aPct":15,"bPct":15.4},{"label":"Files that block Google-Extended","a":666,"b":723,"unit":"files","aPct":12.7,"bPct":13},{"label":"Files that block Googlebot","a":143,"b":154,"unit":"files","aPct":2.7,"bPct":2.8},{"label":"Files that block at least one AI training crawler","a":1049,"b":1126,"unit":"files","aPct":20,"bPct":20.2}]}