{
 "title": "AI crawler blocking rates, top 25000 domains",
 "publisher": "Driftwitness Research",
 "url": "https://driftwitness.com/research/who-blocks-ai-crawlers",
 "license": "CC BY 4.0",
 "scan_date": "2026-08-26",
 "source_list": {
  "name": "Tranco",
  "id": "38LNL",
  "generated": "2026-08-24",
  "url": "https://tranco-list.eu/list/38LNL"
 },
 "method": "One /robots.txt fetch per domain; rules evaluated per RFC 9309. Percentages use the responding-domain denominator; domains with no robots.txt count as not blocking.",
 "population": {
  "attempted": 25000,
  "responding": 17945,
  "parseable_robots": 13035,
  "unreachable": 6875,
  "no_robots": 3772,
  "served_html": 1138
 },
 "ai_aware_domains": {
  "n": 3375,
  "pct_of_responding": 18.8
 },
 "blocking_by_crawler": {
  "GPTBot": {
   "explicit_block_n": 2041,
   "explicit_block_pct_of_responding": 11.4,
   "any_block_pct_of_responding": 14.0,
   "named_by_n": 2727,
   "pct_of_namings_that_block_root": 74.8
  },
  "CCBot": {
   "explicit_block_n": 2060,
   "explicit_block_pct_of_responding": 11.5,
   "any_block_pct_of_responding": 14.2,
   "named_by_n": 2381,
   "pct_of_namings_that_block_root": 86.5
  },
  "Google-Extended": {
   "explicit_block_n": 1680,
   "explicit_block_pct_of_responding": 9.4,
   "any_block_pct_of_responding": 12.0,
   "named_by_n": 2217,
   "pct_of_namings_that_block_root": 75.8
  },
  "Applebot-Extended": {
   "explicit_block_n": 1602,
   "explicit_block_pct_of_responding": 8.9,
   "any_block_pct_of_responding": 11.6,
   "named_by_n": 1843,
   "pct_of_namings_that_block_root": 86.9
  },
  "meta-externalagent": {
   "explicit_block_n": 1647,
   "explicit_block_pct_of_responding": 9.2,
   "any_block_pct_of_responding": 12.0,
   "named_by_n": 1898,
   "pct_of_namings_that_block_root": 86.8
  },
  "Bytespider": {
   "explicit_block_n": 1978,
   "explicit_block_pct_of_responding": 11.0,
   "any_block_pct_of_responding": 13.9,
   "named_by_n": 2156,
   "pct_of_namings_that_block_root": 91.7
  },
  "Amazonbot": {
   "explicit_block_n": 1662,
   "explicit_block_pct_of_responding": 9.3,
   "any_block_pct_of_responding": 12.0,
   "named_by_n": 1931,
   "pct_of_namings_that_block_root": 86.1
  },
  "ClaudeBot": {
   "explicit_block_n": 1845,
   "explicit_block_pct_of_responding": 10.3,
   "any_block_pct_of_responding": 13.0,
   "named_by_n": 2415,
   "pct_of_namings_that_block_root": 76.4
  },
  "anthropic-ai": {
   "explicit_block_n": 776,
   "explicit_block_pct_of_responding": 4.3,
   "any_block_pct_of_responding": 7.2,
   "named_by_n": 1022,
   "pct_of_namings_that_block_root": 75.9
  },
  "Googlebot": {
   "explicit_block_n": 21,
   "explicit_block_pct_of_responding": 0.1,
   "any_block_pct_of_responding": 2.0,
   "named_by_n": 1048,
   "pct_of_namings_that_block_root": 2.0
  },
  "Bingbot": {
   "explicit_block_n": 32,
   "explicit_block_pct_of_responding": 0.2,
   "any_block_pct_of_responding": 2.4,
   "named_by_n": 854,
   "pct_of_namings_that_block_root": 3.7
  },
  "OAI-SearchBot": {
   "explicit_block_n": 388,
   "explicit_block_pct_of_responding": 2.2,
   "any_block_pct_of_responding": 4.8,
   "named_by_n": 965,
   "pct_of_namings_that_block_root": 40.2
  },
  "PerplexityBot": {
   "explicit_block_n": 708,
   "explicit_block_pct_of_responding": 3.9,
   "any_block_pct_of_responding": 6.6,
   "named_by_n": 1326,
   "pct_of_namings_that_block_root": 53.4
  },
  "Claude-SearchBot": {
   "explicit_block_n": 306,
   "explicit_block_pct_of_responding": 1.7,
   "any_block_pct_of_responding": 4.6,
   "named_by_n": 557,
   "pct_of_namings_that_block_root": 54.9
  },
  "ChatGPT-User": {
   "explicit_block_n": 642,
   "explicit_block_pct_of_responding": 3.6,
   "any_block_pct_of_responding": 6.2,
   "named_by_n": 1272,
   "pct_of_namings_that_block_root": 50.5
  },
  "Perplexity-User": {
   "explicit_block_n": 332,
   "explicit_block_pct_of_responding": 1.9,
   "any_block_pct_of_responding": 4.7,
   "named_by_n": 559,
   "pct_of_namings_that_block_root": 59.4
  },
  "Claude-User": {
   "explicit_block_n": 299,
   "explicit_block_pct_of_responding": 1.7,
   "any_block_pct_of_responding": 4.5,
   "named_by_n": 547,
   "pct_of_namings_that_block_root": 54.7
  },
  "MistralAI-User": {
   "explicit_block_n": 265,
   "explicit_block_pct_of_responding": 1.5,
   "any_block_pct_of_responding": 4.5,
   "named_by_n": 343,
   "pct_of_namings_that_block_root": 77.3
  }
 },
 "training_vs_search": {
  "GPTBot": {
   "search_bot": "OAI-SearchBot",
   "loose_n": 1659,
   "reverse_n": 15,
   "both_n": 849,
   "by_name_n": 1643,
   "strict_n": 102,
   "silent_n": 1541,
   "strict_pct": 6.2,
   "silent_pct": 93.8,
   "silent_in_ten": 9.4,
   "examples": [
    "yahoo.com",
    "snapchat.com",
    "forbes.com",
    "ebay.com",
    "tripadvisor.com",
    "globalsign.com",
    "ted.com",
    "academia.edu",
    "ebay.co.uk",
    "figma.com"
   ]
  },
  "ClaudeBot": {
   "search_bot": "Claude-SearchBot",
   "loose_n": 1523,
   "reverse_n": 6,
   "both_n": 813,
   "by_name_n": 1508,
   "strict_n": 65,
   "silent_n": 1443,
   "strict_pct": 4.3,
   "silent_pct": 95.7,
   "silent_in_ten": 9.6,
   "examples": [
    "yahoo.com",
    "snapchat.com",
    "forbes.com",
    "ebay.com",
    "globalsign.com",
    "ted.com",
    "ebay.co.uk",
    "pexels.com",
    "figma.com",
    "vnexpress.net"
   ]
  }
 },
 "posture": {
  "blockers_n": 3091,
  "blockers_pct": 17.2,
  "all_n": 664,
  "all_pct_of_blockers": 21.5,
  "wildcard_only_n": 466,
  "n_training_bots_tested": 9,
  "eight_or_nine_n": 1828,
  "eight_or_nine_pct": 59.1,
  "distribution": {
   "1": 370,
   "2": 160,
   "3": 186,
   "4": 104,
   "5": 104,
   "6": 131,
   "7": 208,
   "8": 1164,
   "9": 664
  }
 },
 "by_rank_band": [
  {
   "band": "1\u20131,000",
   "n": 669,
   "aware_pct": 24.7,
   "GPTBot": 11.8,
   "ClaudeBot": 11.7,
   "PerplexityBot": 9.1,
   "OAI-SearchBot": 4.5
  },
  {
   "band": "1,001\u20135,000",
   "n": 2801,
   "aware_pct": 20.2,
   "GPTBot": 11.6,
   "ClaudeBot": 10.4,
   "PerplexityBot": 5.8,
   "OAI-SearchBot": 2.9
  },
  {
   "band": "5,001\u201310,000",
   "n": 3746,
   "aware_pct": 19.3,
   "GPTBot": 11.8,
   "ClaudeBot": 10.8,
   "PerplexityBot": 4.9,
   "OAI-SearchBot": 2.9
  },
  {
   "band": "10,001\u201325,000",
   "n": 10729,
   "aware_pct": 17.9,
   "GPTBot": 11.1,
   "ClaudeBot": 10.0,
   "PerplexityBot": 2.8,
   "OAI-SearchBot": 1.6
  }
 ]
}