{
  "name": "The 2026 AI Crawler Verification Report — Dataset",
  "publisher": "Unsourced (https://unsourced.app)",
  "report_url": "https://unsourced.app/ai-crawler-report",
  "license": "Free to cite with attribution to Unsourced.",
  "temporal_coverage": "2026-06-10/2026-08-06",
  "totals": {
    "visits": 14227,
    "distinct_ips_approx": 2015,
    "named_crawlers": 18,
    "verified_pct_by_visit": 56,
    "outside_published_range_pct_by_visit": 34,
    "unverifiable_pct_by_visit": 10,
    "verified_pct_by_distinct_ip": 59,
    "outside_published_range_pct_by_distinct_ip": 27,
    "unverifiable_pct_by_distinct_ip": 14
  },
  "method": "Each crawler IP was checked against the operator's live-published CIDR ranges (OpenAI, Anthropic, Apple, Perplexity, Google) and forward-confirmed reverse DNS. Verified = at least one proof matched. Outside a published range = the operator publishes a list and the IP was not on it (reported as-is, NOT asserted as fraud, because published lists can be incomplete). Unverifiable = the operator publishes nothing to check against.",
  "notes": [
    "Percentages for crawlers with fewer than ~50 visits are low-confidence (small sample); the visits column shows sample size.",
    "'operator_publishes_ip_ranges' reflects whether the operating company publishes a machine-readable IP feed. Some legacy user-agents (e.g. anthropic-ai) are not mapped to their operator's current feed and can read as unverifiable despite the company publishing.",
    "'Outside a published range' is a fact about the IP, not a fraud verdict: it can be spoofing OR undisclosed operator infrastructure.",
    "Aggregated by operator only. Contains no per-request, per-IP, or per-site data."
  ],
  "crawlers": [
    {"crawler": "ChatGPT-User", "operator": "OpenAI", "category": "search", "visits": 7888, "distinct_ips": 1255, "verified_pct": 53, "outside_published_range_pct": 47, "unverifiable_pct": 0, "operator_publishes_ip_ranges": true},
    {"crawler": "ClaudeBot", "operator": "Anthropic", "category": "trainer", "visits": 1962, "distinct_ips": 122, "verified_pct": 92, "outside_published_range_pct": 8, "unverifiable_pct": 0, "operator_publishes_ip_ranges": true},
    {"crawler": "Bytespider", "operator": "ByteDance", "category": "scraper", "visits": 1062, "distinct_ips": 276, "verified_pct": 0, "outside_published_range_pct": 13, "unverifiable_pct": 87, "operator_publishes_ip_ranges": false},
    {"crawler": "GPTBot", "operator": "OpenAI", "category": "trainer", "visits": 971, "distinct_ips": 48, "verified_pct": 82, "outside_published_range_pct": 18, "unverifiable_pct": 0, "operator_publishes_ip_ranges": true},
    {"crawler": "OAI-SearchBot", "operator": "OpenAI", "category": "search", "visits": 946, "distinct_ips": 129, "verified_pct": 69, "outside_published_range_pct": 31, "unverifiable_pct": 0, "operator_publishes_ip_ranges": true},
    {"crawler": "PerplexityBot", "operator": "Perplexity", "category": "search", "visits": 477, "distinct_ips": 43, "verified_pct": 66, "outside_published_range_pct": 34, "unverifiable_pct": 0, "operator_publishes_ip_ranges": true},
    {"crawler": "Amazonbot", "operator": "Amazon", "category": "trainer", "visits": 357, "distinct_ips": 226, "verified_pct": 62, "outside_published_range_pct": 2, "unverifiable_pct": 36, "operator_publishes_ip_ranges": true},
    {"crawler": "CCBot", "operator": "Common Crawl", "category": "trainer", "visits": 317, "distinct_ips": 33, "verified_pct": 0, "outside_published_range_pct": 4, "unverifiable_pct": 96, "operator_publishes_ip_ranges": false},
    {"crawler": "PetalBot", "operator": "Huawei", "category": "scraper", "visits": 61, "distinct_ips": 8, "verified_pct": 0, "outside_published_range_pct": 72, "unverifiable_pct": 28, "operator_publishes_ip_ranges": false},
    {"crawler": "anthropic-ai", "operator": "Anthropic", "category": "trainer", "visits": 37, "distinct_ips": 9, "verified_pct": 0, "outside_published_range_pct": 43, "unverifiable_pct": 57, "operator_publishes_ip_ranges": true},
    {"crawler": "cohere-ai", "operator": "Cohere", "category": "trainer", "visits": 36, "distinct_ips": 11, "verified_pct": 0, "outside_published_range_pct": 44, "unverifiable_pct": 56, "operator_publishes_ip_ranges": false},
    {"crawler": "YouBot", "operator": "You.com", "category": "search", "visits": 27, "distinct_ips": 9, "verified_pct": 67, "outside_published_range_pct": 0, "unverifiable_pct": 33, "operator_publishes_ip_ranges": false},
    {"crawler": "Google-Extended", "operator": "Google", "category": "trainer", "visits": 23, "distinct_ips": 4, "verified_pct": 0, "outside_published_range_pct": 100, "unverifiable_pct": 0, "operator_publishes_ip_ranges": true},
    {"crawler": "Diffbot", "operator": "Diffbot", "category": "scraper", "visits": 19, "distinct_ips": 6, "verified_pct": 0, "outside_published_range_pct": 63, "unverifiable_pct": 37, "operator_publishes_ip_ranges": false},
    {"crawler": "GoogleOther", "operator": "Google", "category": "trainer", "visits": 12, "distinct_ips": 12, "verified_pct": 58, "outside_published_range_pct": 42, "unverifiable_pct": 0, "operator_publishes_ip_ranges": true},
    {"crawler": "Applebot-Extended", "operator": "Apple", "category": "trainer", "visits": 12, "distinct_ips": 5, "verified_pct": 0, "outside_published_range_pct": 0, "unverifiable_pct": 100, "operator_publishes_ip_ranges": true},
    {"crawler": "DataForSeoBot", "operator": "DataForSEO", "category": "scraper", "visits": 10, "distinct_ips": 1, "verified_pct": 0, "outside_published_range_pct": 100, "unverifiable_pct": 0, "operator_publishes_ip_ranges": false},
    {"crawler": "Google-CloudVertexBot", "operator": "Google", "category": "trainer", "visits": 10, "distinct_ips": 4, "verified_pct": 0, "outside_published_range_pct": 100, "unverifiable_pct": 0, "operator_publishes_ip_ranges": true}
  ]
}