{
  "schemaVersion": 1,
  "title": "Nice Pick AI Crawler Observatory",
  "publisher": "Nice Pick",
  "published": "2026-07-15",
  "updated": "2026-08-10",
  "generatedAt": "2026-08-10T16:11:55.419Z",
  "canonicalUrl": "https://nicepick.dev/research/ai-crawler-reality-report",
  "dataUrl": "https://nicepick.dev/research/ai-crawler-observatory.json",
  "definitions": {
    "training": "Requests from known user agents associated with model training or bulk collection.",
    "retrieval": "Requests from known user agents associated with search, answer retrieval, or live user-directed fetches.",
    "citation": "A persisted answer-engine probe in which nicepick.dev appears in the answer's cited sources.",
    "aiReferral": "An attributed browser arrival from a recognized answer-engine source. This is separate from crawler traffic."
  },
  "edgeSnapshots": [
    {
      "id": "2026-07-08_2026-07-15",
      "capturedAt": "2026-07-15T17:41:24.348Z",
      "host": "nicepick.dev",
      "scope": "zone-wide",
      "methodologyVersion": "cloudflare-zone-user-agent-v1",
      "window": {
        "since": "2026-07-08T17:41:25.348Z",
        "until": "2026-07-15T17:41:24.348Z",
        "days": 7
      },
      "totals": {
        "requests": 513534,
        "http2xx": 397088,
        "http3xx": 66203,
        "http4xx": 45017,
        "http5xx": 5223,
        "http5xxRate": 0.0102
      },
      "classes": {
        "training": {
          "requests": 254595,
          "share": 0.4958,
          "http5xx": 4603
        },
        "retrieval": {
          "requests": 331,
          "share": 0.0006,
          "http5xx": 4
        },
        "search-engine": {
          "requests": 1926,
          "share": 0.0038,
          "http5xx": 3
        },
        "seo-tool": {
          "requests": 79189,
          "share": 0.1542,
          "http5xx": 142
        },
        "other-bot": {
          "requests": 266,
          "share": 0.0005,
          "http5xx": 0
        },
        "human": {
          "requests": 177058,
          "share": 0.3448,
          "http5xx": 471
        }
      },
      "topActors": [
        {
          "actor": "OAI-SearchBot (ChatGPT search)",
          "class": "retrieval",
          "requests": 256,
          "http5xx": 0
        },
        {
          "actor": "ChatGPT-User (live user fetch)",
          "class": "retrieval",
          "requests": 44,
          "http5xx": 0
        },
        {
          "actor": "PerplexityBot",
          "class": "retrieval",
          "requests": 30,
          "http5xx": 4
        },
        {
          "actor": "Other retrieval-class user agents",
          "class": "retrieval",
          "requests": 1,
          "http5xx": 0
        }
      ],
      "derived": {
        "trainingToRetrievalRatio": 769.1691842900302
      }
    },
    {
      "id": "2026-08-03_2026-08-10",
      "capturedAt": "2026-08-10T16:11:57.369Z",
      "host": "nicepick.dev",
      "scope": "zone-wide",
      "methodologyVersion": "cloudflare-zone-user-agent-v1",
      "window": {
        "since": "2026-08-03T16:11:56.667Z",
        "until": "2026-08-10T16:11:55.667Z",
        "days": 7
      },
      "totals": {
        "requests": 846171,
        "http2xx": 658757,
        "http3xx": 122140,
        "http4xx": 65259,
        "http5xx": 15,
        "http5xxRate": 0.000017726913354392907
      },
      "classes": {
        "retrieval": {
          "requests": 2300,
          "share": 0.0027181267143402458,
          "http5xx": 0
        },
        "training": {
          "requests": 307145,
          "share": 0.3629821868156673,
          "http5xx": 14
        },
        "seo-tool": {
          "requests": 172843,
          "share": 0.2042648589942222,
          "http5xx": 1
        },
        "search-engine": {
          "requests": 3576,
          "share": 0.004226096143687269,
          "http5xx": 0
        },
        "other-bot": {
          "requests": 5888,
          "share": 0.006958404388711029,
          "http5xx": 0
        },
        "human": {
          "requests": 335916,
          "share": 0.39698358842361653,
          "http5xx": 0
        }
      },
      "topActors": [
        {
          "actor": "Meta (Llama)",
          "class": "training",
          "requests": 217316,
          "http5xx": 13
        },
        {
          "actor": "AhrefsBot",
          "class": "seo-tool",
          "requests": 100412,
          "http5xx": 0
        },
        {
          "actor": "Amazonbot",
          "class": "training",
          "requests": 61074,
          "http5xx": 1
        },
        {
          "actor": "SemrushBot",
          "class": "seo-tool",
          "requests": 57493,
          "http5xx": 1
        },
        {
          "actor": "ClaudeBot (Anthropic training)",
          "class": "training",
          "requests": 28269,
          "http5xx": 0
        },
        {
          "actor": "Majestic",
          "class": "seo-tool",
          "requests": 14013,
          "http5xx": 0
        },
        {
          "actor": "Bingbot",
          "class": "search-engine",
          "requests": 2299,
          "http5xx": 0
        },
        {
          "actor": "OAI-SearchBot (ChatGPT search)",
          "class": "retrieval",
          "requests": 1856,
          "http5xx": 0
        },
        {
          "actor": "Googlebot",
          "class": "search-engine",
          "requests": 1263,
          "http5xx": 0
        },
        {
          "actor": "DotBot (Moz)",
          "class": "seo-tool",
          "requests": 861,
          "http5xx": 0
        },
        {
          "actor": "ChatGPT-User (live user fetch)",
          "class": "retrieval",
          "requests": 343,
          "http5xx": 0
        },
        {
          "actor": "CCBot (Common Crawl)",
          "class": "training",
          "requests": 235,
          "http5xx": 0
        },
        {
          "actor": "GPTBot (OpenAI training)",
          "class": "training",
          "requests": 157,
          "http5xx": 0
        },
        {
          "actor": "PerplexityBot",
          "class": "retrieval",
          "requests": 101,
          "http5xx": 0
        },
        {
          "actor": "Google-Extended",
          "class": "training",
          "requests": 93,
          "http5xx": 0
        },
        {
          "actor": "DataForSEO",
          "class": "seo-tool",
          "requests": 64,
          "http5xx": 0
        }
      ],
      "derived": {
        "trainingToRetrievalRatio": 133.5413043478261
      }
    }
  ],
  "currentSignals": {
    "capturedAt": "2026-08-10T16:11:55.419Z",
    "searchVisibility": {
      "status": "available",
      "source": "Google Search Console",
      "reportingLagDays": 2,
      "current": {
        "since": "2026-07-12",
        "until": "2026-08-08",
        "days": 28,
        "clicks": 2,
        "impressions": 8812,
        "ctr": 0.00022696323195642307,
        "position": 61.80764866091693
      },
      "previous": {
        "since": "2026-06-14",
        "until": "2026-07-11",
        "days": 28,
        "clicks": 4,
        "impressions": 8025,
        "ctr": 0.0004984423676012461,
        "position": 76.0802492211838
      }
    },
    "contentRetrieval": {
      "status": "available",
      "source": "Cloudflare Analytics Engine",
      "scope": "matched content middleware routes plus explicit server-side crawler logging",
      "current": {
        "since": "2026-08-03T16:11:55.419Z",
        "until": "2026-08-10T16:11:55.419Z",
        "days": 7,
        "requests": 33,
        "byActor": {
          "oai-searchbot": 22,
          "chatgpt-user": 6,
          "perplexitybot": 5
        }
      },
      "previous": {
        "since": "2026-07-27T16:11:55.419Z",
        "until": "2026-08-03T16:11:55.419Z",
        "days": 7,
        "requests": 205,
        "byActor": {
          "perplexitybot": 100,
          "oai-searchbot": 77,
          "chatgpt-user": 28
        }
      },
      "changePercent": -83.90243902439025
    },
    "citations": {
      "status": "stale",
      "source": "persisted citation probes",
      "latestProbeAt": "2026-07-03T14:54:34.344871+00:00",
      "engine": "openai-manual",
      "model": "chatgpt-search (manual)",
      "cited": 0,
      "probed": 11,
      "citationRate": 0,
      "ageDays": 38.053716145833334,
      "currentPerformanceAvailable": false
    },
    "aiReferrals": {
      "status": "available",
      "source": "Cloudflare Analytics Engine",
      "last7Days": 0,
      "last30Days": 3,
      "bySource7Days": {},
      "bySource30Days": {
        "chatgpt": 2,
        "bing-copilot": 1
      }
    }
  },
  "limitations": [
    "Cloudflare HTTP Requests Adaptive Groups are analytics estimates, not raw access logs.",
    "Crawler class totals cover up to the top 300 user-agent groups in each daily query; the overall edge total uses the zone-wide aggregate.",
    "User agents can be spoofed, and actor identity does not prove how fetched content was used.",
    "Zone-wide retrieval and content-route retrieval have different scopes and must not be compared as one series.",
    "Requests are not unique pages, people, referrals, recommendations, or citations.",
    "Google Search Console data is delayed by about two days.",
    "A stale citation probe means current citation performance is unavailable, not zero.",
    "AI referral attribution is heuristic and uses the post-2026-07-25 hardened series.",
    "This is one website. Its traffic mix should not be generalized to the web."
  ]
}