From c423751b31a29b723c69b460d618404682513a1a Mon Sep 17 00:00:00 2001 From: "releaser-ai-plugin[bot]" <273148615+releaser-ai-plugin[bot]@users.noreply.github.com> Date: Fri, 16 Oct 2026 12:07:35 +0000 Subject: [PATCH] chore: sync skills (agent-skills-v0.3245.0, context-mill@v1.57.0) --- .claude-plugin/marketplace.json | 4 +- .claude-plugin/plugin.json | 2 +- .codex-plugin/plugin.json | 2 +- .cursor-plugin/plugin.json | 2 +- gemini-extension.json | 2 +- skills/.sync-manifest | 1 + .../references/scheduled-precheck.md | 1 + .../references/scout-anatomy.md | 7 +- .../references/graph-schema.md | 40 ++ skills/creating-surveys/SKILL.md | 32 +- .../creating-surveys/references/examples.md | 33 +- .../references/response-rates.md | 41 ++ .../references/empty-experiment.md | 9 +- .../references/interpretation.md | 7 +- .../references/mid-run-changes.md | 3 +- .../references/finding-traces.md | 7 +- skills/exploring-llm-evaluations/SKILL.md | 59 ++- skills/implementing-api-surveys/SKILL.md | 130 ++++++ .../references/event-contract.md | 37 ++ .../references/server-and-mobile.md | 88 ++++ .../references/EXAMPLE-expo.md | 4 +- .../references/EXAMPLE-fastapi.md | 2 +- .../references/EXAMPLE-flask.md | 2 +- .../references/EXAMPLE-javascript-node.md | 2 +- .../references/EXAMPLE-javascript-web.md | 2 +- .../references/EXAMPLE-laravel.md | 4 +- .../references/EXAMPLE-php.md | 2 +- .../references/EXAMPLE-python.md | 2 +- .../references/EXAMPLE-ruby.md | 2 +- .../instrument-integration/references/js.md | 1 + .../references/posthog-python.md | 2 +- .../references/anthropic.md | 404 ++++++++++++++++++ .../references/EXAMPLE-expo.md | 4 +- .../references/EXAMPLE-fastapi.md | 2 +- .../references/EXAMPLE-flask.md | 2 +- .../references/EXAMPLE-laravel.md | 4 +- .../references/EXAMPLE-php.md | 2 +- .../references/EXAMPLE-python.md | 2 +- .../references/EXAMPLE-ruby.md | 2 +- .../references/posthog-python.md | 2 +- skills/managing-experiment-lifecycle/SKILL.md | 6 +- .../references/available-functions.md | 1 + .../references/example-error-tracking.md | 4 +- .../references/example-logs.md | 2 +- .../references/example-session-replay.md | 8 +- .../references/example-sessions.md | 2 +- .../references/guidelines.md | 26 +- .../SKILL.md | 169 +++----- .../signals-scout-ai-observability/SKILL.md | 4 +- skills/signals-scout-conversations/SKILL.md | 5 +- skills/signals-scout-csp-violations/SKILL.md | 11 +- .../SKILL.md | 2 +- .../signals-scout-customer-analytics/SKILL.md | 2 +- skills/signals-scout-mcp-tool-calls/SKILL.md | 4 +- .../signals-scout-product-analytics/SKILL.md | 3 +- telemetry/.claude-plugin/plugin.json | 2 +- 56 files changed, 1007 insertions(+), 200 deletions(-) create mode 100644 skills/creating-surveys/references/response-rates.md create mode 100644 skills/implementing-api-surveys/SKILL.md create mode 100644 skills/implementing-api-surveys/references/event-contract.md create mode 100644 skills/implementing-api-surveys/references/server-and-mobile.md diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 24033b7..80be4e1 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -12,7 +12,7 @@ "displayName": "PostHog", "source": "./", "description": "Access PostHog analytics, feature flags, experiments, error tracking, and insights directly from your AI coding tool. Optionally capture Claude Code sessions to PostHog LLM Analytics.", - "version": "1.1.68", + "version": "1.1.69", "author": { "name": "PostHog", "email": "hey@posthog.com", @@ -38,7 +38,7 @@ "displayName": "PostHog session capture", "source": "./telemetry", "description": "Send Claude Code sessions to PostHog LLM analytics. Hooks only: no skills, no commands, no MCP server. Install this instead of the PostHog plugin, not alongside it.", - "version": "1.0.4", + "version": "1.0.5", "author": { "name": "PostHog", "email": "hey@posthog.com", diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index e0646d6..dbd1336 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "posthog", "description": "Access PostHog analytics, feature flags, experiments, error tracking, and insights directly from your AI coding tool. Optionally capture Claude Code sessions to PostHog LLM Analytics.", - "version": "1.1.68", + "version": "1.1.69", "author": { "name": "PostHog", "email": "hey@posthog.com", diff --git a/.codex-plugin/plugin.json b/.codex-plugin/plugin.json index 639b7b4..b696de3 100644 --- a/.codex-plugin/plugin.json +++ b/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "posthog", - "version": "1.0.65", + "version": "1.0.66", "description": "Access PostHog analytics, feature flags, experiments, error tracking, and insights directly from Codex", "author": { "name": "PostHog", diff --git a/.cursor-plugin/plugin.json b/.cursor-plugin/plugin.json index 64e9930..e51f54f 100644 --- a/.cursor-plugin/plugin.json +++ b/.cursor-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "posthog", "displayName": "PostHog", - "version": "1.1.61", + "version": "1.1.62", "description": "Access PostHog analytics, feature flags, experiments, error tracking, and insights directly from Cursor", "author": { "name": "PostHog", diff --git a/gemini-extension.json b/gemini-extension.json index 9631b94..5d08ef3 100644 --- a/gemini-extension.json +++ b/gemini-extension.json @@ -1,6 +1,6 @@ { "name": "posthog", - "version": "1.0.63", + "version": "1.0.64", "description": "Access PostHog analytics, feature flags, experiments, error tracking, and insights directly from Gemini CLI", "mcpServers": { "posthog": { diff --git a/skills/.sync-manifest b/skills/.sync-manifest index 3e8d2bc..0d72580 100644 --- a/skills/.sync-manifest +++ b/skills/.sync-manifest @@ -75,6 +75,7 @@ finding-replay-for-issue finding-sessions-to-watch formatting-insight-axes grouping-noisy-errors +implementing-api-surveys inbox-exploration instrument-error-tracking instrument-feature-flags diff --git a/skills/authoring-scouts/references/scheduled-precheck.md b/skills/authoring-scouts/references/scheduled-precheck.md index 2cada71..581f28e 100644 --- a/skills/authoring-scouts/references/scheduled-precheck.md +++ b/skills/authoring-scouts/references/scheduled-precheck.md @@ -112,6 +112,7 @@ scout-precheck-query: | - The harness reads the default from the canonical skill file that PostHog ships, not from a project's copy of the skill. Adding `scout-precheck-query` to a custom scout or to a team's edited copy changes nothing. For those scouts, save `precheck_query` on the config. - A merged default does not apply to every project at once. A rollout setting decides which projects get it. +- The project profile's `surfaces` section runs every canonical default too, and reads its rows as "this surface is in use". Keep a default a test of activity on the surface. A backstop row or a "new since the last run" filter changes that answer for every project. `surfaces` binds `{since}` and `{interval_minutes}` to a 30-day span. - When you write a canonical default, use the canonical scouts that already ship one (for example `signals-scout-mcp-tool-calls` above, or `signals-scout-surveys`) as models. ## Tell the body about the rows diff --git a/skills/authoring-scouts/references/scout-anatomy.md b/skills/authoring-scouts/references/scout-anatomy.md index aaac6db..4088564 100644 --- a/skills/authoring-scouts/references/scout-anatomy.md +++ b/skills/authoring-scouts/references/scout-anatomy.md @@ -74,8 +74,9 @@ The fleet's specialists all share this shape: Examples: `count` vs `distinct_users` ratio (error tracking); reach over raw count (CSP); negative+mixed share vs baseline (MCP feedback). Without this, the scout wastes every run re-deciding what "normal" means. -2. **Quick close-out.** A cheap early-exit so a quiet run costs almost nothing: if the watched event is absent from the profile's `top_events` or sitting at baseline (no fresh 24h activity), write one scratchpad entry and stop. +2. **Quick close-out.** A cheap early-exit so a quiet run costs almost nothing: if the surface is not in use, or the watched event sits at baseline (no fresh 24h activity), write one scratchpad entry and stop. This keeps idle scouts cheap. + Decide "not in use" from the profile's `surfaces` entry for the scout, never from absence in `top_events`: `top_events` keeps only the top events by volume, so a quiet surface that is in use falls off it. A canonical scout gets a `surfaces` entry when it ships a `scout-precheck-query`. A scout with no entry, or an entry that reads `in_use: null`, checks with one bounded `execute-sql` count. `top_events` counts are windowed (each row carries `window_days`), not lifetime — a project whose ingestion recently went dark reads identically to one that never had traffic. Before closing out a busy-looking project as empty on `top_events` thinness alone, rule out a capture gap with a direct `execute-sql` over a longer window (e.g. 30d); only close out when the low volume holds there. ```text @@ -157,8 +158,8 @@ signal-vs-noise discriminator. Internalize that shape. ## Quick close-out: is even loud? -If is absent from `top_events` or at baseline (no fresh 24h activity), -isn't where the signal is today. Cheap scratchpad entry + close out empty. +If `surfaces[""]` reads `in_use: false`, or is at baseline (no fresh 24h activity), +isn't where the signal is today. Absence from `top_events` alone proves nothing. Cheap scratchpad entry + close out empty. ## How a run works diff --git a/skills/building-workflows/references/graph-schema.md b/skills/building-workflows/references/graph-schema.md index 3bda790..7938153 100644 --- a/skills/building-workflows/references/graph-schema.md +++ b/skills/building-workflows/references/graph-schema.md @@ -159,6 +159,45 @@ Retrieve `template-native-push` with `cdp-function-templates-retrieve` for the f - Required: `distinctId`, `channels`, `title`. Optional: `body`, `image`, `data`, `ttlSeconds`, `android_*`, `ios_*` (retrieve the `inputs_schema` for the full set). - Never hand-author `bytecode` — the server compiles it from `value`. Omit `order` too: the editor lays fields out in the template's `inputs_schema` order (fixed and consistent), not by the `order` on your inputs, so leaving it off doesn't change the form. Push has no delivered/opened/clicked signal (FCM/APNs respond synchronously), so a successful send means "accepted for delivery", nothing more. +### Replay Vision step + +Replay Vision is PostHog's AI that watches session recordings. It replays a session as video, together with its events, and answers a question about it. Each scan uses credits from the project's Replay Vision quota. + +A `function` node with `template_id: "template-posthog-replay-vision"` scans the trigger event's session and waits for the answer. Use it when the next step depends on what happened in the session, which event properties alone can't tell, for example to add session context to a Slack alert or to branch on whether the user struggled. The project must record sessions, and the trigger event must carry a `$session_id`. + +```json +{ + "id": "vision_1", + "name": "Did checkout fail?", + "type": "function", + "config": { + "template_id": "template-posthog-replay-vision", + "inputs": { + "analysis": { "value": { "type": "monitor", "prompt": "Did the user hit an error during checkout?" } } + } + }, + "output_variable": { "key": "replay_vision" } +} +``` + +- **`analysis` is a literal object, not a template.** Never set `"templating"` on it, and never put `{...}` expressions in it. Its `value` takes one of these shapes: + + | `type` | Required keys | What the step answers | + | ------------ | ----------------------------------------------- | --------------------------------------------------------- | + | `monitor` | `prompt` | A yes/no verdict on the question in `prompt`. | + | `classifier` | `prompt`, `tags` (1 to 100 category strings) | Which of the categories in `tags` fit the session. | + | `scorer` | `prompt`, `scale: {"min": , "max": }` | A score on the scale. `max` must be greater than `min`. | + | `summarizer` | `prompt` | A title, a summary, and chapters. | + | `scanner` | `scanner_id` (a saved scanner's UUID) | The saved scanner's answer, with its own type and prompt. | + +- The inline types take the same optional settings as a saved scanner of that type. See the `creating-replay-vision-scanners` skill. Find saved scanners with the `vision-scanners` tools. +- `session_id` defaults to `{event.properties.$session_id}`. Leave it unset unless the session ID is in another property. +- `wait_for_session_end` defaults to `true`, which scans the recording after the session has been quiet for 35 minutes. Set it to `false` only when the trigger event comes after the session ended. +- The step result is `{"state", "output", "warnings"}`. `state` is `succeeded`, `failed`, `skipped`, `ineligible` or `lost`. `output` is the scanner's answer: `verdict`, `tags` or `score` with `reasoning` for a monitor, classifier or scorer, and `title`, `summary` and `chapters` for a summarizer. Every type also returns `confidence`. Use `output_variable` to keep the result, then branch on it. +- **Main environment only.** The server refuses the step in a child environment of a project. +- **Daily caps.** Each workflow can start 100 scans a day, and the project 500, unless the project settings change `replay_vision_scan_rate_limit_per_day` and `replay_vision_scan_team_rate_limit_per_day`. Past a cap the step returns `state: "skipped"` and the workflow continues. +- **Access.** The token needs `session_recording:read` (or `:write`) and `replay_scanner:write`, and the user needs access to recordings and to the saved scanner. Without them, the save fails with a message that names what is missing. + ## Duration strings (`delay_duration`, `max_wait_duration`) Must match `^\d*\.?\d+[dhms]$` — a number plus unit `s` | `m` | `h` | `d`. Examples: `30s`, `30m`, `2h`, `1d`, `1.5d` (=36h). @@ -216,3 +255,4 @@ A `delay` waits either a fixed span or until a date carried by the person or the - [ ] All durations match `^\d*\.?\d+[dhms]$` and dodge the silent per-unit clamp. - [ ] Every `delay` sets exactly one of `delay_duration` and `delay_until`, and no `delay_until` carries hand-written `bytecode`. - [ ] Function inputs are `{key: {value: ...}}`; no hand-written `bytecode` anywhere; no top-level `trigger` field set. +- [ ] A Replay Vision step's `analysis` value is a literal object of one of the five shapes, with no `templating`. diff --git a/skills/creating-surveys/SKILL.md b/skills/creating-surveys/SKILL.md index c38e369..d4fd614 100644 --- a/skills/creating-surveys/SKILL.md +++ b/skills/creating-surveys/SKILL.md @@ -25,12 +25,38 @@ information that changes who receives the survey or how it is delivered. | A shareable hosted form | `external_survey` | A hosted survey link; no in-app targeting | | A custom form built in application code | `api` | The app renders questions and captures survey events | -Prefer one to three questions unless the user requests more. Use a rating plus -an optional open question for NPS/CSAT, or choice questions with at least two -choices. Inspect the current tool schema for question types, scales, branching, +Inspect the current tool schema for question types, scales, branching, and translations instead of guessing their JSON shapes. Minimal starting points are in [examples](references/examples.md). +### Design for responses + +These choices go with higher response rates in in-app surveys. Apply the +question advice by default. Recommend the trigger and audience advice, and ask +before adding a trigger or audience the user did not request, since both change +who receives the survey. The evidence and its limits are in +[response rates](references/response-rates.md). + +- **Trigger on the moment you are asking about.** Recommend showing the survey + after the event that marks it, such as a completed checkout or a used feature, + through `conditions.events`. Event-triggered surveys get about twice the + responses of page-based ones. A URL condition alone does about as well as + showing the survey on every page. Confirm the event exists in the project + before using it. +- **Make the first question one click.** Start with a single choice or a short + rating. Surveys that open with an open-text question get about a fifth of the + responses. Ask "why" in an optional open question after it. +- **Keep it short.** About 4 in 10 people who start a two- or three-question + survey stop before the end. Prefer one to three questions unless the user + requests more. +- **Turn partial responses on** with `enable_partial_responses: true`, so the + first answer is kept when someone stops. It defaults to off when a survey is + created through the API. +- **Recommend a targeted audience.** Surveys shown to a person or cohort + audience get more responses than surveys shown to everyone. +- **Keep any popup delay under 15 seconds.** Longer delays go with fewer + responses. + Survey names, questions, and appearance text are public content. Do not copy private customer details into them without making that visibility clear first. diff --git a/skills/creating-surveys/references/examples.md b/skills/creating-surveys/references/examples.md index 9473d53..53bb27e 100644 --- a/skills/creating-surveys/references/examples.md +++ b/skills/creating-surveys/references/examples.md @@ -10,6 +10,7 @@ against the project before adding them. { "name": "Product recommendation feedback", "type": "popover", + "enable_partial_responses": true, "questions": [ { "type": "rating", @@ -27,7 +28,37 @@ against the project before adding them. ``` This has no audience restriction. Add the user's intended display conditions -and targeting before launch. For CSAT, use a satisfaction question and scale 5. +and targeting before launch. + +## In-app CSAT after an action + +```json +{ + "name": "Export satisfaction", + "type": "popover", + "enable_partial_responses": true, + "questions": [ + { + "type": "rating", + "question": "How satisfied are you with this export?", + "display": "emoji", + "scale": 5 + }, + { + "type": "open", + "question": "What would make it better?", + "optional": true + } + ], + "conditions": { + "events": { "values": [{ "name": "export_completed" }] } + } +} +``` + +Replace `export_completed` with an event the project already captures at that +moment. The survey shows only after the event fires, which is when the person +can answer. ## Hosted feedback form diff --git a/skills/creating-surveys/references/response-rates.md b/skills/creating-surveys/references/response-rates.md new file mode 100644 index 0000000..a2342c3 --- /dev/null +++ b/skills/creating-surveys/references/response-rates.md @@ -0,0 +1,41 @@ +# Response rates by survey design + +These comparisons come from PostHog popover surveys over one week in October 2026. +Response rate is the share of people who saw a survey and answered at least one question. +Each survey counts once, so a few high-traffic surveys do not dominate. + +| Design choice | Typical response rate | Compared with | +| --------------------------------- | --------------------- | ------------------------------------------------- | +| Triggered by an event or action | 7.5% | 2.9% for a URL condition, 2.1% for none | +| First question is a single choice | 8.0% | 3.5% for a rating, 0.8% for open text | +| Shown on mobile | 13.6% | 3.2% on web | +| Middle-center modal | 12.8% | 2.8% for the default bottom-right (`right`) popup | + +The share of responders who finish every question is about 57% for two questions +and about 45% for three or more. The number of questions does not change how +many people start. + +A model that holds the other choices constant keeps these effects: + +- An open-text first question cuts the odds of a response to about a fifth. +- An event trigger about doubles them. +- A centered modal (`middle_center` or `center`) raises them about 1.8 times. +- A choice question first and a targeted audience each raise them about 1.4 to + 1.5 times. +- A popup delay over 15 seconds cuts them by about a quarter. + +The mobile difference, the 0–10 rating scale, and the number of questions do not +hold up once the other choices are held constant. + +## Limits + +- These are correlations. Teams that set up event triggers may also write better + surveys in other ways. +- With partial responses off, a person counts only after finishing every question, + so surveys with partial responses on look better partly because of how they + count. Turn it on to keep first answers, not to raise the rate. +- A centered modal interrupts the user more than a corner popover. Its higher + response rate does not measure what the interruption costs. Suggest it only + when the user wants the most responses from a short, important question. +- API surveys depend on how each app captures `survey shown`, so their rates + are not comparable. Hosted forms count only people who opened the link. diff --git a/skills/diagnosing-experiment-health/references/empty-experiment.md b/skills/diagnosing-experiment-health/references/empty-experiment.md index 3e288f3..cd7b88d 100644 --- a/skills/diagnosing-experiment-health/references/empty-experiment.md +++ b/skills/diagnosing-experiment-health/references/empty-experiment.md @@ -354,22 +354,21 @@ A user who adds `$feature/` to metric events changes nothing in the re Where the property does matter: -- **Legacy experiments** (E13 in `mid-run-changes.md`) attribute metric events by `$feature/`, so events without it are not counted there. +- **Legacy experiments** (E13 in `mid-run-changes.md`) attributed metric events by `$feature/`. They no longer show results, so migrate them instead. - **A custom exposure event** needs it (B4). - **The user's own insight or SQL** that filters or breaks down by `$feature/`. It undercounts events from a server SDK, which adds the property only when the capture call is given the evaluated flags or asked to evaluate them `[MEDIUM]`. It also overcounts in the other direction: the property is on events of everyone whose SDK holds the flag's value, including people who were never exposed, so it is not a filter for participants. **Verify directly.** `is_legacy` in `experiment-get`. -It is true when any primary, secondary or shared metric is legacy, and only a metric whose `kind` is `ExperimentTrendsQuery` or `ExperimentFunnelsQuery` attributes metric events by the property. -When it is false, or the metric in question has another `kind`, look elsewhere for a missing conversion: B2 (ids), B8 (order), D1 in `numbers-vs-sql.md` (scope). +It is true when any primary, secondary or shared metric is legacy. +When it is false, look elsewhere for a missing conversion: B2 (ids), B8 (order), D1 in `numbers-vs-sql.md` (scope). Part of the public docs still asks for the property on metric events. The exposures page states the current rule. ## B12 — An exposure criteria filter that cannot match [MEDIUM] diff --git a/skills/diagnosing-experiment-health/references/interpretation.md b/skills/diagnosing-experiment-health/references/interpretation.md index d3a8708..c217091 100644 --- a/skills/diagnosing-experiment-health/references/interpretation.md +++ b/skills/diagnosing-experiment-health/references/interpretation.md @@ -126,7 +126,7 @@ significant difference, work through: _Sanity check:_ re-aggregate the warehouse table by org/group once (deduped) and compare to the per-user sum; a large gap confirms repeated-row inflation. -5. **Is it a legacy experiment?** An experiment with legacy metrics runs on the older engine (E13 in `mid-run-changes.md`). +5. **Is it a legacy experiment?** A legacy experiment shows no results (E13 in `mid-run-changes.md`). Migrate it before reading an A/A result. **Recommend:** if chance and exposure do not explain the result, investigate instrumentation rather than @@ -181,9 +181,8 @@ PostHog defaults to Bayesian, unless the project's default method says otherwise - **Flat priors.** PostHog uses a non-informative prior. Early swings aren't the prior pushing things around — they're the data being sparse. - **The displayed effect is relative** to the baseline: 10% against 12% conversion reads as +20%. -- **Legacy experiments.** An experiment with legacy metrics (`is_legacy` in `experiment-get`) uses an older engine with - different multivariate semantics and different significance gates. See E13 in `mid-run-changes.md`, and PostHog's - [legacy-methodology docs page](https://posthog.com/docs/experiments/legacy-methodology). +- **Legacy experiments.** Legacy experiments (`is_legacy` in `experiment-get`) no longer show results. + Migrate them with `experiment-migrate`. See E13 in `mid-run-changes.md`. diff --git a/skills/diagnosing-experiment-health/references/mid-run-changes.md b/skills/diagnosing-experiment-health/references/mid-run-changes.md index e8449a6..b4ff6dd 100644 --- a/skills/diagnosing-experiment-health/references/mid-run-changes.md +++ b/skills/diagnosing-experiment-health/references/mid-run-changes.md @@ -335,8 +335,7 @@ _Fix:_ update the SDK. ## E13 — Editability locks (legacy experiments, ended experiments) [HIGH] - **Legacy experiments** (`is_legacy: true` in `experiment-get`) — only the name, the description and the end date can be edited. - A "This is a legacy experiment. Metrics can no longer be edited." notice appears in the UI. - Where it is rolled out, a second notice says that legacy results will stop being available. + Results are no longer calculated. The UI shows a warning banner with a "Migrate experiment" button. - **Launched experiments** — the variant keys can't be edited (E4). If edits are needed, duplicate (E15), or reset (E8) and re-launch. diff --git a/skills/exploring-ai-failures/references/finding-traces.md b/skills/exploring-ai-failures/references/finding-traces.md index 5c7b108..a6b3e13 100644 --- a/skills/exploring-ai-failures/references/finding-traces.md +++ b/skills/exploring-ai-failures/references/finding-traces.md @@ -87,10 +87,15 @@ posthog:llma-evaluation-list { "enabled": true } SELECT toDate(timestamp) AS day, count() AS fails FROM events WHERE event = '$ai_evaluation' AND properties.$ai_evaluation_id = '' - AND properties.$ai_evaluation_result = false AND timestamp >= now() - INTERVAL 30 DAY + AND () AND (isNull(properties.$ai_evaluation_skipped) OR properties.$ai_evaluation_skipped != 'true') + AND (isNull(properties.$ai_evaluation_applicable) OR properties.$ai_evaluation_applicable != 'false') + AND timestamp >= now() - INTERVAL 30 DAY GROUP BY day ORDER BY day ``` +Build `` from the eval's output type and config, as in the fail predicate table in +`exploring-llm-evaluations`. `$ai_evaluation_result = 'false'` only fits a pass/fail eval that is not a detector. + `exploring-llm-evaluations` covers reading eval results in depth. ## Counting failure modes diff --git a/skills/exploring-llm-evaluations/SKILL.md b/skills/exploring-llm-evaluations/SKILL.md index 3227b35..899f537 100644 --- a/skills/exploring-llm-evaluations/SKILL.md +++ b/skills/exploring-llm-evaluations/SKILL.md @@ -101,14 +101,34 @@ For evaluations that don't support N/A, this property may be `null` — treat nu For boolean evaluations, `output_config.true_is_failure: false` maps `true` to pass and `false` to fail. Set it to `true` for detector-style evaluations where `true` means the evaluator found a problem. +### The fail predicate + +Events store the raw result, not pass or fail. PostHog applies the evaluation's current `output_type` and +`output_config` when it reads them. Build `` from the config that `llma-evaluation-get` +returns, and use it wherever the queries below filter for fails: + +| Evaluation | `` | +| ------------------------------------------------ | --------------------------------------------------------------------- | +| `boolean`, `true_is_failure` false or unset | `properties.$ai_evaluation_result = 'false'` | +| `boolean`, `true_is_failure: true` (detector) | `properties.$ai_evaluation_result = 'true'` | +| `numeric`, `passing_rule` `gte` threshold `T` | `toFloat(properties.$ai_evaluation_numeric_result) < T` | +| `numeric`, `passing_rule` `lte` threshold `T` | `toFloat(properties.$ai_evaluation_numeric_result) > T` | +| `categorical`, `passing_rule.categories` `[a,b]` | `NOT (notEmpty() AND hasAll(['a', 'b'], ))` | +| `categorical`, `passing_rule.categories` `[]` | `notEmpty()` | +| `numeric` or `categorical` with no passing rule | None. Read the score or categories directly, there is no pass or fail | + +`` is `JSONExtract(ifNull(properties.$ai_evaluation_categorical_result, '[]'), 'Array(String)')`. +Always add `(isNull(properties.$ai_evaluation_skipped) OR properties.$ai_evaluation_skipped != 'true')`, because a skipped run can still store +`result = false`. Compare these properties to the strings `'true'` and `'false'`: a project without Boolean +property definitions reads them as strings, and a comparison to a bare `true` or `false` then fails. + ## Workflow: investigate why an evaluation is failing -Works the same way for boolean `llm_judge` and `hog` evaluations — the differences +Works the same way for `llm_judge` and `hog` evaluations — the differences only matter when you eventually go to fix the evaluator (edit the prompt vs. edit the Hog source). Sentiment evaluations should be inspected by sentiment label and score rather than pass/fail filters. -The SQL pass/fail examples below use boolean results. -For numeric and categorical evaluations, read their result property and current passing rule instead of `$ai_evaluation_result`. +The queries below take the evaluation's `` from the table above. ### Step 1 — Find the evaluation @@ -130,20 +150,13 @@ before assuming the failure is in the generation. ```sql posthog:execute-sql SELECT - countIf(properties.$ai_evaluation_applicable = false) AS na_count, - countIf( - (properties.$ai_evaluation_applicable IS NULL - OR properties.$ai_evaluation_applicable != false) - AND properties.$ai_evaluation_result = true - ) AS pass_count, - countIf( - (properties.$ai_evaluation_applicable IS NULL - OR properties.$ai_evaluation_applicable != false) - AND properties.$ai_evaluation_result = false - ) AS fail_count + countIf(properties.$ai_evaluation_applicable = 'false') AS na_count, + countIf((isNull(properties.$ai_evaluation_applicable) OR properties.$ai_evaluation_applicable != 'false') AND NOT ()) AS pass_count, + countIf((isNull(properties.$ai_evaluation_applicable) OR properties.$ai_evaluation_applicable != 'false') AND ()) AS fail_count FROM events WHERE event = '$ai_evaluation' AND properties.$ai_evaluation_id = '' + AND (isNull(properties.$ai_evaluation_skipped) OR properties.$ai_evaluation_skipped != 'true') AND timestamp >= now() - INTERVAL 7 DAY ``` @@ -166,11 +179,9 @@ SELECT FROM events WHERE event = '$ai_evaluation' AND properties.$ai_evaluation_id = '' - AND properties.$ai_evaluation_result = false - AND ( - properties.$ai_evaluation_applicable IS NULL - OR properties.$ai_evaluation_applicable != false - ) + AND () + AND (isNull(properties.$ai_evaluation_applicable) OR properties.$ai_evaluation_applicable != 'false') + AND (isNull(properties.$ai_evaluation_skipped) OR properties.$ai_evaluation_skipped != 'true') AND timestamp >= now() - INTERVAL 7 DAY ORDER BY timestamp DESC LIMIT 25 @@ -344,7 +355,9 @@ identical. FROM events WHERE event = '$ai_evaluation' AND properties.$ai_evaluation_id = '' - AND properties.$ai_evaluation_result = false + AND () + AND (isNull(properties.$ai_evaluation_applicable) OR properties.$ai_evaluation_applicable != 'false') + AND (isNull(properties.$ai_evaluation_skipped) OR properties.$ai_evaluation_skipped != 'true') AND timestamp >= now() - INTERVAL 30 DAY GROUP BY day ORDER BY day @@ -354,8 +367,8 @@ identical. ### "Are passes and fails caused by the same root content?" -1. Pull two samples with the Step 3 query, flipping `$ai_evaluation_result` between - `true` and `false` +1. Pull two samples with the Step 3 query, once with `` and once with + `NOT ()` 2. If the passing and failing runs describe similar content: - For an `llm_judge`: the prompt or rubric is probably ambiguous — reword `evaluation_config.prompt` and use `llma-evaluation-update` @@ -390,7 +403,7 @@ SELECT FROM events WHERE event = '$ai_evaluation' AND properties.$ai_evaluation_id = '' - AND properties.$ai_evaluation_applicable = false + AND properties.$ai_evaluation_applicable = 'false' AND timestamp >= now() - INTERVAL 7 DAY ORDER BY timestamp DESC LIMIT 25 diff --git a/skills/implementing-api-surveys/SKILL.md b/skills/implementing-api-surveys/SKILL.md new file mode 100644 index 0000000..60dcee1 --- /dev/null +++ b/skills/implementing-api-surveys/SKILL.md @@ -0,0 +1,130 @@ +--- +name: implementing-api-surveys +description: > + Implements a PostHog API (headless) survey inside the user's own codebase so it renders with + their UI components and captures the exact events PostHog expects, then verifies the first + response arrived. Use when asked to implement a survey, build a headless survey, build a custom + survey UI, render a survey in my app, or wire up a survey of type `api` in a browser, server, + or custom mobile UI. Covers the posthog-js event contract, server-side SDK capture, and + post-implementation verification through MCP. To design or create the survey definition, use + creating-surveys instead. For a survey that is implemented but not showing or not recording, + use debugging-surveys. +--- + +# Implementing API surveys + +An `api` survey is only a definition in PostHog. PostHog does not render it. The application renders the questions and captures the survey events. If the events or their properties are wrong, responses are missing, double-counted, or unattributed. + +## 1. Get the survey + +Fetch it with `posthog:survey-get` by `id`. Note each question's `id`, `type`, `question`, choices, scale and `optional`, plus `current_iteration`. Never invent question ids. + +If the survey does not exist yet, create it first with the `creating-surveys` skill (type `api`). Design guidance (short, rating or choice first, open text second) lives there too. Do not restate it here. + +## 2. Read the user's codebase + +Before writing UI, find: + +- The framework and the existing component library or design system. Build the survey from those components, not a generic modal. +- Where PostHog is initialized, and the installed SDK and version (`package.json` or lockfile, or the server language's manifest). +- Where the survey should appear. Let the app own the trigger: leave the survey's event, URL and selector `conditions` empty, and run the lookup below at the moment the survey should appear. `getActiveMatchingSurveys` only returns a survey with event conditions after that event fires, and nothing runs the lookup again at that point. + +Pick the path: + +- Browser app with posthog-js: continue below. +- Server-rendered pages, other SDKs (Python, Node, Go, Ruby, PHP), or custom mobile UI: read [server-and-mobile.md](references/server-and-mobile.md), then return to step 5. Mobile SDKs render only popover surveys natively, so a custom mobile UI follows the server pattern. + +## 3. Browser contract (posthog-js) + +Require posthog-js 1.249.2 or newer. Older versions do not set the person properties and seen state on capture. Tell the user to upgrade rather than adding them by hand. + +The lookup only works once both flags and the surveys bundle have loaded, because targeting is evaluated with flags and `getActiveMatchingSurveys` does nothing before surveys load. Register one retry for each at startup, and guard against rendering twice, since `onFeatureFlags` also runs on every flag reload (after `identify`, and after each response). + +```js +let triggered = false +let rendered = false + +function tryShowSurvey() { + if (!triggered || rendered) return + posthog.getActiveMatchingSurveys((surveys) => { + const survey = surveys.find((s) => s.id === SURVEY_ID) + if (!survey || rendered) return + rendered = true + renderSurvey(survey) + }) +} + +// Register once at startup, not per trigger. +posthog.onFeatureFlags(tryShowSurvey) +posthog.onSurveysLoaded(tryShowSurvey) + +function showSurveyIfEligible() { + triggered = true + tryShowSurvey() +} + +// Call when the survey closes, after the final send or a dismiss. +function onSurveyClosed() { + triggered = false + rendered = false +} +``` + +Call `showSurveyIfEligible()` from the app's trigger, such as after a checkout completes, or right after setup if the survey belongs on page load. Call `onSurveyClosed()` when the survey closes, so a survey that can repeat shows again on the next trigger. A survey that cannot repeat stays hidden, because posthog-js marks it as seen. In a single-page app, register the two listeners where the app initializes PostHog, and call the functions they return to unsubscribe if that code can unmount. `SURVEY_ID` is the id from step 1. `renderSurvey` mounts the component you build from the app's own UI, and that component calls the handlers below. `getActiveMatchingSurveys` applies the survey's targeting for this person, so render only what it returns. + +Capture three events. `answers` maps each question id to its answer, and holds every answer given so far in this submission. Exact property names and value shapes are in [event-contract.md](references/event-contract.md). + +```js +function onShown(survey) { + posthog.capture('survey shown', { + $survey_id: survey.id, + $survey_iteration: survey.current_iteration, + }) + // posthog-js reads this key for the survey wait period; only its own renderer writes it. + try { + localStorage.setItem('lastSeenSurveyDate', new Date().toISOString()) + } catch {} +} + +function onSubmit(survey, answers, submissionId, completed) { + posthog.capture('survey sent', { + $survey_id: survey.id, + $survey_iteration: survey.current_iteration, + $survey_submission_id: submissionId, + $survey_completed: completed, + $survey_questions: survey.questions.map(({ id, question }) => ({ id, question })), + ...Object.fromEntries( + Object.entries(answers).map(([questionId, value]) => [`$survey_response_${questionId}`, value]) + ), + }) +} + +function onDismiss(survey) { + posthog.capture('survey dismissed', { + $survey_id: survey.id, + $survey_iteration: survey.current_iteration, + }) +} +``` + +Rules that cause silent data problems: + +- Generate one `$survey_submission_id` (`crypto.randomUUID()`) when the respondent starts, and reuse it for every send of that submission. PostHog deduplicates and bills responses by this id. Events without it each count as a separate response. +- `$survey_completed` is `false` for a partial send and `true` for the final send. +- Send every answer so far on each send, not only the newest one. PostHog keeps the latest event per submission id. +- Ratings go as a number, as posthog-js sends them. Multiple choice goes as an array of strings. Send a `$survey_response_` key only for answered questions. +- Dismiss means close without answering. Do not send `survey dismissed` after a final `survey sent`. + +## 4. Do not set seen state by hand in the browser + +posthog-js sets `$survey_responded/[/]` or `$survey_dismissed/...` and the local seen flag when it captures `survey sent` or `survey dismissed`. That is what stops the survey being offered again. Adding those properties yourself duplicates it. + +## 5. Verify the first response + +Ask the user to trigger the survey once in the real app and submit it. Then: + +1. Call `posthog:surveys-responses-list` for the survey `id`. Rows resolve question text server-side. Confirm the answer appears under the right question, and that multiple choice answers arrive as a list. +2. Call `posthog:survey-stats` for the survey `id`. Confirm `shown` and `sent` counts moved, and that one submission is one response, not two. +3. If nothing arrives, check the project token and that `survey sent` appears in the live events, then follow `debugging-surveys`. Do not guess. + +Report what you implemented, where the trigger lives, and the verified response. diff --git a/skills/implementing-api-surveys/references/event-contract.md b/skills/implementing-api-surveys/references/event-contract.md new file mode 100644 index 0000000..0ee5cc1 --- /dev/null +++ b/skills/implementing-api-surveys/references/event-contract.md @@ -0,0 +1,37 @@ +# Event contract for API surveys + +These are the events and properties posthog-js sends for its own surveys. Other SDKs send the same shapes. + +## Events + +| Event | When | Required properties | +| ------------------ | -------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------- | +| `survey shown` | The survey renders for the person | `$survey_id`, `$survey_iteration` | +| `survey sent` | Each submission, partial or final | `$survey_id`, `$survey_iteration`, `$survey_submission_id`, `$survey_completed`, `$survey_questions`, `$survey_response_` | +| `survey dismissed` | The respondent closes the survey without answering | `$survey_id`, `$survey_iteration` | + +`$survey_iteration` is `survey.current_iteration`. + +## Property values + +- `$survey_submission_id`: one UUID per submission, reused for the partial sends and the final send. +- `$survey_completed`: `true` on the final send, `false` on a partial send. +- `$survey_questions`: `survey.questions.map(({ id, question }) => ({ id, question }))`. +- `$survey_response_`: keyed by the question's id from `posthog:survey-get`. Rating as a number (`9`); a numeric string also works. Multiple choice as an array of strings. Open text as a string. Omit unanswered questions. + +## Person properties that exclude the person afterwards + +| Event | Person property set | +| ------------------ | ---------------------------------------------------------------- | +| `survey sent` | `$survey_responded/` or `$survey_responded//` | +| `survey dismissed` | `$survey_dismissed/` or `$survey_dismissed//` | + +Use the iteration form when `current_iteration` is greater than 0, as posthog-js does. Targeting flags read these keys, so a person who has them set stops matching the survey. posthog-js 1.249.2 and newer sets them on capture in the browser. Server SDKs must set them with `$set`. + +## Counting and billing + +The responses view and billing deduplicate `survey sent` events by `$survey_submission_id`. The latest event for a submission id wins. An event with no submission id counts as its own response. + +## Reading responses back + +Use `posthog:surveys-responses-list` instead of raw SQL. It maps each `$survey_response_` key to its question text. diff --git a/skills/implementing-api-surveys/references/server-and-mobile.md b/skills/implementing-api-surveys/references/server-and-mobile.md new file mode 100644 index 0000000..eb0fabf --- /dev/null +++ b/skills/implementing-api-surveys/references/server-and-mobile.md @@ -0,0 +1,88 @@ +# Server-side and custom mobile implementations + +Use this when the app does not run posthog-js in the page that renders the survey: server-rendered pages, Python, Node, Go, Ruby or PHP backends, or a custom UI in a mobile app. Mobile SDKs render only popover surveys natively. + +## Fetch the survey + +```text +GET /api/surveys/?token= +``` + +Use the PostHog host for the project's region (or the user's proxy). The project token is the public project API key, not a personal API key. The response is `{"surveys": [...]}` and holds running, non-archived surveys. It does not apply per-person eligibility. Find the survey by `id`, then decide eligibility with the server SDK's feature flag evaluation for that person: + +- `linked_flag_key`, when set, must be enabled for the person. If `conditions.linkedFlagVariant` is set to a variant, the flag must return that variant. The value `any` accepts every enabled variant. +- `targeting_flag_key`, when set, must be enabled. +- `internal_targeting_flag_key` must be enabled, unless the survey can show repeatedly (`schedule` is `always`, or `conditions.events.repeatedActivation` is true with events set) or the person has a partial submission in progress that they are finishing. It turns off for people who already responded, dismissed, or saw a survey within the wait period, based on the person properties below. + +Skip any flag that is null. URL, selector and device conditions describe a browser page, so apply them in your own routing if the survey uses them. + +A repeatable survey skips the internal flag, and with it the wait period. If it sets `conditions.seenSurveyWaitPeriodInDays`, store when you last showed a survey to each person, and skip the survey until that many days have passed. + +## Capture the events + +Capture the same events as the browser, using the properties in [event-contract.md](event-contract.md). Identify the person with `distinct_id`. Set the person properties that posthog-js would set: `$survey_last_seen_date` on `survey shown`, which the wait period reads, and the responded or dismissed key that excludes the person from the survey: + +```js +// Node example. Other server SDKs take the same event, properties and $set. +function sendSurvey(posthog, distinctId, survey, answers, submissionId, completed) { + posthog.capture({ + distinctId, + event: 'survey sent', + properties: { + $survey_id: survey.id, + $survey_iteration: survey.current_iteration, + $survey_submission_id: submissionId, + $survey_completed: completed, + $survey_questions: survey.questions.map(({ id, question }) => ({ id, question })), + ...Object.fromEntries( + Object.entries(answers).map(([questionId, value]) => [`$survey_response_${questionId}`, value]) + ), + $set: { [interactionKey(survey, 'responded')]: true }, + }, + }) +} + +function dismissSurvey(posthog, distinctId, survey) { + posthog.capture({ + distinctId, + event: 'survey dismissed', + properties: { + $survey_id: survey.id, + $survey_iteration: survey.current_iteration, + $set: { [interactionKey(survey, 'dismissed')]: true }, + }, + }) +} + +function showSurvey(posthog, distinctId, survey) { + posthog.capture({ + distinctId, + event: 'survey shown', + properties: { + $survey_id: survey.id, + $survey_iteration: survey.current_iteration, + $set: { $survey_last_seen_date: new Date().toISOString() }, + }, + }) +} + +function interactionKey(survey, kind) { + const suffix = survey.current_iteration > 0 ? `/${survey.current_iteration}` : '' + return `$survey_${kind}/${survey.id}${suffix}` +} +``` + +This matches how posthog-js builds the key. + +## Browser rendering a server-sent survey + +If a browser with posthog-js displays a survey your server chose, either path works: + +- Capture the events through posthog-js as in SKILL.md. posthog-js then sets the person properties and the local seen state itself. +- Capture them on the server as above, and call `posthog.surveys.markSurveyAsSeen(survey.id, { iteration: survey.current_iteration })` (posthog-js 1.396.6 and newer) in the browser so the local seen state and wait period still apply. + +## Gotchas + +- Send `$survey_submission_id` on every `survey sent`, or each event bills as its own response. +- Without the `$set` person property, targeting keeps offering the survey to the same person. +- Native mobile popovers capture their own events. This file covers custom UI only. diff --git a/skills/instrument-integration/references/EXAMPLE-expo.md b/skills/instrument-integration/references/EXAMPLE-expo.md index 806788a..1acd12b 100644 --- a/skills/instrument-integration/references/EXAMPLE-expo.md +++ b/skills/instrument-integration/references/EXAMPLE-expo.md @@ -25,7 +25,7 @@ A React Native Expo app demonstrating PostHog product analytics integration with ## Project Structure ``` -basics/expo/ +example-apps/expo/ ├── app/ # Expo Router screens (file-based routing) │ ├── _layout.tsx # Root layout with PostHogProvider + AuthProvider │ ├── index.tsx # Home screen (login/welcome) @@ -77,7 +77,7 @@ Then run `source ~/.zshrc` to apply. 1. Install dependencies: ```bash - cd basics/expo + cd example-apps/expo npm install ``` diff --git a/skills/instrument-integration/references/EXAMPLE-fastapi.md b/skills/instrument-integration/references/EXAMPLE-fastapi.md index 49c19f9..d504534 100644 --- a/skills/instrument-integration/references/EXAMPLE-fastapi.md +++ b/skills/instrument-integration/references/EXAMPLE-fastapi.md @@ -117,7 +117,7 @@ The `/api/test-error` endpoint demonstrates manual exception capture. Use `?capt ## Project Structure ``` -basics/fastapi/ +example-apps/fastapi/ ├── app/ │ ├── __init__.py # Package marker │ ├── config.py # Pydantic Settings configuration diff --git a/skills/instrument-integration/references/EXAMPLE-flask.md b/skills/instrument-integration/references/EXAMPLE-flask.md index a576bf3..0598712 100644 --- a/skills/instrument-integration/references/EXAMPLE-flask.md +++ b/skills/instrument-integration/references/EXAMPLE-flask.md @@ -117,7 +117,7 @@ The `/api/test-error` endpoint demonstrates manual exception capture. Use `?capt ## Project Structure ``` -basics/flask/ +example-apps/flask/ ├── app/ │ ├── __init__.py # Application factory │ ├── config.py # Configuration classes diff --git a/skills/instrument-integration/references/EXAMPLE-javascript-node.md b/skills/instrument-integration/references/EXAMPLE-javascript-node.md index 694f8cd..4ed6b29 100644 --- a/skills/instrument-integration/references/EXAMPLE-javascript-node.md +++ b/skills/instrument-integration/references/EXAMPLE-javascript-node.md @@ -91,7 +91,7 @@ The app tracks these events in PostHog: ## Code structure ``` -basics/javascript-node/ +example-apps/javascript-node/ ├── todo.js # Express server with PostHog tracking ├── package.json # Node.js dependencies ├── .env.example # Environment variable template diff --git a/skills/instrument-integration/references/EXAMPLE-javascript-web.md b/skills/instrument-integration/references/EXAMPLE-javascript-web.md index b0ddf9c..6749c93 100644 --- a/skills/instrument-integration/references/EXAMPLE-javascript-web.md +++ b/skills/instrument-integration/references/EXAMPLE-javascript-web.md @@ -68,7 +68,7 @@ The app tracks these custom events in PostHog (in addition to autocaptured click ## Code Structure ``` -basics/javascript/ +example-apps/javascript-web/ ├── index.html # Entry HTML page ├── package.json # Dependencies (posthog-js, vite) ├── vite.config.js # Vite configuration diff --git a/skills/instrument-integration/references/EXAMPLE-laravel.md b/skills/instrument-integration/references/EXAMPLE-laravel.md index 2248be5..e3435ac 100644 --- a/skills/instrument-integration/references/EXAMPLE-laravel.md +++ b/skills/instrument-integration/references/EXAMPLE-laravel.md @@ -195,7 +195,7 @@ The `/api/test-error` endpoint demonstrates manual exception capture. Use `?capt ## Project Structure ``` -basics/laravel/ +example-apps/laravel/ ├── app/ │ ├── Http/ │ │ ├── Controllers/ @@ -1251,7 +1251,7 @@ This document summarizes the implementation of the Laravel PostHog example appli ## 📁 File Structure ``` -basics/laravel/ +example-apps/laravel/ ├── app/ │ ├── Http/ │ │ ├── Controllers/ diff --git a/skills/instrument-integration/references/EXAMPLE-php.md b/skills/instrument-integration/references/EXAMPLE-php.md index 379c8c7..06362a5 100644 --- a/skills/instrument-integration/references/EXAMPLE-php.md +++ b/skills/instrument-integration/references/EXAMPLE-php.md @@ -80,7 +80,7 @@ The app tracks these events in PostHog: ## Code Structure ``` -basics/php/ +example-apps/php/ ├── todo.php # Main CLI application ├── composer.json # PHP dependencies ├── .env.example # Environment variable template diff --git a/skills/instrument-integration/references/EXAMPLE-python.md b/skills/instrument-integration/references/EXAMPLE-python.md index 6747ba5..26c0c27 100644 --- a/skills/instrument-integration/references/EXAMPLE-python.md +++ b/skills/instrument-integration/references/EXAMPLE-python.md @@ -85,7 +85,7 @@ The app tracks these events in PostHog: ## Code Structure ``` -basics/python/ +example-apps/python/ ├── todo.py # Main CLI application ├── requirements.txt # Python dependencies ├── .env.example # Environment variable template diff --git a/skills/instrument-integration/references/EXAMPLE-ruby.md b/skills/instrument-integration/references/EXAMPLE-ruby.md index f01cfdd..6cd38f0 100644 --- a/skills/instrument-integration/references/EXAMPLE-ruby.md +++ b/skills/instrument-integration/references/EXAMPLE-ruby.md @@ -83,7 +83,7 @@ The app tracks these events in PostHog: ## Code Structure ``` -basics/ruby/ +example-apps/ruby/ ├── todo.rb # Main CLI application ├── Gemfile # Ruby dependencies ├── .env.example # Environment variable template diff --git a/skills/instrument-integration/references/js.md b/skills/instrument-integration/references/js.md index 7fe7afa..3037fad 100644 --- a/skills/instrument-integration/references/js.md +++ b/skills/instrument-integration/references/js.md @@ -214,6 +214,7 @@ posthog.init('', { | `ToolbarExtensions` | [Toolbar](/docs/toolbar.md) | | `LogsExtensions` | [Log capture](/docs/logs.md) | | `ConversationsExtensions` | [Support](/docs/support.md) | +| `WebMCPExtensions` | [WebMCP tool call capture](/docs/mcp-analytics/installation/webmcp.md) for MCP Analytics. Add `ErrorTrackingExtensions` to also send failed calls to Error Tracking. | | `AllExtensions` | Everything (equivalent to the default `posthog-js` bundle) | **Note:** Each extension bundle includes its own dependencies. You don't need to worry about adding them separately. diff --git a/skills/instrument-integration/references/posthog-python.md b/skills/instrument-integration/references/posthog-python.md index 04cc7e9..addfd19 100644 --- a/skills/instrument-integration/references/posthog-python.md +++ b/skills/instrument-integration/references/posthog-python.md @@ -2,7 +2,7 @@ # PostHog Python SDK -**SDK Version:** 7.68.0 +**SDK Version:** 7.70.0 Integrate PostHog into any python application. diff --git a/skills/instrument-llm-analytics/references/anthropic.md b/skills/instrument-llm-analytics/references/anthropic.md index 225e2ba..43c8259 100644 --- a/skills/instrument-llm-analytics/references/anthropic.md +++ b/skills/instrument-llm-analytics/references/anthropic.md @@ -2,6 +2,400 @@ # Anthropic AI Observability installation +## Claude Platform Beta + +Claude Platform+PostHog + +### Become a trace explorer + +Send OpenTelemetry traces from Claude to PostHog with no extra code. Add PostHog as a destination in the Claude Console and run evals on live traces and ship fixes that actually make a difference. + +100K free events, every month + +![Explorer hedgehog holding a folded map](/images/docs/ai-observability/explorer-hedgehog.png) + +Click the map + +1. 1 + + ## Copy your project token + + Required + + Log in to see your project token here + + [Log in to PostHog](https://app.posthog.com/login)[or open your project settings](https://app.posthog.com/settings/project#variables) + + A personal API key (`phx_...`) won't work here. + +2. 2 + + ## Choose how to send traces + + Required + + The Claude Platform can send traces from two places. Pick one for each PostHog project. If you use both, each request reaches PostHog twice. + + - **SDK tracing:** you set it up in your code with the Claude SDK. It covers the calls your app makes and the tools it runs itself. You can link traces to your users and add custom properties. + - **Console destination:** you add PostHog in the Claude Console, with no code changes. It covers your Messages API requests, server tools such as web search, and Managed Agents sessions. + + ## SDK tracing + + Install the Claude SDK with the steps in Anthropic's [OpenTelemetry guide](https://platform.claude.com/docs/en/manage-claude/opentelemetry/sdk#install-the-sdk). Then install the OpenTelemetry packages. PostHog has a span processor for Python, Node, and Go. Java and Ruby use the standard OTLP exporter. + + ### Python + + ```bash + pip install "posthog[otel]" + ``` + + ### Node + + ```bash + npm install @posthog/ai @opentelemetry/sdk-trace-node @opentelemetry/resources @opentelemetry/exporter-trace-otlp-http + ``` + + ### Go + + ```bash + go get github.com/posthog/posthog-go/otel go.opentelemetry.io/otel go.opentelemetry.io/otel/sdk + ``` + + ### Java + + ```kotlin + dependencies { + implementation(platform("io.opentelemetry:opentelemetry-bom:1.57.0")) + implementation("io.opentelemetry:opentelemetry-sdk") + implementation("io.opentelemetry:opentelemetry-exporter-otlp") + } + ``` + + ### Ruby + + ```bash + bundle add opentelemetry-sdk opentelemetry-exporter-otlp + ``` + + Register a tracer provider before your first Claude call. Set the content mode to see prompts and responses, because the SDK sends metadata only by default: + + ### Python + + ```python + from anthropic import Anthropic + from opentelemetry import trace + from opentelemetry.sdk.resources import SERVICE_NAME, Resource + from opentelemetry.sdk.trace import TracerProvider + from posthog.ai.otel import PostHogSpanProcessor + + resource = Resource(attributes={ + SERVICE_NAME: "my-app", + "posthog.distinct_id": "user_123", # optional: identifies the user in PostHog + "foo": "bar", # custom properties are passed through + }) + + provider = TracerProvider(resource=resource) + provider.add_span_processor( + PostHogSpanProcessor( + api_key="", + host="https://us.i.posthog.com", + ) + ) + trace.set_tracer_provider(provider) + + client = Anthropic(open_telemetry={"traces": {"content_mode": "content"}}) + + message = client.messages.create( + model="claude-opus-5", + max_tokens=1024, + messages=[{"role": "user", "content": "Tell me about hedgehogs"}], + ) + + # Flush the batched spans before a script exits. + provider.shutdown() + ``` + + ### Node + + ```typescript + import Anthropic from '@anthropic-ai/sdk' + import { resourceFromAttributes } from '@opentelemetry/resources' + import { NodeTracerProvider } from '@opentelemetry/sdk-trace-node' + import { PostHogSpanProcessor } from '@posthog/ai/otel' + + const provider = new NodeTracerProvider({ + resource: resourceFromAttributes({ + 'service.name': 'my-app', + 'posthog.distinct_id': 'user_123', // optional: identifies the user in PostHog + foo: 'bar', // custom properties are passed through + }), + spanProcessors: [ + new PostHogSpanProcessor({ + projectToken: '', + host: 'https://us.i.posthog.com', + }), + ], + }) + provider.register() + + const client = new Anthropic({ openTelemetry: { traces: { contentMode: 'content' } } }) + + const message = await client.messages.create({ + model: 'claude-opus-5', + max_tokens: 1024, + messages: [{ role: 'user', content: 'Tell me about hedgehogs' }], + }) + + // Flush the batched spans before a script exits. + await provider.shutdown() + ``` + + ### Go + + ```go + package main + + import ( + "context" + "fmt" + + "github.com/anthropics/anthropic-sdk-go" + "github.com/anthropics/anthropic-sdk-go/option" + posthogotel "github.com/posthog/posthog-go/otel" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/propagation" + sdkresource "go.opentelemetry.io/otel/sdk/resource" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + ) + + func main() { + ctx := context.Background() + + processor, err := posthogotel.NewSpanProcessor(ctx, "", + posthogotel.WithHost("https://us.i.posthog.com"), + ) + if err != nil { + panic(err) + } + + resource := sdkresource.NewWithAttributes("", + attribute.String("service.name", "my-app"), + attribute.String("posthog.distinct_id", "user_123"), // optional: identifies the user in PostHog + attribute.String("foo", "bar"), // custom properties are passed through + ) + provider := sdktrace.NewTracerProvider( + sdktrace.WithResource(resource), + sdktrace.WithSpanProcessor(processor), + ) + // Flush the batched spans before the program exits. + defer provider.Shutdown(ctx) + otel.SetTracerProvider(provider) + otel.SetTextMapPropagator(propagation.TraceContext{}) + + client := anthropic.NewClient( + option.WithOpenTelemetryTracesContentMode(option.OpenTelemetryContentModeContent), + ) + + message, err := client.Messages.New(ctx, anthropic.MessageNewParams{ + Model: anthropic.ModelClaudeOpus5, + MaxTokens: 1024, + Messages: []anthropic.MessageParam{ + anthropic.NewUserMessage(anthropic.NewTextBlock("Tell me about hedgehogs")), + }, + }) + if err != nil { + panic(err) + } + fmt.Println(message.StopReason) + } + ``` + + ### Java + + ```java + import com.anthropic.client.okhttp.AnthropicOkHttpClient; + import com.anthropic.core.opentelemetry.OpenTelemetryContentMode; + import com.anthropic.core.opentelemetry.OpenTelemetryOptions; + import com.anthropic.models.messages.MessageCreateParams; + import com.anthropic.models.messages.Model; + import io.opentelemetry.api.common.AttributeKey; + import io.opentelemetry.api.common.Attributes; + import io.opentelemetry.api.trace.propagation.W3CTraceContextPropagator; + import io.opentelemetry.context.propagation.ContextPropagators; + import io.opentelemetry.exporter.otlp.http.trace.OtlpHttpSpanExporter; + import io.opentelemetry.sdk.OpenTelemetrySdk; + import io.opentelemetry.sdk.resources.Resource; + import io.opentelemetry.sdk.trace.SdkTracerProvider; + import io.opentelemetry.sdk.trace.export.BatchSpanProcessor; + + public class Main { + public static void main(String[] args) { + var exporter = OtlpHttpSpanExporter.builder() + .setEndpoint("https://us.i.posthog.com/i/v0/ai/otel") + .addHeader("Authorization", "Bearer ") + .build(); + + var resource = Resource.getDefault().merge(Resource.create(Attributes.of( + AttributeKey.stringKey("service.name"), "my-app", + AttributeKey.stringKey("posthog.distinct_id"), "user_123", // optional: identifies the user in PostHog + AttributeKey.stringKey("foo"), "bar"))); // custom properties are passed through + + var tracerProvider = SdkTracerProvider.builder() + .setResource(resource) + .addSpanProcessor(BatchSpanProcessor.builder(exporter).build()) + .build(); + OpenTelemetrySdk.builder() + .setTracerProvider(tracerProvider) + .setPropagators(ContextPropagators.create(W3CTraceContextPropagator.getInstance())) + .buildAndRegisterGlobal(); + + var envOptions = OpenTelemetryOptions.fromEnv(); + var client = AnthropicOkHttpClient.builder() + .fromEnv() + .openTelemetry(envOptions.toBuilder() + .traces(envOptions.traces().toBuilder() + .contentMode(OpenTelemetryContentMode.CONTENT) + .build()) + .build()) + .build(); + + var message = client.messages().create(MessageCreateParams.builder() + .model(Model.CLAUDE_OPUS_5) + .maxTokens(1024) + .addUserMessage("Tell me about hedgehogs") + .build()); + + // Flush the batched spans before the program exits. + tracerProvider.close(); + } + } + ``` + + ### Ruby + + ```ruby + require "anthropic" + require "opentelemetry/sdk" + require "opentelemetry/exporter/otlp" + + OpenTelemetry::SDK.configure do |config| + config.resource = OpenTelemetry::SDK::Resources::Resource.create( + "service.name" => "my-app", + "posthog.distinct_id" => "user_123", # optional: identifies the user in PostHog + "foo" => "bar" # custom properties are passed through + ) + config.add_span_processor( + OpenTelemetry::SDK::Trace::Export::BatchSpanProcessor.new( + OpenTelemetry::Exporter::OTLP::Exporter.new( + endpoint: "https://us.i.posthog.com/i/v0/ai/otel", + headers: {"Authorization" => "Bearer "} + ) + ) + ) + end + + client = Anthropic::Client.new(open_telemetry: {traces: {content_mode: :content}}) + + message = client.messages.create( + model: Anthropic::Model::CLAUDE_OPUS_5, + max_tokens: 1024, + messages: [{role: "user", content: "Tell me about hedgehogs"}] + ) + + # Flush the batched spans before a script exits. + OpenTelemetry.tracer_provider.shutdown + ``` + + You can also set the content mode with the `ANTHROPIC_OPEN_TELEMETRY_TRACES_CONTENT_MODE=content` environment variable. + + To group several calls into one session, set a conversation ID on the client or the request, as described in [Configure the SDK](https://platform.claude.com/docs/en/manage-claude/opentelemetry/sdk#configure-the-sdk). PostHog uses it as the session ID. + + ## Console destination + + API tracing does not yet support person identification or custom properties. Anthropic is exploring ways to support this in the future. + + In the Claude Console, open the [OpenTelemetry settings](https://platform.claude.com/settings/telemetry-destinations) of your organization and click **Add destination**. You need an admin role. + + 1. **Provider:** PostHog. + 2. **Region:** US if you log in at `us.posthog.com`, EU if you log in at `eu.posthog.com`. + 3. **Project token:** paste your `phc_` token. + 4. **Content mode:** pick **Full content** to see every prompt and response in PostHog. **Metadata only** sends tokens, costs, and timing without the messages. + 5. **Workspaces:** choose which workspaces send their traces to PostHog: **All**, **Selected**, or **All except**. + + ![The Add destination form in the Claude Console with PostHog selected as the provider](/images/docs/ai-observability/claude-platform-add-destination.png) + + Click **Add destination**. Your code doesn't change. + + Anthropic's [OpenTelemetry guide](https://platform.claude.com/docs/en/manage-claude/opentelemetry) covers workspaces, delivery health, and every span and attribute. + +3. 3 + + ## Call Claude and see the trace + + Required + + Call Claude the way you already do, then open [AI Observability](https://app.posthog.com/ai-observability/traces). Each request shows up as an `anthropic.messages.create` trace, with its cost, tokens, and messages. + + - **SDK tracing:** if you use the tool runner, each tool it runs gets its own span. + - **Console destination:** the trace holds an `anthropic.model_request` span for each model request. A request can have more than one, for example when the model uses web search. + + **Using Managed Agents?** A Console destination also sends your sessions, with a span for each turn, model request, and tool use: + + ![A Managed Agents trace in PostHog: session turns with model requests and bash tool uses on a timeline, and the turn input and output](/images/docs/ai-observability/claude-platform-managed-agents-trace.png) + + The turn holds the agent's input and output, each model request holds its tokens and cost, and each tool use holds its arguments and result. A session can span several traces. To follow one session, filter by `anthropic.session.id`. + +4. 4 + + ## Keep an eye on your agent + + Recommended + + Your traces are in. Now put them to work. + + Evaluations + + Quickly monitor how your agent is doing. Score every response for helpfulness, hallucinations, or tone, and see when quality drops. + + [Set up evaluations](/docs/ai-evals.md) + + Dashboard + + Follow cost, usage, and latency over time, broken down by model or workspace. + + [Explore the dashboard](/docs/ai-observability/dashboard.md) + + Clusters + + Group similar conversations to see what people ask your agent most. + + [Find patterns](/docs/ai-observability/clusters.md) + + Sentiment + + See how users feel while they talk to your agent. + + [Track sentiment](/docs/ai-observability/sentiment.md) + +5. 5 + + ## Troubleshooting + + Optional + + - **Delivery shows Failing with a 401:** the token in the Claude Console is a personal API key. Use your project token. + - **Delivery shows Healthy, but PostHog is empty:** check that the token comes from this project and that the region matches your PostHog cloud. + - **No traces from a Console destination:** check that the destination's **Workspaces** setting covers the workspace of your API key. Traces can take up to a minute to arrive. + - **No traces from the Claude SDK:** register the tracer provider before the first Claude call. `ANTHROPIC_OPEN_TELEMETRY=false` turns the spans off. + - **Traces stop at the end of a script:** shut down the tracer provider before the script exits, as the samples do, so the last batch is sent. + - **No prompts or responses:** in the Claude Console, the destination uses **Metadata only**. Organizations with a data-handling restriction, such as HIPAA, always export metadata only. With the Claude SDK, set the content mode to `content` on the client. + - **Each request appears twice:** a Claude Console destination and the Claude SDK both send this project the same requests. Remove one of the two. + + For other questions, see [AI Observability troubleshooting](/docs/ai-observability/troubleshooting.md). + +## Tab + ![](https://res.cloudinary.com/dmukukwp6/image/upload/texture_tan_9608fcca70) ![](https://res.cloudinary.com/dmukukwp6/image/upload/texture_tan_dark_a92b0e022d) @@ -232,6 +626,16 @@ Skip the manual setup — run this in your project and the wizard installs the S | [Spans](/docs/ai-observability/spans.md) | Review spans and their role in representing individual operations. | | [Anaylze LLM performance](/docs/ai-observability/dashboard.md) | Learn how to create dashboards to analyze LLM performance. | +Tourist info + +## If you just landed here, welcome + +PostHog is the context layer for your product. And since we're advertising anyway: 97% of companies use it for free. + +[Get started](https://app.posthog.com/signup)[Read the docs](/docs.md) + +A hedgehog tourist holding a map and a suitcase + ### Still have questions? Ask PostHog AI diff --git a/skills/instrument-product-analytics/references/EXAMPLE-expo.md b/skills/instrument-product-analytics/references/EXAMPLE-expo.md index 806788a..1acd12b 100644 --- a/skills/instrument-product-analytics/references/EXAMPLE-expo.md +++ b/skills/instrument-product-analytics/references/EXAMPLE-expo.md @@ -25,7 +25,7 @@ A React Native Expo app demonstrating PostHog product analytics integration with ## Project Structure ``` -basics/expo/ +example-apps/expo/ ├── app/ # Expo Router screens (file-based routing) │ ├── _layout.tsx # Root layout with PostHogProvider + AuthProvider │ ├── index.tsx # Home screen (login/welcome) @@ -77,7 +77,7 @@ Then run `source ~/.zshrc` to apply. 1. Install dependencies: ```bash - cd basics/expo + cd example-apps/expo npm install ``` diff --git a/skills/instrument-product-analytics/references/EXAMPLE-fastapi.md b/skills/instrument-product-analytics/references/EXAMPLE-fastapi.md index 49c19f9..d504534 100644 --- a/skills/instrument-product-analytics/references/EXAMPLE-fastapi.md +++ b/skills/instrument-product-analytics/references/EXAMPLE-fastapi.md @@ -117,7 +117,7 @@ The `/api/test-error` endpoint demonstrates manual exception capture. Use `?capt ## Project Structure ``` -basics/fastapi/ +example-apps/fastapi/ ├── app/ │ ├── __init__.py # Package marker │ ├── config.py # Pydantic Settings configuration diff --git a/skills/instrument-product-analytics/references/EXAMPLE-flask.md b/skills/instrument-product-analytics/references/EXAMPLE-flask.md index a576bf3..0598712 100644 --- a/skills/instrument-product-analytics/references/EXAMPLE-flask.md +++ b/skills/instrument-product-analytics/references/EXAMPLE-flask.md @@ -117,7 +117,7 @@ The `/api/test-error` endpoint demonstrates manual exception capture. Use `?capt ## Project Structure ``` -basics/flask/ +example-apps/flask/ ├── app/ │ ├── __init__.py # Application factory │ ├── config.py # Configuration classes diff --git a/skills/instrument-product-analytics/references/EXAMPLE-laravel.md b/skills/instrument-product-analytics/references/EXAMPLE-laravel.md index 2248be5..e3435ac 100644 --- a/skills/instrument-product-analytics/references/EXAMPLE-laravel.md +++ b/skills/instrument-product-analytics/references/EXAMPLE-laravel.md @@ -195,7 +195,7 @@ The `/api/test-error` endpoint demonstrates manual exception capture. Use `?capt ## Project Structure ``` -basics/laravel/ +example-apps/laravel/ ├── app/ │ ├── Http/ │ │ ├── Controllers/ @@ -1251,7 +1251,7 @@ This document summarizes the implementation of the Laravel PostHog example appli ## 📁 File Structure ``` -basics/laravel/ +example-apps/laravel/ ├── app/ │ ├── Http/ │ │ ├── Controllers/ diff --git a/skills/instrument-product-analytics/references/EXAMPLE-php.md b/skills/instrument-product-analytics/references/EXAMPLE-php.md index 379c8c7..06362a5 100644 --- a/skills/instrument-product-analytics/references/EXAMPLE-php.md +++ b/skills/instrument-product-analytics/references/EXAMPLE-php.md @@ -80,7 +80,7 @@ The app tracks these events in PostHog: ## Code Structure ``` -basics/php/ +example-apps/php/ ├── todo.php # Main CLI application ├── composer.json # PHP dependencies ├── .env.example # Environment variable template diff --git a/skills/instrument-product-analytics/references/EXAMPLE-python.md b/skills/instrument-product-analytics/references/EXAMPLE-python.md index 6747ba5..26c0c27 100644 --- a/skills/instrument-product-analytics/references/EXAMPLE-python.md +++ b/skills/instrument-product-analytics/references/EXAMPLE-python.md @@ -85,7 +85,7 @@ The app tracks these events in PostHog: ## Code Structure ``` -basics/python/ +example-apps/python/ ├── todo.py # Main CLI application ├── requirements.txt # Python dependencies ├── .env.example # Environment variable template diff --git a/skills/instrument-product-analytics/references/EXAMPLE-ruby.md b/skills/instrument-product-analytics/references/EXAMPLE-ruby.md index f01cfdd..6cd38f0 100644 --- a/skills/instrument-product-analytics/references/EXAMPLE-ruby.md +++ b/skills/instrument-product-analytics/references/EXAMPLE-ruby.md @@ -83,7 +83,7 @@ The app tracks these events in PostHog: ## Code Structure ``` -basics/ruby/ +example-apps/ruby/ ├── todo.rb # Main CLI application ├── Gemfile # Ruby dependencies ├── .env.example # Environment variable template diff --git a/skills/instrument-product-analytics/references/posthog-python.md b/skills/instrument-product-analytics/references/posthog-python.md index 04cc7e9..addfd19 100644 --- a/skills/instrument-product-analytics/references/posthog-python.md +++ b/skills/instrument-product-analytics/references/posthog-python.md @@ -2,7 +2,7 @@ # PostHog Python SDK -**SDK Version:** 7.68.0 +**SDK Version:** 7.70.0 Integrate PostHog into any python application. diff --git a/skills/managing-experiment-lifecycle/SKILL.md b/skills/managing-experiment-lifecycle/SKILL.md index 22a8e1c..fe923dc 100644 --- a/skills/managing-experiment-lifecycle/SKILL.md +++ b/skills/managing-experiment-lifecycle/SKILL.md @@ -214,10 +214,10 @@ Creating a new experiment and copying the metrics over produces a second feature `experiment-migrate` reuses the original flag, so the migrated experiment keeps its audience from the first minute. - **Preconditions**: the experiment must be legacy. A 400 says it is already on the new engine. -- **What happens**: a new experiment is created with the same configuration and the metrics converted to the new format. The legacy one is left untouched and keeps its results, so the project ends up with two experiments on one feature flag. +- **What happens**: a new experiment is created with the same configuration and the metrics converted to the new format. The legacy one is left untouched, so the project ends up with two experiments on one feature flag. Legacy experiments no longer show results, so migrating is the only way to get results for one. - **Legacy shared metrics**: converted in the same call. Each gets a new shared metric that the migrated experiment links to. - **Variants**: unchanged — the flag is shared, so users keep the variant they have. -- **Analysis**: the migrated experiment reads the same exposures. Results are recomputed by the new engine, so numbers can differ from the legacy view. +- **Analysis**: the migrated experiment reads the same exposures. Results are computed by the new engine, so numbers can differ from what the legacy view used to show. **Tell the user there will be two experiments before you call it**, and link both afterwards. Calling it again returns the experiment the first call created, so a retry is safe. @@ -226,7 +226,7 @@ No request body. A legacy experiment also refuses most edits: `experiment-update` returns 400 for anything but name, description and end_date. Those three still work on the legacy experiment, so edit it directly and do not migrate for them. -Migrate only when the user asks for it, or when a requested change touches a field the guard blocks, and then apply the change to the migrated experiment. +Migrate only when the user asks for it, wants results for the experiment, or requests a change to a field the guard blocks. Apply that change to the migrated experiment. ## Decision framework diff --git a/skills/querying-posthog-data/references/available-functions.md b/skills/querying-posthog-data/references/available-functions.md index 0130046..7622407 100644 --- a/skills/querying-posthog-data/references/available-functions.md +++ b/skills/querying-posthog-data/references/available-functions.md @@ -699,6 +699,7 @@ rowNumberInAllBlocks rowNumberInBlock rpad rtrim +sha1 sign simpleLinearRegression sin diff --git a/skills/querying-posthog-data/references/example-error-tracking.md b/skills/querying-posthog-data/references/example-error-tracking.md index a0a8912..03c5d05 100644 --- a/skills/querying-posthog-data/references/example-error-tracking.md +++ b/skills/querying-posthog-data/references/example-error-tracking.md @@ -25,14 +25,14 @@ FROM argMaxState(properties.$exception_functions.-1, timestamp) AS function_state, argMaxState(properties.$exception_sources.-1, timestamp) AS source_state, argMaxState(properties.$lib, timestamp) AS library_state, - least(19, intDiv(dateDiff('seconds', toDateTime(toDateTime('2026-10-14 12:00:00.000000')), timestamp), greatest(1, intDiv(dateDiff('seconds', toDateTime(toDateTime('2026-10-14 12:00:00.000000')), toDateTime(toDateTime('2026-10-15 12:05:05.406822'))), 20)))) AS bin_idx, + least(19, intDiv(dateDiff('seconds', toDateTime(toDateTime('2026-10-15 11:00:00.000000')), timestamp), greatest(1, intDiv(dateDiff('seconds', toDateTime(toDateTime('2026-10-15 11:00:00.000000')), toDateTime(toDateTime('2026-10-16 11:58:28.764957'))), 20)))) AS bin_idx, count() AS occ, uniqState(nullIf(e.$session_id, '')) AS sessions_state, uniqState(coalesce(nullIf(toString(e.event_person_id), '00000000-0000-0000-0000-000000000000'), e.distinct_id)) AS users_state FROM events AS e WHERE - and(equals(e.event, '$exception'), isNotNull(e.properties.$exception_fingerprint), true, greaterOrEquals(e.timestamp, toDateTime(toDateTime('2026-10-14 12:00:00.000000'))), lessOrEquals(e.timestamp, toDateTime(toDateTime('2026-10-15 12:05:05.406822'))), or(greater(multiSearchAnyCaseInsensitive(toString(e.properties.$exception_types), ['constant']), 0), greater(multiSearchAnyCaseInsensitive(toString(e.properties.$exception_values), ['constant']), 0), greater(multiSearchAnyCaseInsensitive(toString(e.properties.$exception_sources), ['constant']), 0), greater(multiSearchAnyCaseInsensitive(toString(e.properties.$exception_functions), ['constant']), 0), greater(multiSearchAnyCaseInsensitive(toString(e.properties.email), ['constant']), 0), greater(multiSearchAnyCaseInsensitive(toString(e.person.properties.email), ['constant']), 0)), equals(properties.tag, 'max_ai')) + and(equals(e.event, '$exception'), isNotNull(e.properties.$exception_fingerprint), true, greaterOrEquals(e.timestamp, toDateTime(toDateTime('2026-10-15 11:00:00.000000'))), lessOrEquals(e.timestamp, toDateTime(toDateTime('2026-10-16 11:58:28.764957'))), or(greater(multiSearchAnyCaseInsensitive(toString(e.properties.$exception_types), ['constant']), 0), greater(multiSearchAnyCaseInsensitive(toString(e.properties.$exception_values), ['constant']), 0), greater(multiSearchAnyCaseInsensitive(toString(e.properties.$exception_sources), ['constant']), 0), greater(multiSearchAnyCaseInsensitive(toString(e.properties.$exception_functions), ['constant']), 0), greater(multiSearchAnyCaseInsensitive(toString(e.properties.email), ['constant']), 0), greater(multiSearchAnyCaseInsensitive(toString(e.person.properties.email), ['constant']), 0)), equals(properties.tag, 'max_ai')) GROUP BY fp_hash, bin_idx) AS ev diff --git a/skills/querying-posthog-data/references/example-logs.md b/skills/querying-posthog-data/references/example-logs.md index 2e72968..a6a07b2 100644 --- a/skills/querying-posthog-data/references/example-logs.md +++ b/skills/querying-posthog-data/references/example-logs.md @@ -31,7 +31,7 @@ SELECT FROM logs WHERE - and(and(greaterOrEquals(toStartOfDay(time_bucket), toStartOfDay(assumeNotNull(toDateTime('2025-12-09 00:00:00')))), lessOrEquals(toStartOfDay(time_bucket), toStartOfDay(assumeNotNull(toDateTime('2025-12-10 00:00:00'))))), 1, greaterOrEquals(timestamp, toDateTime('2026-10-14 12:05:05.693005')), indexHint(like(lower(body), '%timeout%')), ilike(toString(body), '%timeout%'), in(severity_text, tuple('warn', 'error', 'fatal'))) + and(and(greaterOrEquals(toStartOfDay(time_bucket), toStartOfDay(assumeNotNull(toDateTime('2025-12-09 00:00:00')))), lessOrEquals(toStartOfDay(time_bucket), toStartOfDay(assumeNotNull(toDateTime('2025-12-10 00:00:00'))))), 1, greaterOrEquals(timestamp, toDateTime('2026-10-15 11:58:29.327907')), indexHint(like(lower(body), '%timeout%')), ilike(toString(body), '%timeout%'), in(severity_text, tuple('warn', 'error', 'fatal'))) ORDER BY timestamp DESC, uuid DESC diff --git a/skills/querying-posthog-data/references/example-session-replay.md b/skills/querying-posthog-data/references/example-session-replay.md index 2fc434b..8cb3e99 100644 --- a/skills/querying-posthog-data/references/example-session-replay.md +++ b/skills/querying-posthog-data/references/example-session-replay.md @@ -19,18 +19,18 @@ SELECT sum(s.console_error_count) AS console_error_count, max(s.retention_period_days) AS retention_period_days, plus(dateTrunc('DAY', start_time), toIntervalDay(coalesce(retention_period_days, 30))) AS expiry_time, - date_diff('DAY', toDateTime('2026-10-15 12:05:05.836125'), expiry_time) AS recording_ttl, - greaterOrEquals(max(s._timestamp), toDateTime('2026-10-15 12:00:05.835780')) AS ongoing, + date_diff('DAY', toDateTime('2026-10-16 11:58:29.581816'), expiry_time) AS recording_ttl, + greaterOrEquals(max(s._timestamp), toDateTime('2026-10-16 11:53:29.581277')) AS ongoing, round(least(greatest(multiply(divide(plus(plus(plus(divide(sum(s.active_milliseconds), 1000), sum(s.click_count)), sum(s.keypress_count)), sum(s.console_error_count)), plus(plus(plus(plus(sum(s.mouse_activity_count), dateDiff('SECOND', start_time, end_time)), sum(s.console_error_count)), sum(s.console_log_count)), sum(s.console_warn_count))), 100), 0), 100), 2) AS activity_score, coalesce(max(s.surfacing_score), 0.36) AS surfacing_score FROM raw_session_replay_events AS s WHERE - and(greaterOrEquals(s.min_first_timestamp, toDateTime('2026-10-12 00:00:00.000000')), lessOrEquals(s.min_first_timestamp, toDateTime('2026-10-15 12:05:05.835896'))) + and(greaterOrEquals(s.min_first_timestamp, toDateTime('2026-10-13 00:00:00.000000')), lessOrEquals(s.min_first_timestamp, toDateTime('2026-10-16 11:58:29.581462'))) GROUP BY session_id HAVING - and(greaterOrEquals(expiry_time, toDateTime('2026-10-15 12:05:05.836029')), equals(max(s.is_deleted), 0), greater(active_seconds, 5.0)) + and(greaterOrEquals(expiry_time, toDateTime('2026-10-16 11:58:29.581682')), equals(max(s.is_deleted), 0), greater(active_seconds, 5.0)) ORDER BY start_time DESC, session_id DESC diff --git a/skills/querying-posthog-data/references/example-sessions.md b/skills/querying-posthog-data/references/example-sessions.md index 7d849cf..68d22be 100644 --- a/skills/querying-posthog-data/references/example-sessions.md +++ b/skills/querying-posthog-data/references/example-sessions.md @@ -13,7 +13,7 @@ SELECT FROM sessions WHERE - and(less($start_timestamp, toDateTime('2026-10-15 12:05:12.578040')), greater($start_timestamp, toDateTime('2026-10-14 12:05:07.578345'))) + and(less($start_timestamp, toDateTime('2026-10-16 11:58:36.633342')), greater($start_timestamp, toDateTime('2026-10-15 11:58:31.633864'))) ORDER BY $start_timestamp DESC LIMIT 50000 diff --git a/skills/querying-posthog-data/references/guidelines.md b/skills/querying-posthog-data/references/guidelines.md index 154ff92..6ba7cb9 100644 --- a/skills/querying-posthog-data/references/guidelines.md +++ b/skills/querying-posthog-data/references/guidelines.md @@ -147,7 +147,7 @@ FROM system.information_schema.columns WHERE table_name = 'events' ``` -This works for `system.*` entity tables too — query them by full name, e.g. `WHERE table_name = 'system.insights'`. Their column sets differ per entity, so confirm columns before projecting them. +This works for `system.*` entity tables too — query them by full name, e.g. `WHERE table_name = 'system.insights'`. Their column sets differ per entity and from REST/tool response fields, so confirm columns before projecting them. If a required field is absent, use a tool that exposes it or explain the limitation; do not remove the filter or substitute a different field just to make the query succeed. **Discover how a table joins to others:** @@ -221,13 +221,30 @@ This prevents wasted API calls and gives users immediate feedback when the data ##### Progressive exploration -For unfamiliar or potentially large datasets, probe cheaply before running the expensive aggregation. Widen only if the cheap step looks reasonable: +For unfamiliar or potentially large datasets, use a short timestamp window to check the data and intended query shape before widening it: 1. **Count first** — `SELECT count() FROM events WHERE timestamp >= now() - INTERVAL 1 DAY AND event = 'foo'`. Confirms the data exists and gives a sense of volume. 2. **Small sample** — inspect a handful of rows (`LIMIT 10`) to verify property shapes and values match expectations. -3. **Full query** — run the real aggregation with a time range and `LIMIT`, having confirmed it won't scan needlessly or return empty. +3. **Aggregation**: test the intended aggregation over that short window before widening it. Keep `LIMIT` to bound the output. -This is faster than discovering an empty result or a mis-shaped property after the full aggregation, and it costs less. +Counts and samples check volume and property shapes; they do not prove a join or aggregation will fit in memory. +`LIMIT` bounds returned rows. Joins and aggregations can still process the full input before applying it. + +##### Recovering from memory failures + +Follow the error's advice: wait and retry once when it reports temporary server capacity. +The `memory_limit` category alone cannot distinguish server pressure from a per-query limit. + +If the error says the query itself ran out of memory, shorten the time range in `WHERE` and test one adjusted query before running more windows. +Avoid parallel retries of the same failing query shape. Even a query filtered by time or person can be too large. +If the smaller window still fails, reduce join inputs or grouping cardinality only where that preserves the requested calculation. + +When splitting a requested range, use non-overlapping windows such as `timestamp >= start AND timestamp < end` with the same filters. +Split only when each window can be computed independently without changing the calculation. +Combine complete counts and sums by adding them, and compute averages from totals and counts. +Do not sum distinct counts or combine per-window quantiles, funnels, or retention results as if they covered the full range. +Rows cut off by `LIMIT` or truncation are incomplete. +Track which windows succeeded and disclose missing windows or reduced scope; do not present partial coverage as the full answer. ##### Skipping index @@ -364,6 +381,7 @@ Find the reference for [Sparkline, SemVer, Session replays, Actions, Translation ##### Other rules +- Use explicit `AS` and descriptive column aliases: `min(timestamp) AS first_seen_at` instead of `min(timestamp) first`. Some short names, such as `fn`, are reserved even after `AS`; use a descriptive name or backtick-quote the alias and its references. - WHERE clause must come after all JOINs - No semicolons at end of queries - `toStartOfWeek(timestamp, 1)` for Monday start (numeric, not string) diff --git a/skills/scanning-experiments-with-replay-vision/SKILL.md b/skills/scanning-experiments-with-replay-vision/SKILL.md index 192b703..e5e94d9 100644 --- a/skills/scanning-experiments-with-replay-vision/SKILL.md +++ b/skills/scanning-experiments-with-replay-vision/SKILL.md @@ -1,166 +1,131 @@ --- name: scanning-experiments-with-replay-vision -description: "Provisions a Replay Vision scanner scoped to one experiment's exposed sessions: sets `experiment_targeting` so the API derives the person-scoped exposure filter server-side, templates a prompt that stays comparable across variants, sizes credit spend against the experiment's own population, and creates the scanner disabled so its prompt can be previewed on real sessions before it sweeps.\nTRIGGER when: user wants Replay Vision to watch an experiment, asks to scan or analyze an experiment's recordings with AI, asks \"what are users actually doing in the test variant\", or wants a scanner scoped to an experiment's exposed sessions.\nDO NOT TRIGGER when: creating a general-purpose scanner not tied to an experiment (use creating-replay-vision-scanners), reading observations a scanner already produced (use exploring-replay-vision-observations), or manually browsing an experiment's recordings without AI analysis (use analyzing-experiment-session-replays)." +description: "Provisions a Replay Vision `experiment` scanner, which watches one experiment's exposed sessions, records each session's variant, samples the variants evenly, and stops when the experiment ends. Covers status guards, a prompt that stays comparable across variants, sizing credit spend against the experiment's own population, creating it disabled so its prompt can be previewed on real sessions, and reading the results per variant.\nTRIGGER when: user wants Replay Vision to watch an experiment, asks to scan or analyze an experiment's recordings with AI, asks \"what are users actually doing in the test variant\", or wants a scanner scoped to an experiment's exposed sessions.\nDO NOT TRIGGER when: creating a general-purpose scanner not tied to an experiment (use creating-replay-vision-scanners), reading observations a scanner already produced (use exploring-replay-vision-observations), or manually browsing an experiment's recordings without AI analysis (use analyzing-experiment-session-replays)." --- # Scanning experiments with Replay Vision The job: _"I'm running an experiment. Watch the recordings and tell me what's actually happening in each variant."_ -A Replay Vision scanner is a standing LLM probe over session recordings (see [[creating-replay-vision-scanners]] for the general mechanics). Scoping one to an experiment fixes the classic ways scanners go wrong, all at once: the exposure filter is **derived server-side** from the `experiment_targeting` field instead of hand-authored, the prompt is **templated** from the hypothesis and variants instead of vague, the population is **bounded** by enrollment, and the experiment's end date gives the scanner a natural end. This skill covers what is experiment-specific; the generic create/size mechanics stay in the parent skill. +A Replay Vision scanner is a standing LLM probe over session recordings (see [[creating-replay-vision-scanners]] for the general mechanics). The `experiment` scanner type is built for this job, so use it rather than scoping another type to an experiment: -The flow: resolve the experiment → set `experiment_targeting` so the API derives the exposure filter → pick a template → size it → create **disabled** → preview the prompt on a few real sessions → let the user enable it. +- The population is the experiment's exposed people, **derived server-side** from the experiment instead of a hand-built filter. +- Each session's variant comes from the exposure data, never from the model. The model is told what the experiment changes, but not the session's variant, so it describes what the recording shows. +- Variants are **sampled evenly**, so a 90/10 rollout still gives the small variant enough observations to compare. +- Scanning **stops when the experiment ends**, and pauses while it is paused. +- Results split by variant with no join: `vision-scanners-variants-list` and the observations `variant` filter. -## Step 1: Resolve the experiment +This skill covers what is experiment-specific; the generic create and size mechanics stay in the parent skill. + +The flow: resolve the experiment → write the config → size it → create **disabled** → preview the prompt on a few real sessions → let the user enable it → read the results per variant. -`experiment-get` returns everything needed: `feature_flag_key`, the linked `feature_flag` (its `filters.multivariate.variants` list is the source of truth for variant keys — `parameters.feature_flag_variants` can be stale), `exposure_criteria`, `resolved_exposure_event`, `start_date`, `end_date`, and `status`. If the user didn't identify the experiment, resolve it via [[finding-experiments]] rather than guessing. +## Step 1: Resolve the experiment -You no longer derive the exposure event to build the scan query — the API does that from `experiment_targeting` (Step 2). You still need the event name for the per-variant readout join at the end; the readout section covers that derivation where it is used. +`experiment-get` returns everything needed: the linked `feature_flag` (its `filters.multivariate.variants` list is the source of truth for variant keys; `parameters.feature_flag_variants` can be stale), `exposure_criteria`, `start_date`, `end_date`, and `status`. If the user didn't identify the experiment, resolve it via [[finding-experiments]] rather than guessing. Guards before doing anything else: -- **Draft** (no `start_date`): there are no exposures and nothing to scan. Say so and stop. -- **Stopped/complete**: a new scanner only sees sessions from creation time onward, so a concluded experiment needs a backfill over its run (see Limits). -- **Running or exposure-frozen**: proceed. A frozen experiment stops enrolling but already-exposed users keep producing sessions, so scanning stays useful. -- **Already half over**: the scanner watches only the remaining run. Say so, so a per-variant readout isn't mistaken for full-run coverage. +- **Draft** (no `start_date`): there are no exposures and nothing to scan. The API refuses an experiment scanner for an experiment that hasn't launched. Say so and stop. +- **Ended**: a new scanner sweeps only from creation onward, and it stops at the end date, so it scans nothing new. Cover the run with a backfill (see Limits). +- **Running or exposure-frozen**: proceed. A frozen experiment stops enrolling, but already-exposed users keep producing sessions, so scanning stays useful. +- **Already half over**: the scanner watches only the remaining run unless you backfill. Say so, so a per-variant readout isn't mistaken for full-run coverage. -## Step 2: Point the scanner at the experiment +## Step 2: Write the config -The scanner carries no hand-built exposure filter. Set the `experiment_targeting` field and the API derives the exposure filter server-side, at scan time: +The experiment lives in `scanner_config`, next to the prompt: ```json { + "prompt": "Focus on how the user moves through checkout after the payment step first appears: where they hesitate, retry, or leave.", "experiment_id": 123, - "variant": null + "variants": null, + "balance_variants": true } ``` -- `experiment_id` — the experiment from Step 1. -- `variant` — one variant key to narrow the scan to that variant's exposed people, or `null` for every variant. - -**Default to `null` — one scanner for the whole experiment, not one per variant.** The readout splits by variant later (see the readout step). Spend is the same either way — credits are per observation — but one scanner keeps a single prompt version across variants (see Limits on version skew) and one readout, and its random `sampling_rate` is applied _after_ the exposure filter matches, so the sample itself does not bias a variant (the eligibility gates are another story — see Step 4). Set a single `variant` only when the user asks to watch one arm. +- `experiment_id`: the experiment from Step 1. It is fixed once the scanner exists; watching a different experiment means a new scanner. +- `variants`: the variant keys to watch, or `null` for every variant. **Default to `null`, one scanner for the whole experiment.** A comparison needs every variant, and one scanner keeps one prompt version across them (see Limits on version skew). Narrow it only when the user asks to watch some variants. +- `balance_variants` (default `true`): one sampling rate per variant, so each gets roughly equal coverage for the same spend. Leave it on for comparisons. +- `length` (optional): `short`, `medium` (the default) or `long`, as for a summarizer. -**The API owns the exposure filter and its access control.** From `experiment_targeting` the server resolves the same exposed-person population the experiment's Recordings tab shows. The filter is **person-scoped**, so it covers people whose exposure event fired server-side or in an earlier session — sessions the old hand-built event filter missed. You do not build event or property filters for exposure, and you do not run a session-linkability or dedup check first; the server handles both. The API also rejects an `experiment_exposure` set directly inside `query`, and access-checks the targeted experiment, so a scanner can only reach an experiment its editor can view. +**The API owns the exposure filter and its access control.** It resolves the same exposed-person population the experiment's Recordings tab shows. The filter is **person-scoped**, so it covers people whose exposure fired server-side or in an earlier session. Don't build event or property filters for exposure; the API rejects an `experiment_exposure` set directly inside `query`, and it checks access to the experiment, so a scanner can only reach an experiment its editor can view. -**Keep `query` for non-exposure filters only.** Set `filter_test_accounts` from the experiment's own `exposure_criteria.filterTestAccounts`, defaulting to **`false` when absent** — that is what every experiment surface does (`get_test_accounts_filter` backend-side, `?? false` in the replay tab). A minimal query is enough: +**Keep `query` for non-exposure filters only.** Set `filter_test_accounts` from the experiment's own `exposure_criteria.filterTestAccounts`, defaulting to **`false` when absent**, which is what every experiment surface does. A minimal query is enough: ```json { "kind": "RecordingsQuery", "filter_test_accounts": false } ``` -Add other recording filters (duration, console errors, a specific page) only when the user asks. Leave exposure to `experiment_targeting`. - -**No `date_from`/`date_to`.** The scanner strips them on save (its 5-minute sweep controls time) and the estimate ignores them. - -## Step 3: Scanner type and prompt template - -**Default to `classifier`.** A fixed tag set is what makes two variants comparable — free text does not aggregate into a per-variant delta. `scorer` is second choice when the question is "how much"; `monitor` and `summarizer` are for exploration, not comparison. +Add other recording filters (duration, console errors, a specific page) only when the user asks. **No `date_from`/`date_to`**: the scanner strips them on save, because its 5-minute sweep controls time. -Template hygiene, learned the hard way: name the changed surface concretely (not "the new feature"), keep the tag set small, and **always include an escape tag** (`never-reached` or similar) — a classifier must pick from its tags, and most exposed sessions never touch the changed surface, so without an escape tag the model is forced to invent friction on irrelevant sessions and the variants develop a fake delta. +### The prompt -**Don't tell the model which variant is which.** It is tempting to write "control shows X, test shows Y" as context, but that is the one thing to leave out. The scanner has no idea which variant the session belongs to, and if the prompt supplies variant labels the model will confidently attach one anyway — in testing it reported "(test variant)" at confidence 1.0 on a session whose flag value was `control`, _even with an explicit instruction not to infer the variant_. Describe the surface and the forms it can take without labeling them ("the alert flow may appear either as one long form or as a stepped wizard — classify the experience either way"). The model doesn't need the mapping: variant attribution comes from the readout join, so the tags only have to describe behavior that means the same thing in either variant. Treat observation text as evidence about the **surface** the user saw, never about which **variant** they were assigned to — with labels in the prompt you cannot tell model error from a broken feature gate from the text alone; without them, that same disagreement becomes a detectable finding (see the readout step). +An experiment scanner writes a summary of each session, so the prompt says what the summary should focus on. The scanner adds the experiment's name, hypothesis and the names of the flag's variants by itself, but never the session's own variant: a model told the variant describes what that variant should show, not what the recording shows, and a broken flag gate then goes unseen. Don't restate them, and never write variant labels into the prompt yourself ("control shows X, test shows Y"). Describe the surface and the forms it can take without labeling them. When two variants look alike on screen, say what is visibly different, because the variant names alone may not let the model tell them apart. -Every prompt needs the **post-exposure framing** sentence: tell the model to focus on behavior after the point where the experiment's change would first be visible and ignore earlier activity. Be honest with the user that this is a request to the model, not an enforced window — scanners view the whole recording (see Limits). +Name the changed surface concretely, not "the new feature", and include the **post-exposure framing**: tell the model to focus on behavior after the point where the experiment's change would first be visible. Be honest with the user that this is a request to the model, not an enforced window, because scanners view the whole recording (see Limits). -Starter templates: +## Step 3: Size it against the experiment, not the month -1. **"Did anyone notice?"** — `classifier`, tags `reached-and-interacted`, `reached-not-interacted`, `never-reached`. When an experiment lands flat, the numbers can't distinguish "the change did nothing" from "nobody encountered the change"; this can. Needs the user to describe the changed surface. Often the right first scanner. -2. **Post-exposure friction** — `classifier`, tags `confusion`, `hesitation`, `error-or-dead-end`, `smooth`, `never-reached`. The general "why is the test variant losing" probe. -3. **Funnel drop-off explainer** — `classifier` or `monitor`. Honest caveat: a standing `RecordingsQuery` cannot express "was exposed but never completed the funnel" — event filters assert presence, never absence — so a _standing_ scanner can only pose the drop-off question in the prompt over every exposed session, which is weaker and costs more. What does work today: derive drop-off session ids with `execute-sql` (exposure present, completion event absent, non-empty `$session_id`) and `vision-scanners-scan-session` a sample of them. **Don't also require the funnel's first step.** The analysis prepends exposure as the funnel's implicit first step, so the exposure _is_ the entry; requiring the first step on top of it would drop the sessions that bounced before reaching it — the drop-offs most worth watching. Where a funnel lists one event as several steps, "completion absent" becomes "fired fewer times than the series repeats it". PostHog also computes this drop-off bucket server-side with hardening the hand-rolled SQL lacks (the experiments session-buckets REST endpoint) — it is not exposed over MCP yet; prefer it over the SQL once it is. Say which path you're offering. -4. **Per-variant behavior summary** — `summarizer`. Exploration only: summaries do not aggregate into a delta. +Run the standard gut-check from [[creating-replay-vision-scanners]]: `vision-scanners-estimate` with the `query` **and `experiment`** set to the same `experiment_id` and `variants` you will save, then `vision-quota-get`, comparing **credits against credits** (`remaining` is `null` when the org is uncapped; then reason about absolute spend instead). Passing `experiment` makes the estimate count only exposed sessions, so it forecasts the scanner's real spend. Experiment-specific corrections on top: -## Step 4: Size it against the experiment, not the month - -Run the standard gut-check from [[creating-replay-vision-scanners]]: `vision-scanners-estimate` with the `query` **and the same `experiment_targeting`** you will save, then `vision-quota-get`, comparing **credits against credits** (`remaining` is `null` when the org is uncapped — then reason about absolute spend instead). Passing `experiment_targeting` matters: the estimate then derives the same exposure filter and counts only exposed sessions, so it forecasts the scanner's real spend. Experiment-specific corrections on top: - -- **The estimate's window is the wrong window.** It always measures a fixed 30-day lookback — `window_days` shrinks only when the team's recording history is shorter, never to the experiment's age. For an experiment younger than the window, `matched_sessions_in_window / window_days` dilutes the true rate across days the experiment wasn't running (a 3-day-old experiment is understated ~10×), and `estimated_credits_per_month` inherits the dilution. Compute sessions/day as `matched_sessions_in_window / min(window_days, days since start_date)`, and don't quote the monthly figure as the experiment's cost. -- **The experiment gives a better bound than a monthly projection.** Total spend ≈ (exposed sessions/day × days remaining × `sampling_rate`) × `credits_per_observation` — a finite number. Use the experiment's expected remaining run time (`running_time_calculation.recommended_running_time` minus days elapsed, when set — that's its canonical home; it no longer lives in `parameters`). -- **`sampling_rate` is the lever, not a compromise.** A qualitative read does not need every session: on a high-traffic experiment even 0.5–2% sampling yields plenty of observations. The random sample is applied after the query matches, so it does not bias either variant. Floor: non-zero rates below 0.0001 are rejected; `0` means paused. -- **Two other gates are not variant-neutral.** The sweep (and the estimate) drop sessions under 15s total, under 10s of activity, or over 1h of activity, and a `focused`/`balanced` `sampling_mode` additionally keeps only roughly the top 25%/65% of sessions by surfacing score. Bounced and idle sessions are exactly what a `never-reached` tag is meant to count, so when one variant changes bounce behavior these filters clip the variants differently. Keep `sampling_mode: comprehensive` (the default) for experiment scanners, and read tag shares knowing sub-15s bounces never enter at all. -- **Healthy exposures next to `matched_sessions_in_window ≈ 0` means sessions aren't being recorded** — replay disabled or sampled down, or traffic from an SDK that doesn't record. Surface it and stop rather than creating a scanner that will sit idle. +- **The estimate's window is the wrong window.** It always measures a fixed 30-day lookback; `window_days` shrinks only when the team's recording history is shorter, never to the experiment's age. For an experiment younger than the window, `matched_sessions_in_window / window_days` dilutes the true rate across days the experiment wasn't running (a 3-day-old experiment is understated about 10×), and `estimated_credits_per_month` inherits the dilution. Compute sessions/day as `matched_sessions_in_window / min(window_days, days since start_date)`, and don't quote the monthly figure as the experiment's cost. +- **The experiment gives a better bound than a monthly projection.** Total spend ≈ exposed sessions/day × days remaining × `sampling_rate` × `credits_per_observation`, a finite number, because the scanner stops at the end date. Use the experiment's expected remaining run time (`running_time_calculation.recommended_running_time` minus days elapsed, when set). +- **`sampling_rate` is the lever, not a compromise.** A qualitative read does not need every session: on a high-traffic experiment even 0.5–2% sampling yields plenty of observations. With `balance_variants` on, the rate is the overall budget and each variant gets a share of it. Floor: non-zero rates below 0.0001 are rejected; `0` means paused. +- **The eligibility gates are not variant-neutral.** The sweep and the estimate drop sessions under 15s total, under 10s of activity, or over 1h of activity, and a `focused`/`balanced` `sampling_mode` additionally keeps only roughly the top 25%/65% of sessions by surfacing score. When one variant changes bounce behavior, these filters clip the variants differently. Keep `sampling_mode: comprehensive` (the default) for experiment scanners, and read the results knowing sub-15s bounces never enter at all. +- **Healthy exposures next to `matched_sessions_in_window ≈ 0` means sessions aren't being recorded**: replay disabled or sampled down, or traffic from an SDK that doesn't record. Surface it and stop rather than creating a scanner that will sit idle. Show the user the numbers before creating, per the parent skill. -## Step 5: Create disabled, preview, then hand over +## Step 4: Create disabled, preview, then hand over -Create with `vision-scanners-create`, carrying the `experiment_targeting` from Step 2 and **`enabled: false`** — no schedule, no sweep spend, and on-demand triggers still work. Preview scans are not free, though: each one spends credits like any observation (the quota check runs unconditionally) and is rejected outright when the org is exhausted. Name it so it's findable, e.g. `Experiment scan: ·