feat(convoy-metrics): un-gitignore .metrics.jsonl + add CI gate on convoy PRs (#134)
The pipeline metrics shim worked correctly through Jun 4 (65 role events, 25 convoys recorded) but silently stopped capturing thereafter. 8 convoy PRs merged Jun 5-11 (#126-#133) with zero rows logged to .convoys/.metrics.jsonl. The roles' Metrics sections clearly instruct the agents to call scripts/log-convoy-event.sh after every hand-off, but the instruction was skipped during multitask audit fan-outs and longer sessions where the Metrics section fell out of working context. Two changes to prevent the silent gap from recurring: 1. .gitignore: drop the `.convoys/.metrics.jsonl` ignore line. Convoy telemetry now committed in git so gaps surface in PR review. The script comment was already clear that events contain metadata only — no code, no prompts. 2. .github/workflows/convoy-metrics-gate.yml: new CI gate that fails any PR titled `convoy:` if no rows were added to .convoys/.metrics.jsonl between base and head. Bypass with the `skip-metrics` label + a documented reason. Non-convoy PRs are no-op. Also: commits the existing 65-event history to git so future analysis (and the §10 measurement protocol in agent-pipeline's v0.4 plan) has a stable baseline to compare against. Runs on self-hosted axiom runner to inherit the lower GH minutes cost the Jun 5-11 work already migrated to (PR #132). Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
varutasu
GitHub
parent
e933140560
commit
9eef8d9575
3 changed files with 159 additions and 2 deletions
65
.convoys/.metrics.jsonl
Normal file
65
.convoys/.metrics.jsonl
Normal file
|
|
@ -0,0 +1,65 @@
|
|||
{"ts": "2026-05-23T04:18:51Z", "role": "role-conductor", "convoy": "fix-auth-bypass", "repo": "tcg-vault", "skip_flags": ["ia", "ux", "visual", "a11y", "design"], "classification": "server-only", "duration_s": 0}
|
||||
{"ts": "2026-05-23T04:44:56Z", "role": "role-conductor", "convoy": "bump-next-js", "repo": "tcg-vault", "skip_flags": ["ia", "ux", "flag"], "classification": "feature", "duration_s": 0}
|
||||
{"ts": "2026-05-23T04:54:59Z", "role": "role-architect", "convoy": "bump-next-js", "repo": "tcg-vault", "skip_flags": ["ia", "ux", "flag"], "classification": "feature", "duration_s": 101}
|
||||
{"ts": "2026-05-23T05:37:11Z", "role": "role-architect", "convoy": "bump-next-js", "repo": "tcg-vault", "skip_flags": ["ia", "ux", "flag"], "classification": "feature", "duration_s": 311, "outcome": "scope-expanded"}
|
||||
{"ts": "2026-05-23T05:50:00Z", "role": "role-architect", "convoy": "bump-next-js", "repo": "tcg-vault", "skip_flags": ["ia", "ux", "flag"], "classification": "feature", "duration_s": 357, "outcome": "eslint-v10-pivot"}
|
||||
{"ts": "2026-05-23T05:55:05Z", "role": "role-implementer", "convoy": "bump-next-js", "repo": "tcg-vault", "skip_flags": [], "brief": 1, "duration_s": 180}
|
||||
{"ts": "2026-05-23T06:12:57Z", "role": "role-architect", "convoy": "bump-next-js", "repo": "tcg-vault", "skip_flags": [], "duration_s": 319, "outcome": "typescript-devdep-add"}
|
||||
{"ts": "2026-05-23T06:28:25Z", "role": "role-implementer", "convoy": "bump-next-js", "repo": "tcg-vault", "skip_flags": [], "brief": 1, "duration_s": 78}
|
||||
{"ts": "2026-05-23T06:48:30Z", "role": "role-architect", "convoy": "bump-next-js", "repo": "tcg-vault", "skip_flags": [], "duration_s": 347, "outcome": "eslint-v9-fallback-decision-d"}
|
||||
{"ts": "2026-05-23T06:51:22Z", "role": "role-implementer", "convoy": "bump-next-js", "repo": "tcg-vault", "skip_flags": [], "brief": 1, "duration_s": 50}
|
||||
{"ts": "2026-05-23T07:07:35Z", "role": "role-design-system-auditor", "convoy": "bump-next-js", "repo": "tcg-vault", "skip_flags": [], "duration_s": 114, "multitask_group": "audit-bump-next-js-4"}
|
||||
{"ts": "2026-05-23T07:08:15Z", "role": "role-a11y-auditor", "convoy": "bump-next-js", "repo": "tcg-vault", "skip_flags": [], "duration_s": 180, "multitask_group": "audit-bump-next-js-4"}
|
||||
{"ts": "2026-05-23T07:08:55Z", "role": "role-reviewer", "convoy": "bump-next-js", "repo": "tcg-vault", "skip_flags": [], "brief": 1, "duration_s": 42, "multitask_group": "audit-bump-next-js-4"}
|
||||
{"ts": "2026-05-23T07:34:56Z", "role": "role-doc-writer", "convoy": "bump-next-js", "repo": "tcg-vault", "skip_flags": [], "duration_s": 26, "outcome": "complete"}
|
||||
{"ts": "2026-05-23T07:57:10Z", "role": "role-architect", "convoy": "fix-auth-bypass", "repo": "tcg-vault", "skip_flags": ["ia", "ux", "visual", "a11y", "design"], "classification": "server-only", "duration_s": 900}
|
||||
{"ts": "2026-05-23T14:44:38Z", "role": "role-implementer", "convoy": "fix-auth-bypass", "repo": "tcg-vault", "skip_flags": [], "brief": 1, "duration_s": 192}
|
||||
{"ts": "2026-05-23T14:52:56Z", "role": "role-implementer", "convoy": "fix-auth-bypass", "repo": "tcg-vault", "skip_flags": [], "brief": 3, "duration_s": 240}
|
||||
{"ts": "2026-05-23T15:00:15Z", "role": "role-reviewer", "convoy": "fix-auth-bypass", "repo": "tcg-vault", "skip_flags": [], "brief": 3, "duration_s": 129}
|
||||
{"ts": "2026-05-23T15:52:10Z", "role": "role-reviewer", "convoy": "fix-auth-bypass", "repo": "tcg-vault", "skip_flags": [], "brief": 2, "duration_s": 209}
|
||||
{"ts": "2026-05-23T15:56:43Z", "role": "role-reviewer", "convoy": "fix-auth-bypass", "repo": "tcg-vault", "skip_flags": [], "brief": 4, "duration_s": 227}
|
||||
{"ts": "2026-05-23T16:04:17Z", "role": "role-reviewer", "convoy": "fix-auth-bypass", "repo": "tcg-vault", "skip_flags": [], "brief": 6, "duration_s": 34}
|
||||
{"ts": "2026-05-23T16:10:21Z", "role": "role-reviewer", "convoy": "fix-auth-bypass", "repo": "tcg-vault", "skip_flags": [], "brief": 5, "duration_s": 131}
|
||||
{"ts": "2026-05-23T16:25:52Z", "role": "role-doc-writer", "convoy": "fix-auth-bypass", "repo": "tcg-vault", "skip_flags": [], "brief": 0, "duration_s": 724, "outcome": "complete"}
|
||||
{"ts": "2026-05-23T17:31:18Z", "role": "role-architect", "convoy": "drop-public-setup", "repo": "tcg-vault", "skip_flags": [], "duration_s": 720}
|
||||
{"ts": "2026-05-23T19:58:01Z", "role": "role-implementer", "convoy": "drop-public-setup", "repo": "tcg-vault", "skip_flags": [], "brief": 1, "duration_s": 327}
|
||||
{"ts": "2026-05-23T20:07:35Z", "role": "role-implementer", "convoy": "drop-public-setup", "repo": "tcg-vault", "skip_flags": [], "brief": 2, "duration_s": 174}
|
||||
{"ts": "2026-05-23T23:00:44Z", "role": "role-architect", "convoy": "fix-layout-default-user", "repo": "tcg-vault", "skip_flags": [], "duration_s": 1500}
|
||||
{"ts": "2026-05-24T14:14:24Z", "role": "role-implementer", "convoy": "fix-layout-default-user", "repo": "tcg-vault", "skip_flags": [], "brief": 1, "duration_s": 403}
|
||||
{"ts": "2026-05-24T15:05:32Z", "role": "role-implementer", "convoy": "fix-layout-default-user", "repo": "tcg-vault", "skip_flags": [], "brief": 2, "duration_s": 1800}
|
||||
{"ts": "2026-05-24T19:37:56Z", "role": "role-doc-writer", "convoy": "fix-layout-default-user", "repo": "tcg-vault", "skip_flags": [], "duration_s": 242, "outcome": "complete"}
|
||||
{"ts": "2026-05-24T20:26:29Z", "role": "role-implementer", "convoy": "fix-vercel-deployment-protection-in-ci", "repo": "tcg-vault", "skip_flags": [], "brief": 1, "duration_s": 101}
|
||||
{"ts": "2026-05-24T21:32:57Z", "role": "role-doc-writer", "convoy": "fix-vercel-deployment-protection-in-ci", "repo": "tcg-vault", "skip_flags": [], "duration_s": 207, "outcome": "complete"}
|
||||
{"ts": "2026-05-24T23:44:57Z", "role": "role-implementer", "convoy": "adopt-playwright-smoke", "repo": "tcg-vault", "skip_flags": [], "brief": 1, "duration_s": 310}
|
||||
{"ts": "2026-05-25T01:25:55Z", "role": "role-implementer", "convoy": "cors-tighten", "repo": "tcg-vault", "skip_flags": [], "brief": 1, "duration_s": 900}
|
||||
{"ts": "2026-05-25T03:46:40Z", "role": "role-implementer", "convoy": "add-rate-limiting", "repo": "tcg-vault", "skip_flags": [], "brief": 1, "duration_s": 600}
|
||||
{"ts": "2026-05-27T12:55:50Z", "role": "role-conductor", "convoy": "scanner-audit-portfolio", "repo": "tcg-vault", "skip_flags": [], "classification": "docs-only", "duration_s": 0}
|
||||
{"ts": "2026-05-27T19:09:46Z", "role": "role-reviewer", "convoy": "redesign-scanner-flow", "repo": "tcg-vault", "skip_flags": [], "duration_s": 120, "outcome": "comment-only", "multitask_group": "audit-redesign-scanner-flow-44"}
|
||||
{"ts": "2026-05-27T19:09:46Z", "role": "role-design-system-auditor", "convoy": "redesign-scanner-flow", "repo": "tcg-vault", "skip_flags": [], "duration_s": 120, "outcome": "comment-only", "multitask_group": "audit-redesign-scanner-flow-44"}
|
||||
{"ts": "2026-05-27T19:09:46Z", "role": "role-a11y-auditor", "convoy": "redesign-scanner-flow", "repo": "tcg-vault", "skip_flags": [], "duration_s": 120, "outcome": "comment-only", "multitask_group": "audit-redesign-scanner-flow-44"}
|
||||
{"ts": "2026-06-02T05:33:45Z", "role": "role-implementer", "convoy": "collection-vocabulary", "repo": "tcg-vault", "skip_flags": [], "brief": 3, "duration_s": 120}
|
||||
{"ts": "2026-06-02T05:37:04Z", "role": "role-implementer", "convoy": "seed-visual-baselines-on-linux", "repo": "tcg-vault", "skip_flags": [], "brief": 1, "duration_s": 180}
|
||||
{"ts": "2026-06-02T05:37:36Z", "role": "role-implementer", "convoy": "purge-neondatabase-serverless-fully", "repo": "tcg-vault", "skip_flags": [], "brief": 1, "duration_s": 300}
|
||||
{"ts": "2026-06-02T13:48:03Z", "role": "role-implementer", "convoy": "wire-migrate-into-ci", "repo": "tcg-vault", "skip_flags": [], "brief": 1, "duration_s": 300}
|
||||
{"ts": "2026-06-02T13:48:43Z", "role": "role-doc-writer", "convoy": "convoy-doc-housekeeping", "repo": "tcg-vault", "skip_flags": [], "duration_s": 0, "outcome": "complete"}
|
||||
{"ts": "2026-06-02T14:37:35Z", "role": "role-implementer", "convoy": "seed-visual-baselines-on-linux", "repo": "tcg-vault", "skip_flags": [], "brief": 3, "duration_s": 120}
|
||||
{"ts": "2026-06-03T22:42:56Z", "role": "role-conductor", "convoy": "liquid-glass-design-tokens", "repo": "tcg-vault", "skip_flags": ["ux", "ia", "qa", "flag"], "classification": "feature", "duration_s": 180}
|
||||
{"ts": "2026-06-03T23:23:20Z", "role": "role-design-system-auditor", "convoy": "liquid-glass-design-tokens", "repo": "tcg-vault", "skip_flags": [], "duration_s": 420}
|
||||
{"ts": "2026-06-03T23:43:49Z", "role": "role-architect", "convoy": "liquid-glass-design-tokens", "repo": "tcg-vault", "skip_flags": [], "duration_s": 540}
|
||||
{"ts": "2026-06-03T23:48:23Z", "role": "role-a11y-auditor", "convoy": "liquid-glass-design-tokens", "repo": "tcg-vault", "skip_flags": [], "duration_s": 120}
|
||||
{"ts": "2026-06-03T23:48:23Z", "role": "role-implementer", "convoy": "liquid-glass-design-tokens", "repo": "tcg-vault", "skip_flags": [], "brief": 1, "duration_s": 600, "outcome": "merged"}
|
||||
{"ts": "2026-06-03T23:48:23Z", "role": "role-reviewer", "convoy": "liquid-glass-design-tokens", "repo": "tcg-vault", "skip_flags": [], "brief": 1, "duration_s": 180, "outcome": "approved"}
|
||||
{"ts": "2026-06-03T23:54:18Z", "role": "role-architect", "convoy": "liquid-glass-modal-and-surface-primitive", "repo": "tcg-vault", "skip_flags": [], "duration_s": 720}
|
||||
{"ts": "2026-06-03T23:54:18Z", "role": "role-implementer", "convoy": "liquid-glass-modal-and-surface-primitive", "repo": "tcg-vault", "skip_flags": [], "brief": 1, "duration_s": 1800, "outcome": "merged"}
|
||||
{"ts": "2026-06-03T23:58:42Z", "role": "role-architect", "convoy": "liquid-glass-form-primitives", "repo": "tcg-vault", "skip_flags": [], "duration_s": 600}
|
||||
{"ts": "2026-06-03T23:58:42Z", "role": "role-implementer", "convoy": "liquid-glass-form-primitives", "repo": "tcg-vault", "skip_flags": [], "brief": 1, "duration_s": 1500, "outcome": "merged"}
|
||||
{"ts": "2026-06-04T00:00:32Z", "role": "role-architect", "convoy": "liquid-glass-layout-shell", "repo": "tcg-vault", "skip_flags": [], "duration_s": 300}
|
||||
{"ts": "2026-06-04T00:00:32Z", "role": "role-implementer", "convoy": "liquid-glass-layout-shell", "repo": "tcg-vault", "skip_flags": [], "brief": 1, "duration_s": 900, "outcome": "merged"}
|
||||
{"ts": "2026-06-04T00:02:28Z", "role": "role-architect", "convoy": "motion-system-pass", "repo": "tcg-vault", "skip_flags": [], "duration_s": 300}
|
||||
{"ts": "2026-06-04T00:02:29Z", "role": "role-implementer", "convoy": "motion-system-pass", "repo": "tcg-vault", "skip_flags": [], "brief": 1, "duration_s": 420, "outcome": "merged"}
|
||||
{"ts": "2026-06-04T00:03:43Z", "role": "role-architect", "convoy": "liquid-glass-card-surfaces", "repo": "tcg-vault", "skip_flags": [], "duration_s": 240, "outcome": "architecture-only"}
|
||||
{"ts": "2026-06-04T00:04:10Z", "role": "role-architect", "convoy": "liquid-glass-public-and-auth", "repo": "tcg-vault", "skip_flags": [], "duration_s": 180, "outcome": "architecture-only"}
|
||||
{"ts": "2026-06-04T00:07:19Z", "role": "role-implementer", "convoy": "cleanup-legacy-design-css", "repo": "tcg-vault", "skip_flags": [], "brief": 1, "duration_s": 900, "outcome": "merged"}
|
||||
{"ts": "2026-06-04T18:36:17Z", "role": "role-conductor", "convoy": "unify-glass-panel-surfaces", "repo": "tcg-vault", "skip_flags": ["ia"], "classification": "feature", "duration_s": 0}
|
||||
{"ts": "2026-06-04T18:36:17Z", "role": "role-conductor", "convoy": "cleanup-card-item-list-and-share-modal-palette", "repo": "tcg-vault", "skip_flags": ["ia", "ux", "arch"], "classification": "feature", "duration_s": 0}
|
||||
{"ts": "2026-06-04T18:45:32Z", "role": "role-architect", "convoy": "unify-glass-panel-surfaces", "repo": "tcg-vault", "skip_flags": [], "duration_s": 0}
|
||||
90
.github/workflows/convoy-metrics-gate.yml
vendored
Normal file
90
.github/workflows/convoy-metrics-gate.yml
vendored
Normal file
|
|
@ -0,0 +1,90 @@
|
|||
name: Convoy metrics gate
|
||||
|
||||
# Forces convoy PRs to include role-event telemetry. Without this gate, the
|
||||
# pipeline produces work but no signal — exactly what happened during the
|
||||
# Jun 5-11 experiment window when 8 convoy PRs shipped without logging a
|
||||
# single .metrics.jsonl row (see analytics/v0.4-beta1-results.md in
|
||||
# agent-pipeline).
|
||||
#
|
||||
# Triggers: PR titled `convoy:` (case-insensitive prefix). If a convoy PR
|
||||
# does NOT add at least one new role-event line to .convoys/.metrics.jsonl,
|
||||
# this check fails. Bypass: add the `skip-metrics` label to the PR (rare,
|
||||
# document the reason in the PR body).
|
||||
#
|
||||
# Other PRs are no-op (skip the gate).
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
branches: [main]
|
||||
types: [opened, synchronize, reopened, edited, labeled, unlabeled]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: read
|
||||
|
||||
jobs:
|
||||
gate:
|
||||
name: Require role-event telemetry on convoy PRs
|
||||
runs-on: [self-hosted, axiom]
|
||||
steps:
|
||||
- name: Inspect PR
|
||||
id: inspect
|
||||
uses: actions/github-script@v7
|
||||
with:
|
||||
script: |
|
||||
const pr = context.payload.pull_request;
|
||||
const title = pr.title || '';
|
||||
const labels = (pr.labels || []).map(l => l.name);
|
||||
const isConvoy = /^convoy:/i.test(title);
|
||||
const hasBypass = labels.includes('skip-metrics');
|
||||
core.setOutput('is_convoy', isConvoy ? 'true' : 'false');
|
||||
core.setOutput('has_bypass', hasBypass ? 'true' : 'false');
|
||||
core.info(`title="${title}" is_convoy=${isConvoy} has_bypass=${hasBypass}`);
|
||||
if (hasBypass) {
|
||||
core.warning(`skip-metrics label present on PR #${pr.number} — gate will exit early. Document the reason in the PR body.`);
|
||||
}
|
||||
|
||||
- name: Skip — not a convoy PR
|
||||
if: steps.inspect.outputs.is_convoy != 'true'
|
||||
run: |
|
||||
echo "PR title is not 'convoy:' — gate is no-op for this PR."
|
||||
|
||||
- name: Skip — bypass label present
|
||||
if: steps.inspect.outputs.is_convoy == 'true' && steps.inspect.outputs.has_bypass == 'true'
|
||||
run: |
|
||||
echo "::warning::skip-metrics label bypass. Metrics gate not enforced for this PR."
|
||||
|
||||
- name: Checkout PR head
|
||||
if: steps.inspect.outputs.is_convoy == 'true' && steps.inspect.outputs.has_bypass != 'true'
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Verify metrics rows added
|
||||
if: steps.inspect.outputs.is_convoy == 'true' && steps.inspect.outputs.has_bypass != 'true'
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
BASE_SHA="${{ github.event.pull_request.base.sha }}"
|
||||
HEAD_SHA="${{ github.event.pull_request.head.sha }}"
|
||||
METRICS=".convoys/.metrics.jsonl"
|
||||
|
||||
if [ ! -f "$METRICS" ]; then
|
||||
echo "::error::Convoy PR but $METRICS does not exist. The L2 roles must call scripts/log-convoy-event.sh after each hand-off — see .cursor/agents/role-*.md 'Metrics' sections."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
ADDED_LINES=$(git diff "$BASE_SHA"..."$HEAD_SHA" -- "$METRICS" | grep -c '^+{' || true)
|
||||
|
||||
if [ "$ADDED_LINES" -eq 0 ]; then
|
||||
echo "::error::Convoy PR did not add any rows to $METRICS."
|
||||
echo "::error::Did the L2 roles call scripts/log-convoy-event.sh after their hand-off? See .cursor/agents/role-*.md 'Metrics' section."
|
||||
echo "::error::To bypass intentionally (rare), add 'skip-metrics' label to the PR and document why in the PR body."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! git diff "$BASE_SHA"..."$HEAD_SHA" -- "$METRICS" | grep -q '"convoy"'; then
|
||||
echo "::warning::Added lines do not appear to be well-formed JSON with a 'convoy' field. Verify scripts/log-convoy-event.sh ran cleanly."
|
||||
fi
|
||||
|
||||
echo "::notice::Convoy metrics gate passed: $ADDED_LINES new role-event row(s) added to $METRICS."
|
||||
6
.gitignore
vendored
6
.gitignore
vendored
|
|
@ -31,8 +31,10 @@ __pycache__/
|
|||
.env*.local
|
||||
|
||||
# Agent pipeline self-analytics (per-developer; opt-out by default).
|
||||
# Remove this line if you want team-shared convoy metrics committed.
|
||||
.convoys/.metrics.jsonl
|
||||
# Convoy metrics are tracked in git so silent telemetry gaps surface in PR
|
||||
# review (see .github/workflows/convoy-metrics-gate.yml). Events contain
|
||||
# metadata only — no code, no prompts. To revert to local-only telemetry,
|
||||
# restore the `.convoys/.metrics.jsonl` ignore line and drop the metrics gate.
|
||||
|
||||
# Local code-knowledge-graph (per-developer; user-code-review-graph MCP)
|
||||
.code-review-graph/
|
||||
|
|
|
|||
Loading…
Reference in a new issue