diff --git a/packages/pen-ai-skills/corpus/ab-v0/README.md b/packages/pen-ai-skills/corpus/ab-v0/README.md new file mode 100644 index 000000000..bbee0301b --- /dev/null +++ b/packages/pen-ai-skills/corpus/ab-v0/README.md @@ -0,0 +1,27 @@ +# A/B Corpus v0 + +24 prompts for the element-tools A/B quality evaluation. See +`~/workspace/openpencil-docs/superpowers/plans/2026-04-20-element-tools-ab-corpus.md` +for the experiment design. + +Split: 8 prompts × 3 categories (`mobile` / `dashboard` / `landing`), each +category further split 4 `obvious` + 4 `optional`. + +- **obvious** — prompt matches one of the 39 `add_*_v0` tools perfectly; a + well-calibrated decision tree MUST pick that tool. Used to score + treatment-arm routing quality (M5). +- **optional** — prompt has no dominant element-tool fit; either tool path + or `batch_design` is acceptable. Used to check that N-tool availability + doesn't HURT on cases where it doesn't help. + +Schema per file: see `../../../src/corpus/corpus-loader.ts` for the canonical +definition. Keys: + +| key | required | notes | +| -------------------- | -------- | ----------------------------------------------- | +| id | yes | kebab-case, unique, stable | +| category | yes | mobile / dashboard / landing | +| difficulty | yes | obvious / optional | +| prompt | yes | natural-language design request | +| expected | yes | shape gates applied to rendered output | +| expected_tool_if_any | no | obvious-only: which add\_\*\_v0 tool should win | diff --git a/packages/pen-ai-skills/corpus/ab-v0/dashboard-alert-banner.yaml b/packages/pen-ai-skills/corpus/ab-v0/dashboard-alert-banner.yaml new file mode 100644 index 000000000..65a13f622 --- /dev/null +++ b/packages/pen-ai-skills/corpus/ab-v0/dashboard-alert-banner.yaml @@ -0,0 +1,15 @@ +id: dashboard-alert-banner +category: dashboard +difficulty: obvious +prompt: | + Design ONLY a dismissible warning alert banner (one row): leading + triangle-alert icon, message "Your subscription expires in 7 + days. Renew to avoid service interruption.", trailing close (x) + icon. Do NOT include surrounding page, header, or content below + — just the alert banner. +expected: + must_contain_roles: + - alert + - alert-message + - alert-close +expected_tool_if_any: add_alert_v0 diff --git a/packages/pen-ai-skills/corpus/ab-v0/dashboard-analytics-overview.yaml b/packages/pen-ai-skills/corpus/ab-v0/dashboard-analytics-overview.yaml new file mode 100644 index 000000000..030eceed3 --- /dev/null +++ b/packages/pen-ai-skills/corpus/ab-v0/dashboard-analytics-overview.yaml @@ -0,0 +1,12 @@ +id: dashboard-analytics-overview +category: dashboard +difficulty: optional +prompt: | + Design an analytics overview page (1200 wide) with a section titled + "This week" containing three cards side by side: Total visits, Avg + session time, Bounce rate. Each card has a title, a big number, a + small trend delta, and a tiny sparkline placeholder. Below, a + "View full report" text link. +expected: + must_contain_roles: + - heading diff --git a/packages/pen-ai-skills/corpus/ab-v0/dashboard-data-table.yaml b/packages/pen-ai-skills/corpus/ab-v0/dashboard-data-table.yaml new file mode 100644 index 000000000..6d8232ad7 --- /dev/null +++ b/packages/pen-ai-skills/corpus/ab-v0/dashboard-data-table.yaml @@ -0,0 +1,12 @@ +id: dashboard-data-table +category: dashboard +difficulty: optional +prompt: | + Design a data table for an orders dashboard (1200 wide). Columns: + Order ID, Customer, Total, Status, Date. Show a header row and 6 + data rows with varied status badges (Paid, Pending, Refunded). + Above the table, a title "Recent orders" and a pagination control + showing "Page 2 of 14" with previous/next arrow buttons. +expected: + must_contain_roles: + - heading diff --git a/packages/pen-ai-skills/corpus/ab-v0/dashboard-filter-toolbar.yaml b/packages/pen-ai-skills/corpus/ab-v0/dashboard-filter-toolbar.yaml new file mode 100644 index 000000000..f997a849a --- /dev/null +++ b/packages/pen-ai-skills/corpus/ab-v0/dashboard-filter-toolbar.yaml @@ -0,0 +1,11 @@ +id: dashboard-filter-toolbar +category: dashboard +difficulty: optional +prompt: | + Design a dashboard filter toolbar (1200 wide) with a search bar on + the left, a row of toggle chips in the middle ("Active", "Archived", + "Flagged"), and on the right a segmented control switching between + "Day / Week / Month" views. +expected: + must_contain_roles: + - search-bar diff --git a/packages/pen-ai-skills/corpus/ab-v0/dashboard-kpi-strip.yaml b/packages/pen-ai-skills/corpus/ab-v0/dashboard-kpi-strip.yaml new file mode 100644 index 000000000..98d7103d0 --- /dev/null +++ b/packages/pen-ai-skills/corpus/ab-v0/dashboard-kpi-strip.yaml @@ -0,0 +1,17 @@ +id: dashboard-kpi-strip +category: dashboard +difficulty: obvious +prompt: | + Design ONLY a non-scrolling inline stats strip with 4 cells side + by side, sharing the row width equally: Revenue ($128K, icon + "trending-up"), New Users (2,340, icon "user-plus"), Active + Sessions (18,902, icon "activity"), Churn (1.2%, icon + "trending-down"). Each cell has an icon, big value, small label. + Do NOT include page background, header, or any other content — + just the stats strip. +expected: + must_contain_roles: + - stat-grid + min_roles: + stat-cell: 4 +expected_tool_if_any: add_stat_grid_v0 diff --git a/packages/pen-ai-skills/corpus/ab-v0/dashboard-settings-panel.yaml b/packages/pen-ai-skills/corpus/ab-v0/dashboard-settings-panel.yaml new file mode 100644 index 000000000..0fd6254c2 --- /dev/null +++ b/packages/pen-ai-skills/corpus/ab-v0/dashboard-settings-panel.yaml @@ -0,0 +1,12 @@ +id: dashboard-settings-panel +category: dashboard +difficulty: optional +prompt: | + Design a grouped settings panel (1200 wide). Three grouped + sections, each introduced by a heading: "Account" (fields Email, + Password), "Notifications" (two toggle switches: Email digest, + Push), and "Danger zone" (a destructive "Delete account" button). + Dividers between groups. +expected: + must_contain_roles: + - heading diff --git a/packages/pen-ai-skills/corpus/ab-v0/dashboard-upload-progress.yaml b/packages/pen-ai-skills/corpus/ab-v0/dashboard-upload-progress.yaml new file mode 100644 index 000000000..ba88d5ef1 --- /dev/null +++ b/packages/pen-ai-skills/corpus/ab-v0/dashboard-upload-progress.yaml @@ -0,0 +1,12 @@ +id: dashboard-upload-progress +category: dashboard +difficulty: obvious +prompt: | + Design ONLY a linear horizontal progress bar showing roughly 60% + complete. No surrounding card, no title, no subtitle, no + cancel button — just the progress bar itself. +expected: + must_contain_roles: + - progress-bar + - progress-bar-fill +expected_tool_if_any: add_progress_bar_v0 diff --git a/packages/pen-ai-skills/corpus/ab-v0/dashboard-wizard-stepper.yaml b/packages/pen-ai-skills/corpus/ab-v0/dashboard-wizard-stepper.yaml new file mode 100644 index 000000000..9f5d6e70b --- /dev/null +++ b/packages/pen-ai-skills/corpus/ab-v0/dashboard-wizard-stepper.yaml @@ -0,0 +1,16 @@ +id: dashboard-wizard-stepper +category: dashboard +difficulty: obvious +prompt: | + Design ONLY a horizontal numbered stepper (1)─(2)─(3)─(4)─(5) + showing 5 steps where steps 1 and 2 are completed (filled + circles), step 3 is current, and steps 4-5 are pending. + Connectors between steps fill the space evenly. No surrounding + wizard content, no header — just the stepper. +expected: + must_contain_roles: + - stepper + min_roles: + step-connector: 2 + step-connector-active: 1 +expected_tool_if_any: add_stepper_v0 diff --git a/packages/pen-ai-skills/corpus/ab-v0/landing-code-showcase.yaml b/packages/pen-ai-skills/corpus/ab-v0/landing-code-showcase.yaml new file mode 100644 index 000000000..72d1f4240 --- /dev/null +++ b/packages/pen-ai-skills/corpus/ab-v0/landing-code-showcase.yaml @@ -0,0 +1,15 @@ +id: landing-code-showcase +category: landing +difficulty: obvious +prompt: | + Design ONLY a preformatted code block showing this TypeScript + snippet (preserve newlines): "npm install @zseven-w/pen-sdk\n + import { open } from '@zseven-w/pen-sdk';\nopen('./design.pen');" + Light gray fill, rounded corners, monospace text. Do NOT include + surrounding headline, body copy, or any other content — just + the code block. +expected: + must_contain_roles: + - code-block + - code +expected_tool_if_any: add_code_block_v0 diff --git a/packages/pen-ai-skills/corpus/ab-v0/landing-cta-band.yaml b/packages/pen-ai-skills/corpus/ab-v0/landing-cta-band.yaml new file mode 100644 index 000000000..d3b80ffa3 --- /dev/null +++ b/packages/pen-ai-skills/corpus/ab-v0/landing-cta-band.yaml @@ -0,0 +1,12 @@ +id: landing-cta-band +category: landing +difficulty: optional +prompt: | + Design a call-to-action band at the bottom of a landing page (1200 + wide). Dark background, centered heading "Ready to get started?", + subtext one-sentence, and a single primary CTA button "Create your + first design". +expected: + must_contain_roles: + - heading + - button diff --git a/packages/pen-ai-skills/corpus/ab-v0/landing-faq.yaml b/packages/pen-ai-skills/corpus/ab-v0/landing-faq.yaml new file mode 100644 index 000000000..1a6585fbf --- /dev/null +++ b/packages/pen-ai-skills/corpus/ab-v0/landing-faq.yaml @@ -0,0 +1,11 @@ +id: landing-faq +category: landing +difficulty: optional +prompt: | + Design a FAQ section on a landing page (1200 wide) with a heading + "Frequently asked questions" and five expanded question rows, each + showing the question in bold and the answer as body text below it, + separated by hairline dividers. +expected: + must_contain_roles: + - heading diff --git a/packages/pen-ai-skills/corpus/ab-v0/landing-feature-grid.yaml b/packages/pen-ai-skills/corpus/ab-v0/landing-feature-grid.yaml new file mode 100644 index 000000000..ae8d37eb0 --- /dev/null +++ b/packages/pen-ai-skills/corpus/ab-v0/landing-feature-grid.yaml @@ -0,0 +1,12 @@ +id: landing-feature-grid +category: landing +difficulty: optional +prompt: | + Design a feature grid section on a landing page (1200 wide) with + six feature cards in a 3-column × 2-row grid. Each card has an + icon, a short title, and a one-sentence description. Above the + grid, a section heading "Everything you need" and a one-sentence + subtitle. +expected: + must_contain_roles: + - heading diff --git a/packages/pen-ai-skills/corpus/ab-v0/landing-hero.yaml b/packages/pen-ai-skills/corpus/ab-v0/landing-hero.yaml new file mode 100644 index 000000000..37a4bfa18 --- /dev/null +++ b/packages/pen-ai-skills/corpus/ab-v0/landing-hero.yaml @@ -0,0 +1,13 @@ +id: landing-hero +category: landing +difficulty: optional +prompt: | + Design a hero section for a SaaS landing page (1200 wide). Include + a large headline ("Ship your designs faster"), a supporting + one-sentence body, two CTAs side by side ("Start free trial" + primary, "Watch demo" secondary), and a product-screenshot + placeholder on the right. +expected: + must_contain_roles: + - heading + - button diff --git a/packages/pen-ai-skills/corpus/ab-v0/landing-pricing-section.yaml b/packages/pen-ai-skills/corpus/ab-v0/landing-pricing-section.yaml new file mode 100644 index 000000000..de0883e7c --- /dev/null +++ b/packages/pen-ai-skills/corpus/ab-v0/landing-pricing-section.yaml @@ -0,0 +1,16 @@ +id: landing-pricing-section +category: landing +difficulty: obvious +prompt: | + Design ONLY a price typography block for a SaaS "Pro" plan: + currency glyph "$", big amount "29", trailing period "/month". + The three parts render inline with tight baseline alignment. Do + NOT include a surrounding pricing card, plan name, feature list, + CTA button, or any other content — just the price typography. +expected: + must_contain_roles: + - price + - price-currency + - price-amount + - price-period +expected_tool_if_any: add_price_v0 diff --git a/packages/pen-ai-skills/corpus/ab-v0/landing-product-ratings.yaml b/packages/pen-ai-skills/corpus/ab-v0/landing-product-ratings.yaml new file mode 100644 index 000000000..9ef36b95e --- /dev/null +++ b/packages/pen-ai-skills/corpus/ab-v0/landing-product-ratings.yaml @@ -0,0 +1,14 @@ +id: landing-product-ratings +category: landing +difficulty: obvious +prompt: | + Design ONLY a row of 5 star icons representing a 4-out-of-5 + rating — 4 filled stars and 1 empty star, inline horizontally. + No headline, no subtext, no review count label, no surrounding + card — just the 5 stars. +expected: + must_contain_roles: + - rating-stars + - star-filled + - star-empty +expected_tool_if_any: add_rating_stars_v0 diff --git a/packages/pen-ai-skills/corpus/ab-v0/landing-testimonial.yaml b/packages/pen-ai-skills/corpus/ab-v0/landing-testimonial.yaml new file mode 100644 index 000000000..521590844 --- /dev/null +++ b/packages/pen-ai-skills/corpus/ab-v0/landing-testimonial.yaml @@ -0,0 +1,16 @@ +id: landing-testimonial +category: landing +difficulty: obvious +prompt: | + Design ONLY a pull-quote block. Quote text: "OpenPencil replaced + two tools in our stack and shipped design-to-code that we can + actually read." Attribution: "Alex Chen, Principal Engineer at + Acme". Rounded light-fill container with quote above attribution. + Do NOT include surrounding section, headline, avatars, or any + other content — just the quote block. +expected: + must_contain_roles: + - quote-block + - quote-text + - quote-author +expected_tool_if_any: add_quote_block_v0 diff --git a/packages/pen-ai-skills/corpus/ab-v0/mobile-empty-inbox.yaml b/packages/pen-ai-skills/corpus/ab-v0/mobile-empty-inbox.yaml new file mode 100644 index 000000000..52e7ab6ea --- /dev/null +++ b/packages/pen-ai-skills/corpus/ab-v0/mobile-empty-inbox.yaml @@ -0,0 +1,13 @@ +id: mobile-empty-inbox +category: mobile +difficulty: obvious +prompt: | + Design ONLY an empty-state block (centered): inbox icon, title + "No messages yet", subtitle "New conversations will appear here", + and a primary CTA "Start a conversation". Do NOT include + surrounding page, navigation, or other content — just this + empty-state component. +expected: + must_contain_roles: + - empty-state +expected_tool_if_any: add_empty_state_v0 diff --git a/packages/pen-ai-skills/corpus/ab-v0/mobile-filter-chips.yaml b/packages/pen-ai-skills/corpus/ab-v0/mobile-filter-chips.yaml new file mode 100644 index 000000000..7b6fb9271 --- /dev/null +++ b/packages/pen-ai-skills/corpus/ab-v0/mobile-filter-chips.yaml @@ -0,0 +1,15 @@ +id: mobile-filter-chips +category: mobile +difficulty: obvious +prompt: | + Design ONLY a horizontal scrolling row of 8 filter chips: "All" + (active), "Videos", "Photos", "Music", "Articles", "Podcasts", + "Events", "Books". Each chip has a label and optional small + icon. Do NOT include page background, header, tab bar, or any + other content — just the chip row. +expected: + must_contain_roles: + - scroll-row-wrapper + min_roles: + nav-chip: 6 +expected_tool_if_any: add_nav_chip_row_v0 diff --git a/packages/pen-ai-skills/corpus/ab-v0/mobile-fitness-dashboard.yaml b/packages/pen-ai-skills/corpus/ab-v0/mobile-fitness-dashboard.yaml new file mode 100644 index 000000000..77189cba2 --- /dev/null +++ b/packages/pen-ai-skills/corpus/ab-v0/mobile-fitness-dashboard.yaml @@ -0,0 +1,13 @@ +id: mobile-fitness-dashboard +category: mobile +difficulty: obvious +prompt: | + Design ONLY a horizontal scrolling row of 3 metric tiles for a + fitness tracker: Steps (value "8,432", icon "activity"), Kcal + (value "512", icon "flame"), Sleep (value "7h 24m", icon "moon"). + Each tile shows label + big value + small icon. Do NOT include + page background, header, or bottom nav — just the metric row. +expected: + min_roles: + metric-tile: 3 +expected_tool_if_any: add_metric_row_v0 diff --git a/packages/pen-ai-skills/corpus/ab-v0/mobile-notification-center.yaml b/packages/pen-ai-skills/corpus/ab-v0/mobile-notification-center.yaml new file mode 100644 index 000000000..2529451e7 --- /dev/null +++ b/packages/pen-ai-skills/corpus/ab-v0/mobile-notification-center.yaml @@ -0,0 +1,12 @@ +id: mobile-notification-center +category: mobile +difficulty: optional +prompt: | + Design a mobile notification center (375×812). Include a top bar + ("Notifications" + clear-all action), a section header "Today", two + notification rows (each with an app icon, title, body, and time), + then a section header "Earlier" with three more notification rows. +expected: + must_contain_roles: + - top-nav-bar + - section-header diff --git a/packages/pen-ai-skills/corpus/ab-v0/mobile-onboarding.yaml b/packages/pen-ai-skills/corpus/ab-v0/mobile-onboarding.yaml new file mode 100644 index 000000000..492e63df7 --- /dev/null +++ b/packages/pen-ai-skills/corpus/ab-v0/mobile-onboarding.yaml @@ -0,0 +1,12 @@ +id: mobile-onboarding +category: mobile +difficulty: optional +prompt: | + Design a first-run onboarding screen (375×812) with a hero + illustration at the top (placeholder box), a headline "Welcome to + Flow", a one-sentence body describing the app, and two bottom buttons + "Get started" (primary) and "Sign in" (secondary). +expected: + must_contain_roles: + - heading + - button diff --git a/packages/pen-ai-skills/corpus/ab-v0/mobile-profile-edit.yaml b/packages/pen-ai-skills/corpus/ab-v0/mobile-profile-edit.yaml new file mode 100644 index 000000000..f49bd825a --- /dev/null +++ b/packages/pen-ai-skills/corpus/ab-v0/mobile-profile-edit.yaml @@ -0,0 +1,13 @@ +id: mobile-profile-edit +category: mobile +difficulty: optional +prompt: | + Design a mobile profile-edit screen (375×812). Include a top nav bar + ("Edit profile" + back), a large avatar with a camera-edit badge + overlay, form fields for Name, Email, and Bio (multi-line), and a + full-width "Save changes" button at the bottom. +expected: + must_contain_roles: + - top-nav-bar + - avatar + - form-field diff --git a/packages/pen-ai-skills/corpus/ab-v0/mobile-settings-list.yaml b/packages/pen-ai-skills/corpus/ab-v0/mobile-settings-list.yaml new file mode 100644 index 000000000..a215db940 --- /dev/null +++ b/packages/pen-ai-skills/corpus/ab-v0/mobile-settings-list.yaml @@ -0,0 +1,13 @@ +id: mobile-settings-list +category: mobile +difficulty: obvious +prompt: | + Design ONLY one iOS-style settings list row. It should have a + leading icon (bell), a title "Notifications", a subtitle "Push, + email, and in-app alerts", and a trailing chevron-right icon. Do + NOT include page chrome, navigation, or other rows — just this + one row. +expected: + must_contain_roles: + - list-row +expected_tool_if_any: add_list_row_v0 diff --git a/packages/pen-ai-skills/corpus/ab-v0/mobile-tabs-demo.yaml b/packages/pen-ai-skills/corpus/ab-v0/mobile-tabs-demo.yaml new file mode 100644 index 000000000..92a13b4f1 --- /dev/null +++ b/packages/pen-ai-skills/corpus/ab-v0/mobile-tabs-demo.yaml @@ -0,0 +1,12 @@ +id: mobile-tabs-demo +category: mobile +difficulty: optional +prompt: | + Design a mobile screen (375×812) with a top nav bar titled "Explore", + a horizontal tab row with three underline tabs ("For you" active, + "Trending", "Latest"), and a placeholder content area under the + active tab. +expected: + must_contain_roles: + - top-nav-bar + - tabs diff --git a/packages/pen-ai-skills/src/corpus/__tests__/aggregate.test.ts b/packages/pen-ai-skills/src/corpus/__tests__/aggregate.test.ts new file mode 100644 index 000000000..b97909692 --- /dev/null +++ b/packages/pen-ai-skills/src/corpus/__tests__/aggregate.test.ts @@ -0,0 +1,211 @@ +import { describe, it, expect } from 'vitest'; +import { aggregate } from '../aggregate'; +import type { ScoreRow } from '../types'; + +function row(partial: Partial): ScoreRow { + return { + promptId: 'x', + category: 'mobile', + difficulty: 'obvious', + model: 'm1', + variant: 'B', + outputKind: 'batch_design', + toolName: '', + expectedToolIfAny: '', + routing: 'n/a', + m1_legal: true, + m3_success: true, + m3_failure_reason: '', + issues: [], + applyError: '', + rawOutput: '', + ...partial, + }; +} + +describe('aggregate — per-model summary', () => { + it('computes M1 rates and delta per model', () => { + const rows: ScoreRow[] = [ + row({ model: 'a', variant: 'B', m1_legal: true }), + row({ model: 'a', variant: 'B', m1_legal: false }), + row({ model: 'a', variant: 'T', m1_legal: true }), + row({ model: 'a', variant: 'T', m1_legal: true }), + ]; + const r = aggregate(rows); + const a = r.byModel.find((m) => m.model === 'a'); + expect(a).toBeDefined(); + expect(a!.m1_baseline).toBeCloseTo(0.5); + expect(a!.m1_treatment).toBeCloseTo(1.0); + expect(a!.m1_delta_pp).toBeCloseTo(50); + }); + + it('routing breakdown splits 4-way right/wrong/fallback/garbage on obvious treatment', () => { + // 4 obvious-treatment runs: 2 right, 1 wrong, 1 fallback → rates 0.5 / 0.25 / 0.25 / 0 + const rows: ScoreRow[] = [ + row({ + difficulty: 'obvious', + variant: 'T', + outputKind: 'tool_call', + toolName: 'add_link_v0', + expectedToolIfAny: 'add_link_v0', + routing: 'right-tool', + }), + row({ + difficulty: 'obvious', + variant: 'T', + outputKind: 'tool_call', + toolName: 'add_link_v0', + expectedToolIfAny: 'add_link_v0', + routing: 'right-tool', + }), + row({ + difficulty: 'obvious', + variant: 'T', + outputKind: 'tool_call', + toolName: 'add_divider_v0', + expectedToolIfAny: 'add_link_v0', + routing: 'wrong-tool', + }), + row({ + difficulty: 'obvious', + variant: 'T', + outputKind: 'batch_design', + expectedToolIfAny: 'add_link_v0', + routing: 'fallback', + }), + row({ + difficulty: 'optional', + variant: 'T', + outputKind: 'batch_design', + routing: 'n/a', + }), + row({ variant: 'B', routing: 'n/a' }), + ]; + const r = aggregate(rows); + expect(r.byModel[0].m5_right_tool).toBeCloseTo(0.5); + expect(r.byModel[0].m5_wrong_tool).toBeCloseTo(0.25); + expect(r.byModel[0].m5_fallback).toBeCloseTo(0.25); + expect(r.byModel[0].m5_garbage).toBeCloseTo(0); + }); + + it('garbage in denominator prevents inflated right-tool rate (the codex fix)', () => { + // 10 garbage + 1 right + 1 wrong; naively computing right / (right+wrong) + // would yield 50% "routing success" — but only 1/12 runs was actually + // right. 4-way breakdown forces the denominator to include garbage. + const rows: ScoreRow[] = [ + ...Array.from({ length: 10 }, () => + row({ + difficulty: 'obvious', + variant: 'T', + outputKind: 'garbage', + routing: 'garbage', + }), + ), + row({ + difficulty: 'obvious', + variant: 'T', + outputKind: 'tool_call', + toolName: 'add_link_v0', + expectedToolIfAny: 'add_link_v0', + routing: 'right-tool', + }), + row({ + difficulty: 'obvious', + variant: 'T', + outputKind: 'tool_call', + toolName: 'add_fab_v0', + expectedToolIfAny: 'add_link_v0', + routing: 'wrong-tool', + }), + ]; + const r = aggregate(rows); + expect(r.byModel[0].m5_right_tool).toBeCloseTo(1 / 12); + expect(r.byModel[0].m5_wrong_tool).toBeCloseTo(1 / 12); + expect(r.byModel[0].m5_fallback).toBeCloseTo(0); + expect(r.byModel[0].m5_garbage).toBeCloseTo(10 / 12); + }); + + it('four routing rates sum to 1 (obvious-treatment denominator)', () => { + const rows: ScoreRow[] = [ + row({ difficulty: 'obvious', variant: 'T', routing: 'right-tool' }), + row({ difficulty: 'obvious', variant: 'T', routing: 'wrong-tool' }), + row({ difficulty: 'obvious', variant: 'T', routing: 'fallback' }), + row({ difficulty: 'obvious', variant: 'T', routing: 'garbage' }), + ]; + const r = aggregate(rows); + const sum = + r.byModel[0].m5_right_tool + + r.byModel[0].m5_wrong_tool + + r.byModel[0].m5_fallback + + r.byModel[0].m5_garbage; + expect(sum).toBeCloseTo(1); + }); + + it('returns NaN for rates with zero samples in a cell', () => { + const rows: ScoreRow[] = [row({ model: 'empty-treatment', variant: 'B', m1_legal: true })]; + const r = aggregate(rows); + const m = r.byModel.find((x) => x.model === 'empty-treatment'); + expect(m).toBeDefined(); + expect(Number.isNaN(m!.m1_treatment)).toBe(true); + expect(Number.isNaN(m!.m1_delta_pp)).toBe(true); + // All three routing rates NaN when there are no obvious-treatment + // runs at all (denominator is zero). + expect(Number.isNaN(m!.m5_right_tool)).toBe(true); + expect(Number.isNaN(m!.m5_wrong_tool)).toBe(true); + expect(Number.isNaN(m!.m5_fallback)).toBe(true); + expect(Number.isNaN(m!.m5_garbage)).toBe(true); + }); +}); + +describe('aggregate — per-category + per-tool', () => { + it('splits M1 by category, both variants', () => { + const rows: ScoreRow[] = [ + row({ category: 'mobile', variant: 'B', m1_legal: false }), + row({ category: 'mobile', variant: 'T', m1_legal: true }), + row({ category: 'landing', variant: 'B', m1_legal: true }), + row({ category: 'landing', variant: 'T', m1_legal: true }), + ]; + const r = aggregate(rows); + const mobile = r.byCategory.find((c) => c.category === 'mobile')!; + expect(mobile.m1_baseline).toBe(0); + expect(mobile.m1_treatment).toBe(1); + const landing = r.byCategory.find((c) => c.category === 'landing')!; + expect(landing.m1_baseline).toBe(1); + expect(landing.m1_treatment).toBe(1); + }); + + it('ranks tools by invocation count, descending', () => { + const rows: ScoreRow[] = [ + row({ variant: 'T', outputKind: 'tool_call', toolName: 'add_link_v0', m1_legal: true }), + row({ variant: 'T', outputKind: 'tool_call', toolName: 'add_link_v0', m1_legal: false }), + row({ variant: 'T', outputKind: 'tool_call', toolName: 'add_divider_v0', m1_legal: true }), + ]; + const r = aggregate(rows); + expect(r.byTool).toHaveLength(2); + expect(r.byTool[0]).toEqual({ + tool: 'add_link_v0', + invocations: 2, + successfulInvocations: 1, + }); + expect(r.byTool[1]).toEqual({ + tool: 'add_divider_v0', + invocations: 1, + successfulInvocations: 1, + }); + }); + + it('ignores baseline runs in tool usage', () => { + const rows: ScoreRow[] = [ + row({ variant: 'B', outputKind: 'batch_design' }), + row({ variant: 'T', outputKind: 'tool_call', toolName: 'add_fab_v0' }), + ]; + const r = aggregate(rows); + expect(r.byTool).toHaveLength(1); + expect(r.byTool[0].tool).toBe('add_fab_v0'); + }); + + it('totalRuns reflects input length', () => { + const r = aggregate([row({}), row({}), row({})]); + expect(r.totalRuns).toBe(3); + }); +}); diff --git a/packages/pen-ai-skills/src/corpus/__tests__/corpus-loader.test.ts b/packages/pen-ai-skills/src/corpus/__tests__/corpus-loader.test.ts new file mode 100644 index 000000000..01cb27dd1 --- /dev/null +++ b/packages/pen-ai-skills/src/corpus/__tests__/corpus-loader.test.ts @@ -0,0 +1,215 @@ +import { describe, it, expect, beforeEach, afterEach } from 'vitest'; +import { mkdtempSync, writeFileSync, mkdirSync, rmSync } from 'node:fs'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; +import { loadCorpus } from '../corpus-loader'; + +const REPO_CORPUS_DIR = join(__dirname, '..', '..', '..', 'corpus', 'ab-v0'); + +let tmp: string; + +beforeEach(() => { + tmp = mkdtempSync(join(tmpdir(), 'corpus-loader-')); +}); +afterEach(() => { + rmSync(tmp, { recursive: true, force: true }); +}); + +describe('loadCorpus — real corpus', () => { + it('loads all 24 yaml files (stable order)', () => { + const prompts = loadCorpus(REPO_CORPUS_DIR); + expect(prompts).toHaveLength(24); + const ids = prompts.map((p) => p.id); + expect(new Set(ids).size).toBe(24); + }); + + it('24 prompts split 8 per category', () => { + const prompts = loadCorpus(REPO_CORPUS_DIR); + const byCat = new Map(); + for (const p of prompts) byCat.set(p.category, (byCat.get(p.category) ?? 0) + 1); + expect(byCat.get('mobile')).toBe(8); + expect(byCat.get('dashboard')).toBe(8); + expect(byCat.get('landing')).toBe(8); + }); + + it('each category has 4 obvious + 4 optional', () => { + const prompts = loadCorpus(REPO_CORPUS_DIR); + for (const cat of ['mobile', 'dashboard', 'landing'] as const) { + const forCat = prompts.filter((p) => p.category === cat); + const obvious = forCat.filter((p) => p.difficulty === 'obvious').length; + const optional = forCat.filter((p) => p.difficulty === 'optional').length; + expect(obvious, `${cat} obvious count`).toBe(4); + expect(optional, `${cat} optional count`).toBe(4); + } + }); + + it('every obvious prompt has expected_tool_if_any (loader contract)', () => { + const prompts = loadCorpus(REPO_CORPUS_DIR); + const obviousMissingTool = prompts + .filter((p) => p.difficulty === 'obvious' && !p.expected_tool_if_any) + .map((p) => p.id); + expect(obviousMissingTool).toEqual([]); + }); + + it('expected_tool_if_any references a real add_*_v0 name shape', () => { + const prompts = loadCorpus(REPO_CORPUS_DIR); + for (const p of prompts) { + if (p.expected_tool_if_any) { + expect(p.expected_tool_if_any).toMatch(/^add_[a-z_]+_v0$/); + } + } + }); +}); + +describe('loadCorpus — validation errors', () => { + function writeYaml(name: string, body: string): void { + writeFileSync(join(tmp, name), body, 'utf-8'); + } + + it('rejects missing id', () => { + writeYaml( + 'bad.yaml', + `category: mobile +difficulty: optional +prompt: hi +expected: {} +`, + ); + expect(() => loadCorpus(tmp)).toThrow(/field "id" must be a non-empty string/); + }); + + it('rejects invalid category', () => { + writeYaml( + 'bad.yaml', + `id: x +category: web +difficulty: optional +prompt: hi +expected: {} +`, + ); + expect(() => loadCorpus(tmp)).toThrow(/invalid category "web"/); + }); + + it('rejects invalid difficulty', () => { + writeYaml( + 'bad.yaml', + `id: x +category: mobile +difficulty: hard +prompt: hi +expected: {} +`, + ); + expect(() => loadCorpus(tmp)).toThrow(/invalid difficulty "hard"/); + }); + + it('rejects obvious without expected_tool_if_any', () => { + writeYaml( + 'bad.yaml', + `id: x +category: mobile +difficulty: obvious +prompt: hi +expected: {} +`, + ); + expect(() => loadCorpus(tmp)).toThrow(/obvious requires expected_tool_if_any/); + }); + + it('rejects duplicate ids across files', () => { + writeYaml( + 'a.yaml', + `id: same +category: mobile +difficulty: optional +prompt: a +expected: {} +`, + ); + writeYaml( + 'b.yaml', + `id: same +category: mobile +difficulty: optional +prompt: b +expected: {} +`, + ); + expect(() => loadCorpus(tmp)).toThrow(/duplicate prompt id "same"/); + }); + + it('rejects malformed min_roles values', () => { + writeYaml( + 'bad.yaml', + `id: x +category: mobile +difficulty: optional +prompt: hi +expected: + min_roles: + card: negative +`, + ); + expect(() => loadCorpus(tmp)).toThrow(/min_roles.card must be a non-negative number/); + }); + + it('accepts empty expected object', () => { + writeYaml( + 'ok.yaml', + `id: x +category: mobile +difficulty: optional +prompt: hi +expected: {} +`, + ); + const out = loadCorpus(tmp); + expect(out).toHaveLength(1); + expect(out[0].expected).toEqual({}); + }); + + it('ignores non-yaml files in directory', () => { + writeYaml('README.md', '# hi'); + writeYaml( + 'ok.yaml', + `id: x +category: mobile +difficulty: optional +prompt: hi +expected: {} +`, + ); + const out = loadCorpus(tmp); + expect(out).toHaveLength(1); + }); + + it('throws on nonexistent directory', () => { + const missing = join(tmp, 'does-not-exist'); + expect(() => loadCorpus(missing)).toThrow(/cannot read directory/); + }); + + it('yaml files sorted by filename (deterministic)', () => { + mkdirSync(tmp, { recursive: true }); + writeYaml( + 'z.yaml', + `id: z-id +category: mobile +difficulty: optional +prompt: z +expected: {} +`, + ); + writeYaml( + 'a.yaml', + `id: a-id +category: mobile +difficulty: optional +prompt: a +expected: {} +`, + ); + const out = loadCorpus(tmp); + expect(out.map((p) => p.id)).toEqual(['a-id', 'z-id']); + }); +}); diff --git a/packages/pen-ai-skills/src/corpus/__tests__/output-parser.test.ts b/packages/pen-ai-skills/src/corpus/__tests__/output-parser.test.ts new file mode 100644 index 000000000..2261cdddf --- /dev/null +++ b/packages/pen-ai-skills/src/corpus/__tests__/output-parser.test.ts @@ -0,0 +1,351 @@ +import { describe, it, expect } from 'vitest'; +import { parseModelOutput } from '../output-parser'; + +describe('parseModelOutput — tool_call path', () => { + it('extracts name + arguments from tags', () => { + const raw = `{"name": "add_card_row_v0", "arguments": {"items": [{"title": "A"}]}}`; + const out = parseModelOutput(raw); + expect(out.kind).toBe('tool_call'); + if (out.kind !== 'tool_call') throw new Error(); + expect(out.name).toBe('add_card_row_v0'); + expect(out.arguments).toEqual({ items: [{ title: 'A' }] }); + }); + + it('trims whitespace inside tags', () => { + const raw = ` + {"name": "add_link_v0", "arguments": {"label": "hi"}} + `; + const out = parseModelOutput(raw); + expect(out.kind).toBe('tool_call'); + }); + + it('survives chatty prose around the tag', () => { + const raw = `Sure, here's the tool call:\n{"name":"add_divider_v0","arguments":{}}\nDone!`; + const out = parseModelOutput(raw); + expect(out.kind).toBe('tool_call'); + if (out.kind !== 'tool_call') throw new Error(); + expect(out.name).toBe('add_divider_v0'); + }); + + it('returns garbage when payload is not valid JSON', () => { + const raw = `{not json}`; + const out = parseModelOutput(raw); + expect(out.kind).toBe('garbage'); + if (out.kind !== 'garbage') throw new Error(); + expect(out.reason).toMatch(/payload was not valid JSON/); + }); + + it('returns garbage when payload is missing "name"', () => { + const raw = `{"arguments": {}}`; + const out = parseModelOutput(raw); + expect(out.kind).toBe('garbage'); + if (out.kind !== 'garbage') throw new Error(); + expect(out.reason).toMatch(/missing string "name"/); + }); + + it('returns garbage when "arguments" is not an object', () => { + const raw = `{"name":"add_x_v0","arguments":"oops"}`; + const out = parseModelOutput(raw); + expect(out.kind).toBe('garbage'); + if (out.kind !== 'garbage') throw new Error(); + expect(out.reason).toMatch(/missing object "arguments"/); + }); +}); + +describe('parseModelOutput — batch_design path', () => { + it('detects bare DSL without code fence', () => { + const raw = `root=I(null, {"type": "frame", "children": []}) +nav=I(root, {"type": "frame", "role": "navbar"})`; + const out = parseModelOutput(raw); + expect(out.kind).toBe('batch_design'); + if (out.kind !== 'batch_design') throw new Error(); + expect(out.dsl).toContain('root=I(null'); + }); + + it('strips markdown code fence around DSL', () => { + const raw = '```\nroot=I(null, {"type": "frame"})\n```'; + const out = parseModelOutput(raw); + expect(out.kind).toBe('batch_design'); + if (out.kind !== 'batch_design') throw new Error(); + expect(out.dsl).toBe('root=I(null, {"type": "frame"})'); + }); + + it('detects U(...) / D(...) / C(...) / R(...) / M(...) forms', () => { + for (const dsl of [ + 'U(root, {"fill": []})', + 'D(some-id)', + 'copy=C(src, dst, {})', + 'repl=R(target, {"type":"frame"})', + 'M(nodeId, parent)', + ]) { + const out = parseModelOutput(dsl); + expect(out.kind, `expected batch_design for: ${dsl}`).toBe('batch_design'); + } + }); + + it('detects bindless I/C/R/G forms (binding is optional per batch-design.ts)', () => { + // Codex stop-hook 2026-04-20: earlier isCleanDsl regex required + // `word=` prefix for I/C/R/G and rejected bindless forms that + // handleBatchDesign's `bindlessAssignMatch` accepts. + for (const dsl of [ + 'I(null, {"type":"frame","name":"Root"})', + 'C("source-id", "parent-id", {})', + 'R("target-id", {"type":"frame"})', + 'G("pattern", {})', + ]) { + const out = parseModelOutput(dsl); + expect(out.kind, `expected batch_design for bindless: ${dsl}`).toBe('batch_design'); + } + }); + + it('multi-line payload mixing bound and bindless forms stays batch_design', () => { + const raw = + 'I(null, {"type":"frame","name":"Root"})\n' + + 'nav=I("parent-id", {"type":"frame","role":"navbar"})\n' + + 'U("nav-id", {"gap": 16})'; + const out = parseModelOutput(raw); + expect(out.kind).toBe('batch_design'); + }); + + it('returns garbage on prose-only output', () => { + const raw = "I'm happy to help! What kind of design would you like?"; + const out = parseModelOutput(raw); + expect(out.kind).toBe('garbage'); + }); + + it('returns garbage on empty output', () => { + expect(parseModelOutput('').kind).toBe('garbage'); + expect(parseModelOutput(' ').kind).toBe('garbage'); + }); +}); + +describe('parseModelOutput — tool_call takes precedence over DSL when both present', () => { + it('prefers even if DSL-looking text follows it', () => { + const raw = `{"name":"add_link_v0","arguments":{"label":"x"}}\nroot=I(null, {})`; + const out = parseModelOutput(raw); + expect(out.kind).toBe('tool_call'); + }); +}); + +describe('parseModelOutput — multi-tag preference (real-world weak-model output)', () => { + it('prefers element-tool name over earlier batch_design scaffold tag', () => { + // Exact shape observed from MiniMax M2.7 treatment run — model + // emitted batch_design scaffold + add_nav_chip_row_v0 + another + // batch_design. Parser must see through the scaffolding. + const raw = + '{"name": "batch_design", "arguments": {"operations": "root=I(null, {})"}}\n' + + '{"name": "add_nav_chip_row_v0", "arguments": {"items": [{"label":"All"}]}}\n' + + '{"name": "batch_design", "arguments": {"operations": "U(x,{})"}}'; + const out = parseModelOutput(raw); + expect(out.kind).toBe('tool_call'); + if (out.kind !== 'tool_call') throw new Error(); + expect(out.name).toBe('add_nav_chip_row_v0'); + }); + + it('picks first element-tool tag when multiple element tools appear', () => { + const raw = + '{"name":"add_divider_v0","arguments":{}}\n' + + '{"name":"add_link_v0","arguments":{"label":"x"}}'; + const out = parseModelOutput(raw); + expect(out.kind).toBe('tool_call'); + if (out.kind !== 'tool_call') throw new Error(); + expect(out.name).toBe('add_divider_v0'); + }); + + it('falls back to batch_design when only batch_design tags present', () => { + const raw = + '{"name":"batch_design","arguments":{"operations":"root=I(null, {\\"type\\":\\"frame\\"})"}}'; + const out = parseModelOutput(raw); + expect(out.kind).toBe('batch_design'); + if (out.kind !== 'batch_design') throw new Error(); + expect(out.dsl).toContain('root=I(null'); + }); + + it('batch_design with array operations serializes to DSL lines', () => { + const raw = + '{"name":"batch_design","arguments":{"operations":[{"type":"I","id":"r","node":{"type":"frame","name":"R"}}]}}'; + const out = parseModelOutput(raw); + expect(out.kind).toBe('batch_design'); + if (out.kind !== 'batch_design') throw new Error(); + expect(out.dsl).toContain('r=I(null, '); + expect(out.dsl).toContain('"name":"R"'); + }); + + it('batch_design with no usable operations → garbage', () => { + const raw = '{"name":"batch_design","arguments":{"operations":null}}'; + const out = parseModelOutput(raw); + expect(out.kind).toBe('garbage'); + if (out.kind !== 'garbage') throw new Error(); + expect(out.reason).toMatch(/no usable.*operations/); + }); + + it('unknown tool name (not element, not batch_design) → tool_call with that name for scorer to mark wrong-tool', () => { + const raw = '{"name":"made_up_tool","arguments":{}}'; + const out = parseModelOutput(raw); + expect(out.kind).toBe('tool_call'); + if (out.kind !== 'tool_call') throw new Error(); + expect(out.name).toBe('made_up_tool'); + }); +}); + +describe('parseModelOutput — malformed tag must not eclipse valid DSL', () => { + it('DSL after a malformed block still classified as batch_design', () => { + // Codex stop-hook 2026-04-20 regression: earlier parser returned + // garbage-with-parse-reason whenever any tag failed, + // even if valid batch_design DSL existed elsewhere in the output. + const raw = + 'not valid json here\n' + + 'root=I(null, {"type":"frame","name":"Root"})\n' + + 'nav=I(root, {"type":"frame","role":"navbar"})'; + const out = parseModelOutput(raw); + expect(out.kind).toBe('batch_design'); + if (out.kind !== 'batch_design') throw new Error(); + expect(out.dsl).toContain('root=I(null'); + expect(out.dsl).toContain('nav=I(root'); + }); + + it('DSL wrapped inside a malformed body still recovered', () => { + // Model emits DSL directly inside without the JSON + // wrapper. Earlier parser returned garbage; now we peek inside + // and reclassify as batch_design. + const raw = 'root=I(null, {"type":"frame"})'; + const out = parseModelOutput(raw); + expect(out.kind).toBe('batch_design'); + if (out.kind !== 'batch_design') throw new Error(); + expect(out.dsl).toContain('root=I(null'); + }); + + it('valid with unusable batch_design operations falls through to DSL detection in body', () => { + // {name:batch_design, operations:null} is unusable BUT + // the response body also contains valid raw DSL. Prefer the DSL. + const raw = + '{"name":"batch_design","arguments":{"operations":null}}\n' + + 'root=I(null, {"type":"frame"})'; + const out = parseModelOutput(raw); + expect(out.kind).toBe('batch_design'); + if (out.kind !== 'batch_design') throw new Error(); + expect(out.dsl).toContain('root=I(null'); + }); + + it('malformed tag + no DSL anywhere → garbage with specific parse reason (existing behavior preserved)', () => { + const raw = 'not valid json\nnothing else useful here.'; + const out = parseModelOutput(raw); + expect(out.kind).toBe('garbage'); + if (out.kind !== 'garbage') throw new Error(); + expect(out.reason).toMatch(/not valid JSON/); + }); + + it('malformed-tag body with prose + one DSL line buried → NOT recovered as batch_design', () => { + // Codex stop-hook 2026-04-20 regression fix: earlier recovery + // fired whenever ANY line in the body looked like DSL, so a + // mostly-prose body with a single DSL-ish line would be + // returned as a "batch_design" payload that handleBatchDesign + // would then reject line-by-line. Now recovery requires the + // body to START with DSL. + const raw = + '\nLet me think about this request carefully.\nroot=I(null, {"type":"frame"})\nMore prose that trails off.\n'; + const out = parseModelOutput(raw); + expect(out.kind).toBe('garbage'); + }); + + it('cleaned text with prose before DSL line → NOT classified as batch_design', () => { + // Same principle at the outer (non-tag) fallback path. + const raw = 'Here\'s my design:\nroot=I(null, {"type":"frame"})\nLet me know!'; + const out = parseModelOutput(raw); + expect(out.kind).toBe('garbage'); + }); + + it('fenced-block content that starts with prose (not DSL) → NOT classified as batch_design', () => { + const raw = '```\nLet me think:\nroot=I(null, {})\n```'; + const out = parseModelOutput(raw); + expect(out.kind).toBe('garbage'); + }); + + it('pure DSL at the very start of body still recovered', () => { + // Sanity — must not regress the happy path that motivated the + // recovery in the first place. + const raw = 'root=I(null, {"type":"frame"})\nnav=I(root, {"type":"frame"})'; + const out = parseModelOutput(raw); + expect(out.kind).toBe('batch_design'); + if (out.kind !== 'batch_design') throw new Error(); + expect(out.dsl).toContain('root=I(null'); + }); +}); + +describe('parseModelOutput — mixed DSL + prose rejected as garbage', () => { + it('DSL line + prose line + DSL line → garbage (codex stop-hook fix)', () => { + // Earlier parser accepted this as batch_design because it started + // with DSL; handleBatchDesign would then split on top-level + // newlines and emit a parse error for the prose chunk. That + // misclassifies "malformed output" as "model chose batch_design". + const raw = + 'root=I(null, {"type":"frame"})\n' + + "Let me explain what I'm doing here...\n" + + 'nav=I(root, {"type":"frame"})'; + const out = parseModelOutput(raw); + expect(out.kind).toBe('garbage'); + }); + + it('DSL line + trailing prose → garbage', () => { + const raw = 'root=I(null, {"type":"frame"})\nHope this helps!'; + const out = parseModelOutput(raw); + expect(out.kind).toBe('garbage'); + }); + + it('DSL across multiple physical lines (pretty-printed JSON inside op) → batch_design', () => { + // Happy-path sanity — the splitter's brace/paren awareness means + // pretty-printed JSON inside I(...) stays ONE op, not N mixed + // chunks. Must not regress or pretty-printed model output fails. + const raw = + 'root=I(null, {\n' + + ' "type": "frame",\n' + + ' "name": "Root",\n' + + ' "children": []\n' + + '})\n' + + 'nav=I(root, {"type":"frame","role":"navbar"})'; + const out = parseModelOutput(raw); + expect(out.kind).toBe('batch_design'); + if (out.kind !== 'batch_design') throw new Error(); + expect(out.dsl).toContain('"name": "Root"'); + }); + + it('mixed payload inside malformed tag body → garbage', () => { + const raw = + 'root=I(null, {"type":"frame"})\n' + + 'Some prose in between.\n' + + 'nav=I(root, {"type":"frame"})'; + const out = parseModelOutput(raw); + expect(out.kind).toBe('garbage'); + }); + + it('DSL string literal containing ( or { does not confuse splitter', () => { + const raw = + 'root=I(null, {"type":"text","content":"price: $(100)","name":"{bold}"})\nnav=I(root, {})'; + const out = parseModelOutput(raw); + expect(out.kind).toBe('batch_design'); + }); +}); + +describe('parseModelOutput — stripping (reasoning-model robustness)', () => { + it('strips ... prefix before matching', () => { + const raw = + '\nThe user wants X. Let me use Y.\n\n\n' + + '{"name":"add_link_v0","arguments":{"label":"ok"}}'; + const out = parseModelOutput(raw); + expect(out.kind).toBe('tool_call'); + }); + + it('strips multiple blocks (MiniMax sometimes re-thinks)', () => { + const raw = + 'first\nsecond\n' + + '{"name":"add_divider_v0","arguments":{}}'; + const out = parseModelOutput(raw); + expect(out.kind).toBe('tool_call'); + }); + + it('output that is entirely → garbage (after stripping nothing remains)', () => { + const raw = 'just thinking, never answered'; + const out = parseModelOutput(raw); + expect(out.kind).toBe('garbage'); + }); +}); diff --git a/packages/pen-ai-skills/src/corpus/__tests__/score-run.test.ts b/packages/pen-ai-skills/src/corpus/__tests__/score-run.test.ts new file mode 100644 index 000000000..4b0d3a397 --- /dev/null +++ b/packages/pen-ai-skills/src/corpus/__tests__/score-run.test.ts @@ -0,0 +1,272 @@ +import { describe, it, expect } from 'vitest'; +import { scoreRun, countRoles } from '../score-run'; +import type { ApplyFn, CorpusPrompt, ParsedOutput } from '../types'; +import type { PenDocument } from '@zseven-w/pen-types'; + +const baseDoc: PenDocument = { + version: '1.0.0', + children: [ + { + id: 'root', + type: 'frame', + name: 'Root', + role: 'section', + children: [ + { + id: 'a', + type: 'frame', + name: 'A', + role: 'card', + children: [{ id: 'a1', type: 'text', role: 'heading', content: 'X' }], + }, + { + id: 'b', + type: 'frame', + name: 'B', + role: 'card', + children: [], + }, + ], + }, + ], +} as PenDocument; + +const prompt: CorpusPrompt = { + id: 'p1', + category: 'mobile', + difficulty: 'obvious', + prompt: 'test', + expected: { + must_contain_roles: ['section'], + min_roles: { card: 2 }, + }, + expected_tool_if_any: 'add_card_row_v0', +}; + +const okParsed: ParsedOutput = { kind: 'batch_design', dsl: 'root=I(null, {})', raw: '' }; +const toolParsed: ParsedOutput = { + kind: 'tool_call', + name: 'add_card_row_v0', + arguments: {}, + raw: '', +}; +const garbageParsed: ParsedOutput = { kind: 'garbage', reason: 'empty output', raw: '' }; + +function makeApply(result: { ok: boolean; doc: PenDocument | null; error?: string }): ApplyFn { + return async () => ({ ok: result.ok, doc: result.doc, error: result.error ?? '' }); +} + +describe('countRoles', () => { + it('counts roles across the full tree including nested', () => { + const counts = countRoles(baseDoc); + expect(counts.section).toBe(1); + expect(counts.card).toBe(2); + expect(counts.heading).toBe(1); + }); + + it('ignores nodes without a role', () => { + const doc: PenDocument = { + version: '1.0.0', + children: [{ id: 'x', type: 'frame', name: 'X', children: [] }], + } as PenDocument; + expect(countRoles(doc)).toEqual({}); + }); +}); + +describe('scoreRun — M1 / M3 gates', () => { + it('passes M1 + M3 when apply succeeds and shape matches', async () => { + const row = await scoreRun({ + prompt, + parsed: okParsed, + apply: makeApply({ ok: true, doc: baseDoc }), + model: 'm1', + variant: 'B', + }); + expect(row.m1_legal).toBe(true); + expect(row.m3_success).toBe(true); + expect(row.m3_failure_reason).toBe(''); + }); + + it('fails M1 when apply throws', async () => { + const row = await scoreRun({ + prompt, + parsed: okParsed, + apply: makeApply({ ok: false, doc: null, error: 'DSL parse failed' }), + model: 'm1', + variant: 'B', + }); + expect(row.m1_legal).toBe(false); + expect(row.m3_success).toBe(false); + expect(row.applyError).toBe('DSL parse failed'); + expect(row.m3_failure_reason).toBe('apply failed before shape checks'); + }); + + it('fails M3 when required role is missing', async () => { + const shapeOnlyPrompt: CorpusPrompt = { + ...prompt, + expected: { must_contain_roles: ['hero'] }, + }; + const row = await scoreRun({ + prompt: shapeOnlyPrompt, + parsed: okParsed, + apply: makeApply({ ok: true, doc: baseDoc }), + model: 'm1', + variant: 'B', + }); + expect(row.m1_legal).toBe(true); + expect(row.m3_success).toBe(false); + expect(row.m3_failure_reason).toMatch(/missing required role.*hero/); + }); + + it('fails M3 when min_roles not met', async () => { + const strictPrompt: CorpusPrompt = { + ...prompt, + expected: { min_roles: { card: 5 } }, + }; + const row = await scoreRun({ + prompt: strictPrompt, + parsed: okParsed, + apply: makeApply({ ok: true, doc: baseDoc }), + model: 'm1', + variant: 'B', + }); + expect(row.m3_success).toBe(false); + expect(row.m3_failure_reason).toMatch(/role counts below minimum/); + }); + + it('short-circuits to garbage when parse failed', async () => { + const row = await scoreRun({ + prompt, + parsed: garbageParsed, + apply: makeApply({ ok: true, doc: baseDoc }), + model: 'm1', + variant: 'T', + }); + expect(row.m1_legal).toBe(false); + expect(row.m3_success).toBe(false); + expect(row.outputKind).toBe('garbage'); + expect(row.m3_failure_reason).toMatch(/output unparseable/); + }); + + it('carries tool name through on tool_call output', async () => { + const row = await scoreRun({ + prompt, + parsed: toolParsed, + apply: makeApply({ ok: true, doc: baseDoc }), + model: 'm1', + variant: 'T', + }); + expect(row.outputKind).toBe('tool_call'); + expect(row.toolName).toBe('add_card_row_v0'); + }); +}); + +describe('scoreRun — routing classification (expected_tool_if_any)', () => { + it('right-tool when treatment obvious + tool matches expected', async () => { + const row = await scoreRun({ + prompt, // expected_tool_if_any = 'add_card_row_v0' + parsed: toolParsed, // name = 'add_card_row_v0' + apply: makeApply({ ok: true, doc: baseDoc }), + model: 'm', + variant: 'T', + }); + expect(row.routing).toBe('right-tool'); + expect(row.expectedToolIfAny).toBe('add_card_row_v0'); + }); + + it('wrong-tool when treatment obvious + tool routed but mis-matched', async () => { + const wrongTool: ParsedOutput = { + kind: 'tool_call', + name: 'add_metric_row_v0', + arguments: {}, + raw: '', + }; + const row = await scoreRun({ + prompt, + parsed: wrongTool, + apply: makeApply({ ok: true, doc: baseDoc }), + model: 'm', + variant: 'T', + }); + expect(row.routing).toBe('wrong-tool'); + expect(row.toolName).toBe('add_metric_row_v0'); + expect(row.expectedToolIfAny).toBe('add_card_row_v0'); + }); + + it('fallback when treatment obvious + output is batch_design', async () => { + const row = await scoreRun({ + prompt, + parsed: okParsed, + apply: makeApply({ ok: true, doc: baseDoc }), + model: 'm', + variant: 'T', + }); + expect(row.routing).toBe('fallback'); + }); + + it('n/a on baseline runs (routing is undefined there)', async () => { + const row = await scoreRun({ + prompt, + parsed: toolParsed, + apply: makeApply({ ok: true, doc: baseDoc }), + model: 'm', + variant: 'B', + }); + expect(row.routing).toBe('n/a'); + }); + + it('n/a on optional difficulty (expected_tool_if_any undefined there)', async () => { + const optionalPrompt: CorpusPrompt = { + ...prompt, + difficulty: 'optional', + expected_tool_if_any: undefined, + }; + const row = await scoreRun({ + prompt: optionalPrompt, + parsed: toolParsed, + apply: makeApply({ ok: true, doc: baseDoc }), + model: 'm', + variant: 'T', + }); + expect(row.routing).toBe('n/a'); + expect(row.expectedToolIfAny).toBe(''); + }); + + it("'garbage' (not 'n/a') on garbage outputs — stays in M5 denominator so right-tool rate doesn't inflate when most runs fail to parse", async () => { + const row = await scoreRun({ + prompt, + parsed: garbageParsed, + apply: makeApply({ ok: true, doc: baseDoc }), + model: 'm', + variant: 'T', + }); + expect(row.routing).toBe('garbage'); + }); + + it('n/a on garbage outputs in baseline (routing is undefined for baseline regardless of parse status)', async () => { + const row = await scoreRun({ + prompt, + parsed: garbageParsed, + apply: makeApply({ ok: true, doc: baseDoc }), + model: 'm', + variant: 'B', + }); + expect(row.routing).toBe('n/a'); + }); + + it('n/a on garbage outputs for optional prompts (routing is undefined regardless of parse status)', async () => { + const optionalPrompt: CorpusPrompt = { + ...prompt, + difficulty: 'optional', + expected_tool_if_any: undefined, + }; + const row = await scoreRun({ + prompt: optionalPrompt, + parsed: garbageParsed, + apply: makeApply({ ok: true, doc: baseDoc }), + model: 'm', + variant: 'T', + }); + expect(row.routing).toBe('n/a'); + }); +}); diff --git a/packages/pen-ai-skills/src/corpus/aggregate.ts b/packages/pen-ai-skills/src/corpus/aggregate.ts new file mode 100644 index 000000000..9dbbbbef6 --- /dev/null +++ b/packages/pen-ai-skills/src/corpus/aggregate.ts @@ -0,0 +1,136 @@ +import type { + CategorySummary, + CorpusPrompt, + ModelSummary, + Report, + ScoreRow, + ToolUsage, +} from './types'; + +/** + * Reduce a list of per-run ScoreRows into a single Report. Pure + * function — no I/O, deterministic output. Harness writes the report + * to disk in its own step. + * + * Grouping choices and why: + * - `byModel` keeps B and T side by side so the A/B delta is + * legible without cross-referencing rows. + * - `byCategory` collapses models so we can answer "does N-tool help + * mobile more than landing?". + * - `byTool` only counts treatment-arm runs where the model actually + * emitted a tool_call — the whole point of M5 diagnostics. + * + * Zero-sample cells (e.g. a model that had no baseline runs) return + * NaN for rates rather than 0 so a missing cell never looks like a 0% + * pass rate. + */ +export function aggregate(rows: ScoreRow[]): Report { + const models = unique(rows.map((r) => r.model)); + const byModel: ModelSummary[] = models.map((model) => buildModelSummary(model, rows)); + + const categories: CorpusPrompt['category'][] = ['mobile', 'dashboard', 'landing']; + const byCategory: CategorySummary[] = categories.map((cat) => buildCategorySummary(cat, rows)); + + const byTool: ToolUsage[] = buildToolUsage(rows); + + return { + generatedAt: new Date().toISOString(), + totalRuns: rows.length, + byModel, + byCategory, + byTool, + }; +} + +function buildModelSummary(model: string, rows: ScoreRow[]): ModelSummary { + const forModel = rows.filter((r) => r.model === model); + const baseline = forModel.filter((r) => r.variant === 'B'); + const treatment = forModel.filter((r) => r.variant === 'T'); + const m1_baseline = rate(baseline, (r) => r.m1_legal); + const m1_treatment = rate(treatment, (r) => r.m1_legal); + const m3_baseline = rate(baseline, (r) => r.m3_success); + const m3_treatment = rate(treatment, (r) => r.m3_success); + // Routing denominator = ALL obvious-treatment runs (routing !== 'n/a'). + // Garbage is a bucket of its own, not dropped — see the types.ts + // ScoreRow.routing JSDoc for why dropping garbage inflates the + // right-tool rate when most runs fail to parse. + const obviousRouted = treatment.filter((r) => r.difficulty === 'obvious' && r.routing !== 'n/a'); + const m5_right_tool = rate(obviousRouted, (r) => r.routing === 'right-tool'); + const m5_wrong_tool = rate(obviousRouted, (r) => r.routing === 'wrong-tool'); + const m5_fallback = rate(obviousRouted, (r) => r.routing === 'fallback'); + const m5_garbage = rate(obviousRouted, (r) => r.routing === 'garbage'); + return { + model, + m1_baseline, + m1_treatment, + m1_delta_pp: delta(m1_baseline, m1_treatment), + m3_baseline, + m3_treatment, + m3_delta_pp: delta(m3_baseline, m3_treatment), + m5_right_tool, + m5_wrong_tool, + m5_fallback, + m5_garbage, + runCountBaseline: baseline.length, + runCountTreatment: treatment.length, + }; +} + +function buildCategorySummary( + category: CorpusPrompt['category'], + rows: ScoreRow[], +): CategorySummary { + const forCat = rows.filter((r) => r.category === category); + const baseline = forCat.filter((r) => r.variant === 'B'); + const treatment = forCat.filter((r) => r.variant === 'T'); + const m1_baseline = rate(baseline, (r) => r.m1_legal); + const m1_treatment = rate(treatment, (r) => r.m1_legal); + return { + category, + m1_baseline, + m1_treatment, + m1_delta_pp: delta(m1_baseline, m1_treatment), + }; +} + +function buildToolUsage(rows: ScoreRow[]): ToolUsage[] { + const invocations = new Map(); + for (const r of rows) { + if (r.variant !== 'T') continue; + if (r.outputKind !== 'tool_call') continue; + if (!r.toolName) continue; + const cur = invocations.get(r.toolName) ?? { total: 0, successful: 0 }; + cur.total += 1; + if (r.m1_legal) cur.successful += 1; + invocations.set(r.toolName, cur); + } + return [...invocations.entries()] + .map(([tool, c]) => ({ + tool, + invocations: c.total, + successfulInvocations: c.successful, + })) + .sort((a, b) => b.invocations - a.invocations); +} + +function rate(items: T[], pred: (x: T) => boolean): number { + if (items.length === 0) return Number.NaN; + return items.filter(pred).length / items.length; +} + +function delta(baseline: number, treatment: number): number { + if (Number.isNaN(baseline) || Number.isNaN(treatment)) return Number.NaN; + return (treatment - baseline) * 100; +} + +function unique(arr: T[]): T[] { + const out: T[] = []; + const seen = new Set(); + for (const x of arr) { + if (!seen.has(x)) { + seen.add(x); + out.push(x); + } + } + return out; +} diff --git a/packages/pen-ai-skills/src/corpus/corpus-loader.ts b/packages/pen-ai-skills/src/corpus/corpus-loader.ts new file mode 100644 index 000000000..b1752f987 --- /dev/null +++ b/packages/pen-ai-skills/src/corpus/corpus-loader.ts @@ -0,0 +1,121 @@ +import { readdirSync, readFileSync, statSync } from 'node:fs'; +import { join } from 'node:path'; +import { load as loadYaml } from 'js-yaml'; +import type { CorpusPrompt } from './types'; + +const VALID_CATEGORIES = new Set(['mobile', 'dashboard', 'landing']); +const VALID_DIFFICULTIES = new Set(['obvious', 'optional']); + +/** + * Load every *.yaml file in the corpus directory. Validates the schema + * per entry and throws on malformed files so a typo doesn't silently + * drop a prompt from the run. Sort order is stable (filename ascending) + * so downstream score aggregation is deterministic. + * + * Pass the DIRECTORY containing the yaml files — not the repo root. + */ +export function loadCorpus(dir: string): CorpusPrompt[] { + let entries: string[]; + try { + entries = readdirSync(dir); + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + throw new Error(`loadCorpus: cannot read directory ${dir}: ${msg}`); + } + const files = entries.filter((f) => f.endsWith('.yaml')).sort(); + const prompts: CorpusPrompt[] = []; + for (const file of files) { + const fullPath = join(dir, file); + const st = statSync(fullPath); + if (!st.isFile()) continue; + const raw = readFileSync(fullPath, 'utf-8'); + const parsed = loadYaml(raw) as unknown; + const prompt = validateCorpusPrompt(parsed, file); + prompts.push(prompt); + } + const ids = new Set(); + for (const p of prompts) { + if (ids.has(p.id)) { + throw new Error(`loadCorpus: duplicate prompt id "${p.id}"`); + } + ids.add(p.id); + } + return prompts; +} + +function validateCorpusPrompt(raw: unknown, file: string): CorpusPrompt { + if (!raw || typeof raw !== 'object') { + throw new Error(`loadCorpus: ${file} is not a YAML object at the top level`); + } + const o = raw as Record; + const id = requireString(o.id, file, 'id'); + const category = requireString(o.category, file, 'category'); + if (!VALID_CATEGORIES.has(category)) { + throw new Error(`loadCorpus: ${file}: invalid category "${category}"`); + } + const difficulty = requireString(o.difficulty, file, 'difficulty'); + if (!VALID_DIFFICULTIES.has(difficulty)) { + throw new Error(`loadCorpus: ${file}: invalid difficulty "${difficulty}"`); + } + const prompt = requireString(o.prompt, file, 'prompt'); + const expected = validateExpected(o.expected, file); + const toolHint = o.expected_tool_if_any; + if (toolHint !== undefined && typeof toolHint !== 'string') { + throw new Error(`loadCorpus: ${file}: expected_tool_if_any must be a string when present`); + } + if (difficulty === 'obvious' && !toolHint) { + throw new Error( + `loadCorpus: ${file}: difficulty=obvious requires expected_tool_if_any (used to score M5 routing)`, + ); + } + return { + id, + category: category as CorpusPrompt['category'], + difficulty: difficulty as CorpusPrompt['difficulty'], + prompt, + expected, + expected_tool_if_any: toolHint, + }; +} + +function validateExpected(raw: unknown, file: string): CorpusPrompt['expected'] { + if (raw === undefined || raw === null) return {}; + if (typeof raw !== 'object') { + throw new Error(`loadCorpus: ${file}: expected must be an object or omitted`); + } + const o = raw as Record; + const out: CorpusPrompt['expected'] = {}; + if (o.must_contain_roles !== undefined) { + if (!Array.isArray(o.must_contain_roles)) { + throw new Error(`loadCorpus: ${file}: must_contain_roles must be an array`); + } + for (const r of o.must_contain_roles) { + if (typeof r !== 'string') { + throw new Error(`loadCorpus: ${file}: must_contain_roles entries must be strings`); + } + } + out.must_contain_roles = o.must_contain_roles as string[]; + } + if (o.min_roles !== undefined) { + if (!o.min_roles || typeof o.min_roles !== 'object' || Array.isArray(o.min_roles)) { + throw new Error(`loadCorpus: ${file}: min_roles must be an object of role→number`); + } + const m = o.min_roles as Record; + const validated: Record = {}; + for (const [role, count] of Object.entries(m)) { + if (typeof count !== 'number' || !Number.isFinite(count) || count < 0) { + throw new Error(`loadCorpus: ${file}: min_roles.${role} must be a non-negative number`); + } + validated[role] = count; + } + out.min_roles = validated; + } + return out; +} + +function requireString(v: unknown, file: string, key: string): string { + if (typeof v !== 'string' || v.length === 0) { + throw new Error(`loadCorpus: ${file}: field "${key}" must be a non-empty string`); + } + return v; +} diff --git a/packages/pen-ai-skills/src/corpus/index.ts b/packages/pen-ai-skills/src/corpus/index.ts new file mode 100644 index 000000000..ef2822f51 --- /dev/null +++ b/packages/pen-ai-skills/src/corpus/index.ts @@ -0,0 +1,16 @@ +export { loadCorpus } from './corpus-loader'; +export { parseModelOutput } from './output-parser'; +export { scoreRun, countRoles } from './score-run'; +export { aggregate } from './aggregate'; +export type { + ApplyFn, + ApplyResult, + CategorySummary, + CorpusExpected, + CorpusPrompt, + ModelSummary, + ParsedOutput, + Report, + ScoreRow, + ToolUsage, +} from './types'; diff --git a/packages/pen-ai-skills/src/corpus/js-yaml.d.ts b/packages/pen-ai-skills/src/corpus/js-yaml.d.ts new file mode 100644 index 000000000..a495d35ee --- /dev/null +++ b/packages/pen-ai-skills/src/corpus/js-yaml.d.ts @@ -0,0 +1,7 @@ +// Local ambient declaration for `js-yaml`. The package ships without +// types; adding `@types/js-yaml` as a dev dep would churn the root +// package.json for one call site. Keep the surface minimal — just the +// one function `corpus-loader.ts` uses. Extend as needed. +declare module 'js-yaml' { + export function load(s: string): unknown; +} diff --git a/packages/pen-ai-skills/src/corpus/output-parser.ts b/packages/pen-ai-skills/src/corpus/output-parser.ts new file mode 100644 index 000000000..410e8561c --- /dev/null +++ b/packages/pen-ai-skills/src/corpus/output-parser.ts @@ -0,0 +1,349 @@ +import type { ParsedOutput } from './types'; + +/** + * The in-prompt tool-call convention used by the treatment variant: + * + * {"name": "add_X_v0", "arguments": {...}} + * + * Matches the A/B plan §2 "Option Y — in-prompt simulation". The tag + * `op_tool` (not `tool_call`) avoids confusion with Claude's native + * tool_use blocks so reference Claude runs stay parseable. + * + * Convention intentionally permits one call per output — the plan's + * "element tool OR batch_design, not both" rule. Multi-call support + * can come later if the corpus grows composite prompts. + */ +const OP_TOOL_RE_G = /\s*([\s\S]*?)\s*<\/op_tool>/g; + +const ELEMENT_TOOL_NAME_RE = /^add_[a-z_]+_v0$/; + +/** + * Reasoning-model chain-of-thought wrapper. MiniMax M-series, + * DeepSeek-R1, and some others emit `...` before the + * actual answer. Strip it defensively so the parser sees the real + * output; no legitimate design payload should contain these tags. + * Greedy to the LAST `` so nested / multiple blocks collapse + * cleanly. + */ +const THINK_RE = /[\s\S]*?<\/think>\s*/gi; + +/** + * Heuristic marker for a batch_design DSL payload. + * + * `DSL_AT_START` requires the string's FIRST non-whitespace content + * to be a DSL operation — any of the three executeLine forms + * (binding=I/C/R/M/G, bindless I/C/R/G, or standalone U/D/M). Used + * as the cheap first-pass check before the stricter `isCleanDsl` + * full scan. + */ +const DSL_AT_START = /^\s*(?:\w+\s*=\s*[ICRMG]\s*\(|[ICRGUDM]\s*\()/; + +/** + * Shape of a single DSL operation line after top-level splitting. + * Mirrors the THREE regexes in + * `packages/pen-mcp/src/tools/batch-design.ts::executeLine`: + * + * assignMatch: /^(\w+)\s*=\s*([ICRMG])\((.+)\)$/ + * bindlessAssignMatch: /^([ICRG])\((.+)\)$/ // I/C/R/G only + * callMatch: /^([UDM])\((.+)\)$/ + * + * Combined: the line must start with either `word=[ICRMG]` or a bare + * opcode letter `[ICRGUDM]` and end with `)`. Keeping this in sync + * with executeLine's regexes ensures a payload `isCleanDsl` accepts + * is also parsed cleanly by `handleBatchDesign` — the whole point + * of the strict check is to avoid letting mixed content through that + * later fails line-by-line in the downstream parser. + */ +const DSL_OP_LINE = /^(?:\w+\s*=\s*[ICRMG]|[ICRGUDM])\([\s\S]+\)$/; + +/** + * Verify a payload is CLEAN batch_design DSL — no prose preambles, + * no interleaved explanation lines, no trailing commentary. Uses the + * same string-aware brace/paren/bracket splitter as `handleBatchDesign` + * so "one logical op" stays one chunk even when pretty-printed JSON + * spans multiple physical lines. + * + * Why a full scan rather than just `DSL_AT_START`: a model may emit + * `root=I(null, {...})\nLet me know if that works!\nnav=I(root, {...})` + * which passes `DSL_AT_START` but the middle "Let me know..." line + * splits into its own chunk that isn't a valid DSL op. Feeding the + * whole thing to handleBatchDesign produces a noisy per-line error + * array instead of a clean apply result — and the scorer ends up + * classifying the run as "fallback" (model chose batch_design) when + * it should really be "garbage" (model couldn't emit valid DSL). + */ +export function isCleanDsl(payload: string): boolean { + if (!DSL_AT_START.test(payload)) return false; + const chunks = splitDslTopLevel(payload); + if (chunks.length === 0) return false; + for (const chunk of chunks) { + if (!DSL_OP_LINE.test(chunk)) return false; + } + return true; +} + +/** + * Mirror of `splitOperations` in pen-mcp's batch-design.ts — splits + * on newlines that are OUTSIDE quoted strings and balanced brackets. + * Kept as a local copy so the corpus package doesn't depend on + * pen-mcp internals (that would create the same cycle the scorer + * itself was structured to avoid). + */ +function splitDslTopLevel(raw: string): string[] { + const result: string[] = []; + let buf = ''; + let depth = 0; + let inString = false; + let escape = false; + for (let i = 0; i < raw.length; i += 1) { + const ch = raw[i]; + buf += ch; + if (escape) { + escape = false; + continue; + } + if (ch === '\\' && inString) { + escape = true; + continue; + } + if (ch === '"') { + inString = !inString; + continue; + } + if (inString) continue; + if (ch === '(' || ch === '[' || ch === '{') depth += 1; + else if (ch === ')' || ch === ']' || ch === '}') depth -= 1; + else if (ch === '\n' && depth === 0) { + const trimmed = buf.trim(); + if (trimmed && !trimmed.startsWith('//')) result.push(trimmed); + buf = ''; + } + } + const tail = buf.trim(); + if (tail && !tail.startsWith('//')) result.push(tail); + return result; +} + +/** + * Optional prefix / suffix the model might wrap around its DSL payload: + * ``` + * ... or explanatory prose ... + * ``` + * strip the markdown fence and surrounding prose when extracting the + * DSL so downstream `handleBatchDesign` gets a clean payload. + */ +const CODE_FENCE_RE = /```(?:\w+)?\s*([\s\S]*?)```/; + +/** + * Parse a raw model output into a tagged union. Order of checks: + * + * 1. `{...}` — treatment-arm tool call + * 2. Code-fenced block whose inner text matches the batch_design + * pattern → batch_design DSL + * 3. Free-form text that still matches the batch_design pattern → + * batch_design DSL (fence optional) + * 4. Otherwise garbage + * + * Never throws — garbage responses get a `kind: 'garbage'` tag with a + * human-readable `reason` so aggregation can distinguish parse failure + * from apply failure. + */ +export function parseModelOutput(raw: string): ParsedOutput { + if (typeof raw !== 'string' || raw.trim().length === 0) { + return { kind: 'garbage', reason: 'empty output', raw: raw ?? '' }; + } + + // Strip reasoning-model blocks before pattern-matching. + // `raw` field in the return value keeps the full original for + // debugging; parse logic sees the cleaned text. + const cleaned = raw.replace(THINK_RE, '').trim(); + + // Extract every `` tag (models frequently emit multiple when + // composing). We prefer the FIRST element-tool call over any + // `batch_design` scaffold tags, because intent matching is what M5 + // measures — scaffolding doesn't reflect routing quality. + // + // When a tag body fails to parse as JSON we ALSO inspect whether + // the body happens to contain raw DSL (seen in the wild: some + // models wrap DSL directly inside `` tags without the + // JSON wrapper). Such a tag should be recoverable as batch_design + // rather than wasted as garbage. + const tags: ParsedOpTool[] = []; + let firstParseFailureReason: string | null = null; + let dslFromMalformedTag: string | null = null; + for (const match of cleaned.matchAll(OP_TOOL_RE_G)) { + const body = match[1].trim(); + const parsed = parseOpTool(body); + if (parsed.ok) { + tags.push(parsed.value); + continue; + } + if (firstParseFailureReason === null) firstParseFailureReason = parsed.reason; + // Only recover a malformed-tag body as DSL when the body is + // CLEAN DSL end-to-end — otherwise we'd be feeding prose to + // handleBatchDesign, which just produces a pile of per-line + // parse errors and surfaces a useless "invalid DSL" result. + if (dslFromMalformedTag === null && isCleanDsl(body)) { + dslFromMalformedTag = body; + } + } + + // Preserved across tag-processing so we can emit a specific garbage + // reason after DSL fallback also fails. `null` means we never + // encountered a batch_design tag with bad operations. + let batchDesignUnusableReason: string | null = null; + if (tags.length > 0) { + const elementCall = tags.find((t) => ELEMENT_TOOL_NAME_RE.test(t.name)); + if (elementCall) { + return { + kind: 'tool_call', + name: elementCall.name, + arguments: elementCall.arguments, + raw, + }; + } + const batchCall = tags.find((t) => t.name === 'batch_design'); + if (batchCall) { + const dsl = extractDslFromBatchDesignArgs(batchCall.arguments); + if (dsl) return { kind: 'batch_design', dsl, raw }; + batchDesignUnusableReason = '{name:"batch_design"} had no usable `operations` field'; + // Fall through to DSL-outside-tag detection rather than silently + // drop the tag. + } else { + // Tag present but with a name that's neither an element tool + // nor batch_design — unknown tool call. Keep the first as-is + // so the scorer records it as a tool_call (routing='wrong-tool' + // for obvious prompts, since it's not the expected tool). + return { + kind: 'tool_call', + name: tags[0].name, + arguments: tags[0].arguments, + raw, + }; + } + } + + // DSL detection — runs AFTER tag parsing so valid DSL in the + // response body (or wrapped in a malformed tag) isn't lost to an + // early "parse error" short-circuit. Every path uses `isCleanDsl` + // so we only return payloads that are CLEAN DSL end-to-end — a + // mixed payload with prose interleaved between ops would pass + // splitOperations as a prose chunk and fail regex match inside + // handleBatchDesign, producing a noisy error array. Classifying + // those as garbage (not batch_design) gives the scorer a truer + // routing signal. + // + // Order of precedence: + // 1. Fenced block whose content is clean DSL + // 2. Text OUTSIDE all `` tags is clean DSL (common case + // of a tag followed by raw DSL on new lines) + // 3. Clean DSL recovered from inside a malformed `` body + const fenceMatch = cleaned.match(CODE_FENCE_RE); + if (fenceMatch) { + const fenceBody = fenceMatch[1].trim(); + if (isCleanDsl(fenceBody)) { + return { kind: 'batch_design', dsl: fenceBody, raw }; + } + } + const outsideTags = cleaned.replace(OP_TOOL_RE_G, '').trim(); + if (outsideTags.length > 0 && isCleanDsl(outsideTags)) { + return { kind: 'batch_design', dsl: outsideTags, raw }; + } + if (dslFromMalformedTag) { + return { kind: 'batch_design', dsl: dslFromMalformedTag, raw }; + } + + // Nothing usable remains. Prefer specific reasons over generic + // ones, in priority order: "batch_design tag had unusable + // operations" → " tag parse failure" → generic "no DSL". + if (batchDesignUnusableReason !== null) { + return { kind: 'garbage', reason: batchDesignUnusableReason, raw }; + } + if (firstParseFailureReason !== null) { + return { kind: 'garbage', reason: firstParseFailureReason, raw }; + } + return { + kind: 'garbage', + reason: 'no block and no batch_design DSL lines detected', + raw, + }; +} + +interface ParsedOpTool { + name: string; + arguments: Record; +} + +type ParseOpToolResult = { ok: true; value: ParsedOpTool } | { ok: false; reason: string }; + +function parseOpTool(payload: string): ParseOpToolResult { + let obj: unknown; + try { + obj = JSON.parse(payload); + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + return { ok: false, reason: ` payload was not valid JSON: ${msg}` }; + } + if (!obj || typeof obj !== 'object' || Array.isArray(obj)) { + return { ok: false, reason: ' payload must be an object' }; + } + const o = obj as Record; + const name = o.name; + const args = o.arguments; + if (typeof name !== 'string' || name.length === 0) { + return { ok: false, reason: ' missing string "name"' }; + } + if (!args || typeof args !== 'object' || Array.isArray(args)) { + return { ok: false, reason: ' missing object "arguments"' }; + } + return { ok: true, value: { name, arguments: args as Record } }; +} + +/** + * Models sometimes wrap `batch_design` in an `` tag (e.g. when + * the baseline prompt teaches the tool-call format uniformly). Unpack + * the `operations` argument into a DSL string. Accepts either: + * - `operations: "DSL text"` — passed through + * - `operations: [{type, ...}, ...]` — serialized back to DSL lines + * (best effort; unknown op shapes are dropped). + * Returns null if no usable form is found. + */ +function extractDslFromBatchDesignArgs(args: Record): string | null { + const ops = args.operations; + if (typeof ops === 'string' && ops.trim().length > 0) return ops; + if (Array.isArray(ops) && ops.length > 0) { + const lines: string[] = []; + for (const op of ops) { + if (!op || typeof op !== 'object') continue; + const line = serializeOpToDsl(op as Record); + if (line) lines.push(line); + } + if (lines.length > 0) return lines.join('\n'); + } + return null; +} + +function serializeOpToDsl(op: Record): string | null { + // Minimal shim — covers the shape most models invent when they treat + // batch_design's operations as a structured array. Not a full DSL + // reimplementation. If a model emits some other shape, apply will + // throw and the run scores as M1=false (correct behavior — the + // model produced unusable output). + const type = op.type; + if (typeof type !== 'string') return null; + const id = op.id; + const node = op.node; + const parent = op.parent ?? null; + if (type === 'I' && node && typeof node === 'object') { + const parentRef = parent == null ? 'null' : JSON.stringify(parent); + const binding = typeof id === 'string' && id.length > 0 ? id : 'n'; + return `${binding}=I(${parentRef}, ${JSON.stringify(node)})`; + } + if (type === 'U' && typeof id === 'string' && op.props && typeof op.props === 'object') { + return `U(${JSON.stringify(id)}, ${JSON.stringify(op.props)})`; + } + if (type === 'D' && typeof id === 'string') { + return `D(${JSON.stringify(id)})`; + } + return null; +} diff --git a/packages/pen-ai-skills/src/corpus/score-run.ts b/packages/pen-ai-skills/src/corpus/score-run.ts new file mode 100644 index 000000000..720949749 --- /dev/null +++ b/packages/pen-ai-skills/src/corpus/score-run.ts @@ -0,0 +1,166 @@ +import type { PenDocument, PenNode } from '@zseven-w/pen-types'; +import { detectAllIssues } from '../diagnostics/detectors'; +import type { ApplyFn, CorpusPrompt, ParsedOutput, ScoreRow } from './types'; + +/** + * Walk a PenNode tree and tally `role` occurrences. Roles are optional + * on PenNode so we skip nodes without one. Used for expected-shape + * checks (must_contain_roles / min_roles). + */ +export function countRoles(doc: PenDocument): Record { + const counts: Record = {}; + const walk = (node: PenNode): void => { + const role = (node as { role?: string }).role; + if (typeof role === 'string' && role.length > 0) { + counts[role] = (counts[role] ?? 0) + 1; + } + const kids = (node as { children?: unknown }).children; + if (Array.isArray(kids)) { + for (const k of kids) { + if (k && typeof k === 'object') walk(k as PenNode); + } + } + }; + const topLevel: PenNode[] = + (doc.children as PenNode[] | undefined) ?? + (doc.pages?.[0]?.children as PenNode[] | undefined) ?? + []; + for (const n of topLevel) walk(n); + return counts; +} + +/** + * Evaluate a single model output against a corpus prompt. Runs the + * caller-supplied `apply` (which in turn calls into pen-mcp), then + * layers legality + expected-shape checks on top. + * + * M1 legality = apply returned ok AND zero `severity: 'error'` issues + * from detectAllIssues. (All current detectors emit + * 'warning', so M1 today is essentially "apply ok". + * When detectors add error-severity classes M1 tightens + * automatically with no API change.) + * + * M3 success = M1 passes AND every `expected.must_contain_roles` role + * appears at least once AND every `expected.min_roles` + * entry meets its threshold. + */ +export async function scoreRun(args: { + prompt: CorpusPrompt; + parsed: ParsedOutput; + apply: ApplyFn; + model: string; + variant: 'B' | 'T'; +}): Promise { + const { prompt, parsed, apply, model, variant } = args; + const toolName = parsed.kind === 'tool_call' ? parsed.name : ''; + const base = { + promptId: prompt.id, + category: prompt.category, + difficulty: prompt.difficulty, + model, + variant, + outputKind: parsed.kind, + toolName, + expectedToolIfAny: prompt.expected_tool_if_any ?? '', + routing: classifyRouting(prompt, variant, parsed.kind, toolName), + rawOutput: parsed.raw, + }; + + if (parsed.kind === 'garbage') { + return { + ...base, + m1_legal: false, + m3_success: false, + m3_failure_reason: `output unparseable: ${parsed.reason}`, + issues: [], + applyError: '', + }; + } + + const applied = await apply(parsed); + if (!applied.ok || !applied.doc) { + return { + ...base, + m1_legal: false, + m3_success: false, + m3_failure_reason: 'apply failed before shape checks', + issues: [], + applyError: applied.error, + }; + } + + const errorIssues: ReturnType = []; + const allIssues: ReturnType = []; + const topLevel = (applied.doc.children ?? applied.doc.pages?.[0]?.children ?? []) as PenNode[]; + for (const root of topLevel) { + const found = detectAllIssues(root, applied.doc); + for (const f of found) { + allIssues.push(f); + if (f.severity === 'error') errorIssues.push(f); + } + } + + const m1 = errorIssues.length === 0; + const shape = checkExpectedShape(prompt, applied.doc); + + return { + ...base, + m1_legal: m1, + m3_success: m1 && shape.ok, + m3_failure_reason: m1 ? shape.reason : 'M1 failed — skipped shape checks', + issues: allIssues, + applyError: '', + }; +} + +/** + * Classify a run's routing verdict. Only meaningful on + * `difficulty: 'obvious'` treatment runs — baseline and optional both + * map to 'n/a' because the concept "right tool vs wrong tool" isn't + * defined there. Used by aggregate to split routing into 4 categories + * (right-tool / wrong-tool / fallback / garbage) that sum to 100% of + * the obvious-treatment subset. + * + * Garbage is counted as its own category rather than 'n/a' because + * otherwise a model that emits 10 unparseable outputs + 1 right + 1 + * wrong would score 50% right-tool — hiding the fact that only 2/12 + * runs were actually usable. With garbage in the denominator, the + * right-tool rate stays interpretable on its own. + */ +function classifyRouting( + prompt: CorpusPrompt, + variant: 'B' | 'T', + kind: ParsedOutput['kind'], + toolName: string, +): ScoreRow['routing'] { + if (variant !== 'T') return 'n/a'; + if (prompt.difficulty !== 'obvious') return 'n/a'; + if (kind === 'garbage') return 'garbage'; + if (kind === 'batch_design') return 'fallback'; + // kind === 'tool_call' + return toolName === prompt.expected_tool_if_any ? 'right-tool' : 'wrong-tool'; +} + +function checkExpectedShape( + prompt: CorpusPrompt, + doc: PenDocument, +): { ok: boolean; reason: string } { + const counts = countRoles(doc); + const missing: string[] = []; + for (const role of prompt.expected.must_contain_roles ?? []) { + if ((counts[role] ?? 0) === 0) missing.push(role); + } + if (missing.length > 0) { + return { ok: false, reason: `missing required role(s): ${missing.join(', ')}` }; + } + const underMin: string[] = []; + for (const [role, min] of Object.entries(prompt.expected.min_roles ?? {})) { + if ((counts[role] ?? 0) < min) { + underMin.push(`${role}<${min} (got ${counts[role] ?? 0})`); + } + } + if (underMin.length > 0) { + return { ok: false, reason: `role counts below minimum: ${underMin.join(', ')}` }; + } + return { ok: true, reason: '' }; +} diff --git a/packages/pen-ai-skills/src/corpus/types.ts b/packages/pen-ai-skills/src/corpus/types.ts new file mode 100644 index 000000000..0db1e415b --- /dev/null +++ b/packages/pen-ai-skills/src/corpus/types.ts @@ -0,0 +1,157 @@ +import type { PenDocument } from '@zseven-w/pen-types'; +import type { Issue } from '../diagnostics/types'; + +/** + * Corpus prompt definition — matches the YAML schema under + * `packages/pen-ai-skills/corpus/ab-v0/*.yaml`. + */ +export interface CorpusPrompt { + id: string; + category: 'mobile' | 'dashboard' | 'landing'; + difficulty: 'obvious' | 'optional'; + prompt: string; + expected: CorpusExpected; + expected_tool_if_any?: string; +} + +export interface CorpusExpected { + /** Roles whose presence somewhere in the rendered tree is required. */ + must_contain_roles?: string[]; + /** role → minimum instance count. Failing any key → M3 = 0. */ + min_roles?: Record; +} + +/** + * Parsed model output. Weak models can emit either an element-tool call + * (our treatment-arm simulation) or the legacy `batch_design` DSL; we + * detect which and surface a tagged union so the harness dispatches + * correctly. `garbage` captures parse failures — both tags missing or + * malformed payload. + */ +export type ParsedOutput = + | { kind: 'tool_call'; name: string; arguments: Record; raw: string } + | { kind: 'batch_design'; dsl: string; raw: string } + | { kind: 'garbage'; reason: string; raw: string }; + +/** + * Abstracted apply-to-document interface. The scorer consumes this; the + * harness implements it on top of `pen-mcp`'s `handleBatchDesign` + + * element-tool handlers. Kept as an interface so the scorer stays + * pen-mcp-independent (avoids the pen-ai-skills ↔ pen-mcp cycle). + */ +export interface ApplyResult { + /** True iff apply ran without throwing and produced a non-null root. */ + ok: boolean; + /** Error message when ok=false, empty string when ok=true. */ + error: string; + /** The resulting document state, or null when apply failed. */ + doc: PenDocument | null; +} + +export type ApplyFn = (parsed: ParsedOutput) => Promise; + +/** + * Single-run score row. One row per (model, variant, prompt) triple. + */ +export interface ScoreRow { + promptId: string; + category: CorpusPrompt['category']; + difficulty: CorpusPrompt['difficulty']; + model: string; + variant: 'B' | 'T'; + outputKind: ParsedOutput['kind']; + /** Tool name when outputKind='tool_call', else empty. */ + toolName: string; + /** Copied from the prompt so the aggregator can score routing without + * re-joining against the corpus. Empty for `difficulty: 'optional'` + * (M5 routing is undefined there). */ + expectedToolIfAny: string; + /** Routing verdict on `difficulty: 'obvious'` treatment runs only. + * Values: + * - 'right-tool' — outputKind=tool_call AND toolName=expectedToolIfAny + * - 'wrong-tool' — outputKind=tool_call AND toolName≠expectedToolIfAny + * - 'fallback' — outputKind=batch_design (didn't route to any tool) + * - 'garbage' — outputKind=garbage (unparseable; nothing to route) + * - 'n/a' — baseline run OR `difficulty: 'optional'` + * Aggregate's denominator = obvious-treatment runs (routing !== 'n/a'), + * so the four rates sum to 1. Including garbage in the denominator + * prevents "right-tool rate looks great while most outputs are + * broken" (e.g. 10 garbage + 1 right + 1 wrong would otherwise report + * 50% right-tool even though only 2/12 runs were usable). */ + routing: 'right-tool' | 'wrong-tool' | 'fallback' | 'garbage' | 'n/a'; + /** M1: parse + apply + no detector errors. */ + m1_legal: boolean; + /** M3: M1 AND expected shape checks pass. */ + m3_success: boolean; + /** Non-empty when the expected shape check failed — for diagnosis. */ + m3_failure_reason: string; + /** Detector issues found on the applied tree (empty when apply failed). */ + issues: Issue[]; + /** Apply-phase error (empty on success). */ + applyError: string; + /** Raw model output — kept for debugging, not for aggregation. */ + rawOutput: string; +} + +/** + * Aggregated report across a set of ScoreRows. Emitted as report.md + + * report.json by the harness. + */ +export interface Report { + /** Timestamp when the report was generated (ISO 8601). */ + generatedAt: string; + /** How many rows this report summarizes. */ + totalRuns: number; + /** Per-model summary rows. */ + byModel: ModelSummary[]; + /** Per-category summary rows (both variants collapsed). */ + byCategory: CategorySummary[]; + /** Per-tool usage: which element tools got invoked in treatment runs. */ + byTool: ToolUsage[]; +} + +export interface ModelSummary { + model: string; + /** M1 rate in baseline variant (0..1). */ + m1_baseline: number; + /** M1 rate in treatment variant (0..1). */ + m1_treatment: number; + /** Treatment - baseline, expressed in percentage points. */ + m1_delta_pp: number; + m3_baseline: number; + m3_treatment: number; + m3_delta_pp: number; + /** Routing breakdown on the `difficulty: 'obvious'` treatment subset — + * four rates that sum to 1 (modulo NaN when the subset is empty). + * Including garbage in the denominator (rather than silently + * dropping it) prevents right-tool rates from looking rosy when + * most outputs failed to parse. Splits: + * - right-tool: routed to expected_tool_if_any + * - wrong-tool: routed to an element tool BUT not the expected one + * - fallback: emitted batch_design instead of any tool + * - garbage: output unparseable — counted here so the denominator + * is "all obvious treatment runs" and the rate is + * interpretable without cross-referencing M1. */ + m5_right_tool: number; + m5_wrong_tool: number; + m5_fallback: number; + m5_garbage: number; + /** Run counts for each cell so we can show sample size in the report. */ + runCountBaseline: number; + runCountTreatment: number; +} + +export interface CategorySummary { + category: CorpusPrompt['category']; + m1_baseline: number; + m1_treatment: number; + m1_delta_pp: number; +} + +export interface ToolUsage { + tool: string; + /** How many times this tool was invoked across all treatment runs. */ + invocations: number; + /** How many of those invocations passed M1. */ + successfulInvocations: number; +} diff --git a/packages/pen-ai-skills/src/index.ts b/packages/pen-ai-skills/src/index.ts index a5f0699d5..46e6e7dcd 100644 --- a/packages/pen-ai-skills/src/index.ts +++ b/packages/pen-ai-skills/src/index.ts @@ -31,3 +31,6 @@ export { // Diagnostics (pure detectors for design quality issues) export * from './diagnostics'; + +// Corpus (A/B evaluation harness types + pure scoring logic) +export * from './corpus';