From 0f22f2dd3ef626488d1b6171cb330b62e1a95de3 Mon Sep 17 00:00:00 2001 From: Fini Date: Wed, 29 Apr 2026 08:50:00 +0800 Subject: [PATCH] test(ab-corpus): composite multi-tool stub fixtures for dry-run MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds stub-model FIXTURES entries for all 5 ab-v3 composite prompts so `--dry-run --corpus ab-v3` exercises the new tool_calls list + apply loop end-to-end without burning live credits. Each fixture is a multi-tag raw string: - dashboard-settings-page-composite → 1× section_header + 4× setting_row - dashboard-team-people-page-composite → 1× section_header + 5× member_row + 1× invite_row - dashboard-search-filters-composite → 2× filter_group - dashboard-audit-feed-composite → 1× section_header + 6× activity_log - mobile-onboarding-flow-composite → 4× step_card Verifies the parser/apply/aggregate pipeline end-to-end: - parseModelOutput surfaces all element-tool tags in emit order (e.g. 7 tags → 7 calls in tool_calls) - apply.ts loops `handleElementToolCall` for every call - countRoles in scoreRun finds every required role across the multi-tool tree (so M3 min_roles passes) - aggregate.byTool tallies per-name (12 add_activity_log + 10 add_member_row + 8 add_setting_row + 8 add_step_card + 4 add_filter_group + 4 add_section_header + 2 add_invite_row in a 2-model dry-run) Result on `--dry-run --corpus ab-v3`: - Composite routing 100% multi-tool (was 100% fallback before fixtures — stub default emitted batch_design) - M3 T 9.6% (5 composite passes / 52 prompts) — was 0% before, proving every composite-T row genuinely apply-passes M3 --- scripts/ab-corpus/stub-model.ts | 41 +++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/scripts/ab-corpus/stub-model.ts b/scripts/ab-corpus/stub-model.ts index 4fbbba57e..4c87e535d 100644 --- a/scripts/ab-corpus/stub-model.ts +++ b/scripts/ab-corpus/stub-model.ts @@ -123,6 +123,47 @@ const FIXTURES: Record = { '}}', '', ].join('\n'), + // Composite multi-tool fixtures — exercise the new tool_calls list + // path end-to-end without burning live API credits. Each value is a + // multi-tag raw string the parser will split into multiple element- + // tool calls (apply.ts loops over them, byTool tallies per-name). + // These are the only stubs that produce >1 op_tool block; obvious + // prompts still emit 1 by default. + 'dashboard-settings-page-composite|T': [ + '{"name":"add_section_header_v0","arguments":{"title":"Notifications","subtitle":"Choose what you want to be alerted about."}}', + '{"name":"add_setting_row_v0","arguments":{"title":"Email digest","subtitle":"Daily summary at 9am","leading_icon":"bell","trailing":{"kind":"switch","on":true}}}', + '{"name":"add_setting_row_v0","arguments":{"title":"Push notifications","subtitle":"All channels","leading_icon":"phone","trailing":{"kind":"switch","on":true}}}', + '{"name":"add_setting_row_v0","arguments":{"title":"Direct messages","subtitle":"Mentions only","leading_icon":"message-circle","trailing":{"kind":"switch","on":false}}}', + '{"name":"add_setting_row_v0","arguments":{"title":"Critical alerts","subtitle":"Always on","leading_icon":"alert-triangle","trailing":{"kind":"switch","on":true}}}', + ].join('\n'), + 'dashboard-team-people-page-composite|T': [ + '{"name":"add_section_header_v0","arguments":{"title":"Members","subtitle":"5 people"}}', + '{"name":"add_member_row_v0","arguments":{"name":"Sarah Lee","subtitle":"sarah@acme.com","initial":"S","trailing":{"kind":"role_badge","value":"Owner"}}}', + '{"name":"add_member_row_v0","arguments":{"name":"Marcus Chen","subtitle":"marcus@acme.com","initial":"M","trailing":{"kind":"role_badge","value":"Admin"}}}', + '{"name":"add_member_row_v0","arguments":{"name":"Aiko Tanaka","subtitle":"aiko@acme.com","initial":"A","trailing":{"kind":"role_badge","value":"Editor"}}}', + '{"name":"add_member_row_v0","arguments":{"name":"Raj Patel","subtitle":"raj@acme.com","initial":"R","trailing":{"kind":"role_badge","value":"Editor"}}}', + '{"name":"add_member_row_v0","arguments":{"name":"Jordan Kim","subtitle":"jordan@acme.com","initial":"J","trailing":{"kind":"role_badge","value":"Viewer"}}}', + '{"name":"add_invite_row_v0","arguments":{"email":"leon@acme.com","role":"Editor","status":"pending","action_label":"Resend"}}', + ].join('\n'), + 'dashboard-search-filters-composite|T': [ + '{"name":"add_filter_group_v0","arguments":{"title":"Category","options":[{"label":"Apparel","count":124,"selected":true},{"label":"Footwear","count":86},{"label":"Bags","count":41},{"label":"Accessories","count":67}]}}', + '{"name":"add_filter_group_v0","arguments":{"title":"Brand","options":[{"label":"Nike","count":32},{"label":"Adidas","count":28},{"label":"Patagonia","count":15,"selected":true},{"label":"Arc Teryx","count":9}]}}', + ].join('\n'), + 'dashboard-audit-feed-composite|T': [ + '{"name":"add_section_header_v0","arguments":{"title":"Recent activity","subtitle":"Last 24 hours"}}', + '{"name":"add_activity_log_v0","arguments":{"actor":"Sarah Lee","action":"approved the production deploy","timestamp":"2h ago","icon":"check","tone":"success"}}', + '{"name":"add_activity_log_v0","arguments":{"actor":"Marcus Chen","action":"uploaded final-mockups.zip","timestamp":"3h ago","icon":"upload","tone":"info"}}', + '{"name":"add_activity_log_v0","arguments":{"actor":"Aiko Tanaka","action":"invited jordan@acme.com to the workspace","timestamp":"5h ago","icon":"user-plus","tone":"info"}}', + '{"name":"add_activity_log_v0","arguments":{"actor":"System","action":"rate-limited an IP after 50 failed sign-ins","timestamp":"8h ago","icon":"alert-triangle","tone":"warning"}}', + '{"name":"add_activity_log_v0","arguments":{"actor":"Raj Patel","action":"updated billing details","timestamp":"yesterday","icon":"settings","tone":"neutral"}}', + '{"name":"add_activity_log_v0","arguments":{"actor":"Sarah Lee","action":"deleted archive-2024.zip","timestamp":"yesterday","icon":"trash","tone":"danger"}}', + ].join('\n'), + 'mobile-onboarding-flow-composite|T': [ + '{"name":"add_step_card_v0","arguments":{"number":1,"title":"Create your account","description":"Use your email and a strong password to sign up. No credit card required."}}', + '{"name":"add_step_card_v0","arguments":{"number":2,"title":"Connect your bank","description":"Link your account in seconds. We use 256-bit encryption to keep your data safe."}}', + '{"name":"add_step_card_v0","arguments":{"number":3,"title":"Set your goals","description":"Tell us what you want to save for — we will do the rest."}}', + '{"name":"add_step_card_v0","arguments":{"number":"4","title":"You are all set","description":"You are ready to use the app. Tap below to continue.","completed":true}}', + ].join('\n'), // Baseline garbage response (simulates a weak model producing prose) 'mobile-tabs-demo|B': 'I can help you design that! Let me create a tabs demo for you with three underline tabs.',