From 7305dbe634e1ec7167d32afae9ed6b16e9f0bb0b Mon Sep 17 00:00:00 2001 From: Anton Soldatov Date: Sun, 15 Mar 2026 15:42:41 +0300 Subject: [PATCH] AI chat polishing: skeleton workflow, batched tools, visual feedback, stock photos (#111) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fix text rendering: always use paragraph wrapping, font fallback, batch tools, severity levels Rendering: - renderText: always use paragraph when fontProvider available, not just when node-specific font loaded — fixes single-line overflow for Georgia/ non-Inter fonts - buildParagraph: add Inter as fallback fontFamily so paragraph wraps text even when requested font not yet loaded - measureTextNode/buildTextPicture: remove isNodeFontLoaded gate — measure with fallback font for consistent layout↔rendering - TEXT_PICTURE_KEYS: add width/height so textPicture invalidates on resize - renderText drawText fallback: clip to node bounds - onAfterExecute: invalidate textPictures after font load Layout: - estimateTextSize: accept maxWidth, compute multi-line height - toJSON: depth parameter with childCount fallback AI tools: - batch_update tool: multiple set_layout/set_layout_child in one call - describe: accept ids array for multi-node inspection - Issue severity (error/warning/info) with pattern classification - checkNestedFlexWithoutFill: cross-axis fill tolerance - CORE_TOOLS trimmed 34→27, rest moved to EXTENDED - System prompt: batch_update, describe ids, severity rules, font/text wrapping guidance, card grid patterns * Add Pexels stock photo integration for AI chat - stock_photo tool: search Pexels by query, download image, apply as IMAGE fill on any node. Picks resolution based on node dimensions. - Pexels API key stored in localStorage, synced to core via setPexelsApiKey on change - ProviderSettings UI: Pexels API key input with link to get free key - System prompt: stock photo usage guidance - Added to CORE_TOOLS so AI can use it in every session * Batch stock_photo + calc, 4-phase workflow, fix wrap false positives - stock_photo: accepts JSON array of requests, all fetched in parallel. Rejects nodes with children (leaf shapes only). One call replaces 14. - calc: accepts JSON array of expressions or single string. One call replaces 3 separate calc invocations. - System prompt: 4-phase workflow (plan → skeleton → content → polish). Batch stock_photo and calc examples. Updated step budget. - describe-layout-issues: skip 'nested flex may collapse' for wrap containers (pill/tag frames are intentionally HUG). Extract childNeedsFill helper to reduce complexity. * Fix nested flex false positive for pills, single-root render rule - describe-layout-issues: skip 'nested flex may collapse' when a sibling already has grow or fill — pill/badge frames next to flex-grow content are intentionally small (e.g. BreakingLabel next to ticker text) - System prompt: render must have exactly one root element. Stock_photo 401 fallback: tell user, don't eval gradients. * Rework workflow: render complete sections, not empty frames Phase 2 now prescribes 5-7 renders, each producing a complete section (nav, hero, stories, opinions, sidebar, footer) with placeholder images, real text, proper auto-layout. No more empty frames to fill later. Removes the fill content phase — skeleton IS the content, just without real photos. Phase 3 is now just polish (batch stock_photo + fixes). Step budget: 12-15 steps instead of 15-30. * Pexels key input: match AI key behavior Same pattern as the AI provider key — intermediate ref, 'Key saved — enter new to replace' placeholder, Clear button, save on change. No longer exposes the raw key in the input field. * No empty frames in skeleton, select nodes after each tool - System prompt: explicit rules against empty frames, no pre-creating wrapper frames for future content. Each render = complete section with visible placeholders. Render 1 = only nav+ticker, not the whole page with empty containers. - tools.ts: set selection to flash node IDs after each mutating tool, so user sees which section AI is working on (selection border persists vs flash which is transient). * AI overlay: pulsing blue while working, green flash on done; skeleton workflow Rendering: - ai-overlays.ts: new module with pulsing blue border (sinusoidal opacity 0.3-0.8 at 1.5s period) for nodes AI is working on, green fade-out flash (800ms) when done - renderer.ts: aiMarkActive/aiMarkDone/aiClearAll methods, drawn after regular flashes - constants.ts: AI_ACTIVE_COLOR, AI_DONE_COLOR, timing constants Integration: - ai-adapter.ts: onBeforeExecute now receives args (for target ID) - tools.ts: marks target node (id or parent_id) active before tool executes, marks done via onFlashNodes after completion - editor.ts: aiMarkActive/aiMarkDone/aiClearAll forwarded to renderer Workflow: - System prompt: 3-phase skeleton workflow. Phase 2 renders gray placeholder blocks (Rectangle bg=#E2E8F0) for every section — page looks like wireframe before any real content. Phase 3 replaces skeletons with real content. Phase 4 batch stock_photo + polish. * render replace_id: atomic skeleton-to-content swap, fix AI overlay lifecycle render tool: - New replace_id parameter — renders new content, places it at the skeleton's position in parent, then deletes the skeleton. No visual gap between skeleton removal and content appearance. AI overlay fixes: - Track lastActiveIds across before/after execute — aiMarkDone clears the exact nodes that were marked active, not just the result IDs - aiFlashDone: direct green flash on result nodes (not gated by _aiActiveNodes membership) - replace_id included in target ID detection for active marking System prompt: - Phase 3 uses render({ replace_id }) instead of delete_node + render * Fix gradient fills, nested flex false positives, render insert_index, h=hug prompt - set_fill: gradient support with color_end + direction (top-bottom, left-right, etc). Uses proper Matrix type. No more eval for gradients. - describe-layout-issues: skip nested flex warning for all non-MIN alignments (SPACE_BETWEEN, CENTER, MAX) — not just SPACE_BETWEEN - render: insert_index parameter for controlling child order - system prompt: page frame h="hug" not fixed height - AI overlay: mark active on any tool with target ID (not just mutating), clear previous active in onBeforeExecute * Add desktop business news site example to system prompt Full workflow example: skeleton with gray placeholders → replace_id content fills → batch stock_photo. Shows 12-col grid pattern, card components, section headers, sidebar widgets, h=hug page frame. Complements the existing mobile app UI example. * Drop blue active pulse, keep only green done flash Blue active marking was invisible — tool execution is 20-80ms (one frame), AI thinking happens between tool calls where no hooks fire. Green done flash on result nodes works and is visible. * Blue pulse on skeleton during AI tool streaming Watch streaming message parts in ChatPanel — when a tool part appears with state 'call'/'partial-call' and has replace_id or parent_id in args, mark that node active (blue pulse). When tool transitions to 'output-available', mark done (green flash). This shows the pulsing blue border on the skeleton wireframe frame while AI generates JSX, and switches to green when the content replaces it. * AI overlay debug log section, watch tool parts by toolCallId - chat-debug.ts: aiOverlayLog array + AI OVERLAY LOG section in debug output showing every part/mark-active/mark-done event with timestamps - ChatPanel.vue: check parts by toolCallId instead of isToolUIPart, log all overlay events to aiOverlayLog for debugging * Fix AI overlay: use 'input' not 'args' for tool part params Vercel AI SDK UIToolInvocation uses 'input' field (not 'args') for tool call parameters. The watch was checking p.args which was always undefined, so no overlay events ever fired. Also use p.type instead of p.toolName for the log. * Move AI overlay to onBeforeExecute/onFlashNodes, reorder render params - onBeforeExecute: mark replace_id/parent_id active (blue pulse) - onFlashNodes: clear active, flash done on new nodes (green) - Reorder render tool params: replace_id first so streaming partial input reveals target ID before JSX body - Remove ChatPanel watch — all overlay logic in tools.ts now - Add aiClearActive() to renderer (clears active without touching done flashes) * Wait 2 rAF frames after marking active before tool execute onBeforeExecute marks the target node active (blue pulse), but execute ran immediately after without yielding to the render loop. The blue frame was never painted because markActive → execute → clearActive all happened in the same JS task. Now await two requestAnimationFrame callbacks so the canvas renders at least one blue-pulse frame before execution starts. * Add comprehensive AI overlay debug logging to chat debug log Both streaming watch (ChatPanel) and tool execution hooks (tools.ts) now log every event to aiOverlayLog with full state: renderer exists, activeNodes count, hasFlashes, node existence. This will show exactly what happens and why the blue pulse is or isn't visible. * Filter partial node IDs in streaming overlay watch During input-streaming, replace_id may be partially received (e.g. '0:' without the number part). Guard with isValidNodeId check. Also refactor watch callback into collectPendingTargets + syncOverlayState to stay under complexity limit. Analysis of last debug log: - Hero: only 235ms active (streaming didn't deliver replace_id early) - Stories: 14.8s active — should be clearly visible - Opinions: 12.5s, LatestNews: 11.5s, Stocks: 9.8s — all long enough - Hero was the only one too fast to notice * Fix undo race, JSX pragma conflict, partial ID overlay, add deletion log - Fix undo snapshot race condition: replace shared `let beforeSnapshot` with a per-tool stack. Vercel AI SDK runs tool calls from one step via Promise.all — concurrent mutating tools overwrote the shared snapshot, causing wrong undo entries and potential content loss. - Fix 'h is not a function' JSX error: rename sucrase pragma from `h` to `__h`. When AI writes `.map((h, i) => ...)`, the loop variable shadowed the pragma, crashing render. - Fix partial node ID `0:1` in overlay: strengthen `isValidNodeId` to require both parts ≥2 chars and verify node exists in graph. During streaming, `replace_id: "0:1811"` arrives as `0:1` → `0:18` → etc. - Add deletion log to SceneGraph: `_deletionLog` records id, name, type, parentId, timestamp, and truncated stack for last 200 deletions. Included in debug log as NODE DELETION LOG section to diagnose the intermittent frame disappearance bug. * Reduce find_nodes spam and describe bloat in system prompt - Instruct AI to reuse IDs from render results and describe output instead of calling find_nodes to rediscover them (saves 8+ calls) - Recommend depth=1 for full-page describe, depth=2 only for sections * Auto-depth describe + validate-after-render workflow Describe tool now auto-picks depth based on subtree size when depth is omitted: ≤15 nodes → depth 4, ≤40 → 3, ≤100 → 2, larger → 1. Max raised to 5 for explicit overrides. Prompt updated: after each render replace_id, call describe on that block (auto depth gives full detail on small sections). Every 2-3 content renders, describe root at depth=1 to catch cross-section layout issues early. * Enforce describe-after-every-render in system prompt Show explicit render → describe → fix pattern with code example. AI was skipping per-block validation and batching all describes at the end, causing compounding layout errors. * Add describe rules: nested text, missing padding, button padding, row centering Four new layout issue detectors in describe-issues.ts: - Nested Text (error): Text node inside Text causes overflow. Suggestion: split into separate Text elements in flex row. - Zero padding container (warning): Frame with fill + text children but no padding — content touches edges. - Button without padding (warning): Button-like frame with children but zero horizontal padding. - Row without vertical centering (warning): Horizontal flex with items=start, tall container, and short children — suggests items=center. All verified with bun -e tests including false positive checks. * Fix gradient alpha loss, add undersize child detection Gradient fix: set_fill with color_end was overwriting alpha to 1 on both gradient stops (`{ ...c, a: 1 }`). Now preserves original alpha from parsed hex — #00000000 → a:0, #000000AA → a:0.667. This caused transparent-to-dark overlays to render as solid black. New describe rule: detect child significantly narrower than parent in a container without auto-layout (e.g. 100px Rectangle inside 896px Frame). Suggests adding flex + w=fill. Caught the HeroImg bug that describe previously missed. * Auto-include issues in render result, no separate describe needed Render tool now scans the created subtree for errors/warnings and includes them in the response. AI gets issues automatically without needing to call describe — eliminates the pattern where AI skips describe on 'simple' sidebar blocks. Example response: { id: '0:42', name: 'Card', issues: [ { node: 'Text', id: '0:45', message: 'no color', severity: 'error' } ] } * Run layout before collecting render issues Issues were checked on raw node sizes (default 100px) before computeAllLayouts ran in onAfterExecute. Now render calls computeAllLayouts inline before collectSubtreeIssues, so issues reflect actual computed dimensions. This fixes hundreds of false 'X 100px overflows parent' warnings that made AI chase nonexistent problems. * Revert auto-issues in render and layout-before-issues Reverts 09a9b89 and 70c116a. Auto-including issues in render result caused false positives (100px default sizes before layout) and made AI chase nonexistent problems, degrading output quality. * Clean up AI chat PR: remove debug scaffolding, add stock photo provider adapter - Delete _deletionLog (stack traces on every node deletion) - Delete aiOverlayLog and all debug push() calls - Delete succesful_prompt_diffucult_ux.md (raw debug dump) and tokens.md (empty) - Remove double-rAF hack from ai-adapter (added 32ms to every tool call) - Restore simple beforeSnapshot for undo (was matching by tool name) - Remove 70-line overlay state tracker from ChatPanel.vue - Remove Math.* regex hack from calc (expr-eval has builtins) - Refactor stock-photo into provider adapter: Pexels + Unsplash - Add Unsplash key management to settings UI and use-chat composable --------- Co-authored-by: Danila Poyarkov --- packages/core/src/constants.ts | 5 + packages/core/src/figma-api-proxy.ts | 8 +- packages/core/src/index.ts | 8 + packages/core/src/layout.ts | 19 +- packages/core/src/render/render-jsx.ts | 4 +- packages/core/src/renderer/ai-overlays.ts | 73 ++++ packages/core/src/renderer/renderer.ts | 35 +- packages/core/src/renderer/scene.ts | 5 +- packages/core/src/renderer/text.ts | 12 +- packages/core/src/scene-graph.ts | 1 + packages/core/src/tools/ai-adapter.ts | 4 +- packages/core/src/tools/calc.ts | 38 +- packages/core/src/tools/create.ts | 33 +- packages/core/src/tools/describe-issues.ts | 93 +++++ .../core/src/tools/describe-layout-issues.ts | 71 +++- packages/core/src/tools/describe.ts | 96 +++-- packages/core/src/tools/modify.ts | 36 +- packages/core/src/tools/read.ts | 9 +- packages/core/src/tools/registry.ts | 39 +- packages/core/src/tools/schema.ts | 4 +- packages/core/src/tools/stock-photo.ts | 257 ++++++++++++ packages/core/src/tools/structure.ts | 80 ++++ src/ai/chat-debug.ts | 4 +- src/ai/system-prompt.md | 391 ++++++++++++++++-- src/ai/tools.ts | 34 +- src/components/chat/ProviderSettings.vue | 98 ++++- src/composables/use-chat.ts | 28 +- src/composables/use-menu.ts | 2 +- src/stores/editor.ts | 27 ++ tests/engine/layout.test.ts | 31 ++ tests/engine/render-text.test.ts | 42 +- 31 files changed, 1417 insertions(+), 170 deletions(-) create mode 100644 packages/core/src/renderer/ai-overlays.ts create mode 100644 packages/core/src/tools/stock-photo.ts diff --git a/packages/core/src/constants.ts b/packages/core/src/constants.ts index 2e4f3cb41..7d8ccdaf8 100644 --- a/packages/core/src/constants.ts +++ b/packages/core/src/constants.ts @@ -80,6 +80,11 @@ export const FLASH_PADDING = 5 export const FLASH_OVERSHOOT = 30 export const FLASH_RADIUS = 4 +export const AI_ACTIVE_COLOR = { r: 0.26, g: 0.52, b: 0.96 } +export const AI_DONE_COLOR = { r: 0.16, g: 0.73, b: 0.36 } +export const AI_PULSE_PERIOD_MS = 1500 +export const AI_DONE_DURATION_MS = 800 + export const TEXT_SELECTION_COLOR = { r: 0.26, g: 0.52, b: 0.96, a: 0.3 } export const TEXT_CARET_COLOR = BLACK export const TEXT_CARET_WIDTH = 1 diff --git a/packages/core/src/figma-api-proxy.ts b/packages/core/src/figma-api-proxy.ts index 690710fc5..625b33e40 100644 --- a/packages/core/src/figma-api-proxy.ts +++ b/packages/core/src/figma-api-proxy.ts @@ -949,7 +949,7 @@ export class FigmaNodeProxy { // --- Serialization --- - toJSON(): Record { + toJSON(maxDepth?: number, currentDepth = 0): Record { const n = this._raw() const obj: Record = { id: n.id, @@ -973,7 +973,11 @@ export class FigmaNodeProxy { } const children = this[INTERNAL_GRAPH].getChildren(this[INTERNAL_ID]) if (children.length > 0) { - obj.children = children.map((c) => this[INTERNAL_API].wrapNode(c.id).toJSON()) + if (maxDepth !== undefined && currentDepth >= maxDepth) { + obj.childCount = children.length + } else { + obj.children = children.map((c) => this[INTERNAL_API].wrapNode(c.id).toJSON(maxDepth, currentDepth + 1)) + } } return obj } diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index ea5d480ad..23976c338 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -246,3 +246,11 @@ export { } from './kiwi' export { CODEGEN_PROMPT } from './tools/prompts/codegen-prompt' +export { + setPexelsApiKey, + setUnsplashAccessKey, + registerStockPhotoProvider, + setActiveStockPhotoProvider, + getStockPhotoProviders +} from './tools/stock-photo' +export type { StockPhotoProvider, StockPhotoResult } from './tools/stock-photo' diff --git a/packages/core/src/layout.ts b/packages/core/src/layout.ts index c82aaec5b..4f1271579 100644 --- a/packages/core/src/layout.ts +++ b/packages/core/src/layout.ts @@ -30,14 +30,18 @@ const GLYPH_WIDTH_FACTOR = 0.6 // DO NOT REMOVE: without this, text nodes keep their 100×100 default size // and blow up every HUG container. The real MeasureFunc (CanvasKit) overrides // this when available — this is only the fallback. -function estimateTextSize(node: SceneNode): { width: number; height: number } { +function estimateTextSize(node: SceneNode, maxWidth?: number): { width: number; height: number } { const fontSize = node.fontSize || 14 const text = node.text || '' const charWidth = fontSize * GLYPH_WIDTH_FACTOR - return { - width: Math.ceil(text.length * charWidth), - height: Math.ceil(fontSize * 1.2), + const singleLineWidth = Math.ceil(text.length * charWidth) + const lineH = (node.lineHeight ?? 0) > 0 ? (node.lineHeight as number) : Math.ceil(fontSize * 1.4) + + if (maxWidth && maxWidth > 0 && singleLineWidth > maxWidth) { + const lines = Math.ceil(singleLineWidth / maxWidth) + return { width: maxWidth, height: Math.ceil(lines * lineH) } } + return { width: singleLineWidth, height: lineH } } export function setTextMeasurer(measurer: TextMeasurer | null): void { @@ -365,8 +369,8 @@ function configureChildAsLeaf(yogaChild: YogaNode, child: SceneNode, parent: Sce } else if (isText && !globalTextMeasurer && child.textAutoResize !== 'NONE') { // No CanvasKit — use rough estimate so HUG containers don't inherit // the 100×100 default SceneNode size. See estimateTextSize above. - const est = estimateTextSize(child) if (child.textAutoResize === 'WIDTH_AND_HEIGHT') { + const est = estimateTextSize(child) yogaChild.setWidth(est.width) yogaChild.setHeight(est.height) } else if (child.textAutoResize === 'HEIGHT') { @@ -375,6 +379,7 @@ function configureChildAsLeaf(yogaChild: YogaNode, child: SceneNode, parent: Sce if (!(!isRow && stretches)) { yogaChild.setWidth(child.width) } + const est = estimateTextSize(child, child.width) yogaChild.setHeight(est.height) } } else { @@ -410,7 +415,7 @@ function configureTextLeaf( if (cached) return cached const measured = globalTextMeasurer?.(child, maxW) - const result = measured ?? estimateTextSize(child) + const result = measured ?? estimateTextSize(child, maxW) cache.set(cacheKey, result) return result }) @@ -434,7 +439,7 @@ function configureTextLeaf( if (cached) return cached const measured = globalTextMeasurer?.(child, constraintW) - const result = { width: constraintW, height: measured?.height ?? estimateTextSize(child).height } + const result = { width: constraintW, height: measured?.height ?? estimateTextSize(child, constraintW).height } cache.set(cacheKey, result) return result }) diff --git a/packages/core/src/render/render-jsx.ts b/packages/core/src/render/render-jsx.ts index fc72bd9e1..a5bd1670f 100644 --- a/packages/core/src/render/render-jsx.ts +++ b/packages/core/src/render/render-jsx.ts @@ -12,7 +12,7 @@ import type { SceneGraph } from '../scene-graph' */ export function buildComponent(jsxString: string): () => unknown { const code = ` - const h = React.createElement + const __h = React.createElement const Frame = 'frame', Text = 'text', Rectangle = 'rectangle', Ellipse = 'ellipse' const Line = 'line', Star = 'star', Polygon = 'polygon', Vector = 'vector' const Group = 'group', Section = 'section', View = 'frame', Rect = 'rectangle' @@ -22,7 +22,7 @@ export function buildComponent(jsxString: string): () => unknown { const result = transform(code, { transforms: ['typescript', 'jsx'], - jsxPragma: 'h', + jsxPragma: '__h', production: true }) diff --git a/packages/core/src/renderer/ai-overlays.ts b/packages/core/src/renderer/ai-overlays.ts new file mode 100644 index 000000000..d592f89a5 --- /dev/null +++ b/packages/core/src/renderer/ai-overlays.ts @@ -0,0 +1,73 @@ +import { + FLASH_PADDING, + FLASH_STROKE_WIDTH, + FLASH_RADIUS, + AI_ACTIVE_COLOR, + AI_DONE_COLOR, + AI_PULSE_PERIOD_MS, + AI_DONE_DURATION_MS +} from '../constants' + +import type { SceneGraph } from '../scene-graph' +import type { Canvas } from 'canvaskit-wasm' +import type { SkiaRenderer } from './renderer' + +function drawAiNodeOverlay( + r: SkiaRenderer, + canvas: Canvas, + graph: SceneGraph, + nodeId: string, + color: { r: number; g: number; b: number }, + opacity: number +): void { + const node = graph.getNode(nodeId) + if (!node) return + const ck = r.ck + const abs = graph.getAbsolutePosition(nodeId) + const cx = (abs.x + node.width / 2) * r.zoom + r.panX + const cy = (abs.y + node.height / 2) * r.zoom + r.panY + const hw = (node.width / 2) * r.zoom + const hh = (node.height / 2) * r.zoom + const pad = FLASH_PADDING + + if (!r._flashPaint) { + r._flashPaint = new ck.Paint() + r._flashPaint.setStyle(ck.PaintStyle.Stroke) + r._flashPaint.setAntiAlias(true) + } + + const paint = r._flashPaint + paint.setColor(ck.Color4f(color.r, color.g, color.b, opacity)) + paint.setStrokeWidth(FLASH_STROKE_WIDTH) + + canvas.save() + if (node.rotation !== 0) canvas.rotate(node.rotation, cx, cy) + const rect = ck.RRectXY( + ck.LTRBRect(cx - hw - pad, cy - hh - pad, cx + hw + pad, cy + hh + pad), + FLASH_RADIUS, FLASH_RADIUS + ) + canvas.drawRRect(rect, paint) + canvas.restore() +} + +export function drawAiOverlays(r: SkiaRenderer, canvas: Canvas, graph: SceneGraph): void { + const now = performance.now() + + for (const nodeId of r._aiActiveNodes) { + const phase = (now % AI_PULSE_PERIOD_MS) / AI_PULSE_PERIOD_MS + const opacity = 0.3 + 0.5 * (0.5 + 0.5 * Math.sin(phase * Math.PI * 2)) + drawAiNodeOverlay(r, canvas, graph, nodeId, AI_ACTIVE_COLOR, opacity) + } + + for (let i = r._aiDoneFlashes.length - 1; i >= 0; i--) { + const flash = r._aiDoneFlashes[i] + const elapsed = now - flash.startTime + if (elapsed > AI_DONE_DURATION_MS) { + r._aiDoneFlashes.splice(i, 1) + continue + } + const t = elapsed / AI_DONE_DURATION_MS + const opacity = t < 0.3 ? t / 0.3 : 1 - (t - 0.3) / 0.7 + drawAiNodeOverlay(r, canvas, graph, flash.nodeId, AI_DONE_COLOR, opacity) + } +} diff --git a/packages/core/src/renderer/renderer.ts b/packages/core/src/renderer/renderer.ts index 32d4100cd..f7861f1f1 100644 --- a/packages/core/src/renderer/renderer.ts +++ b/packages/core/src/renderer/renderer.ts @@ -66,6 +66,7 @@ import { drawLayoutInsertIndicator as drawLayoutInsertIndicatorFn, drawTextEditOverlay as drawTextEditOverlayFn, } from './overlays' +import { drawAiOverlays as drawAiOverlaysFn } from './ai-overlays' import { drawPenOverlay as drawPenOverlayFn, drawRemoteCursors as drawRemoteCursorsFn @@ -221,6 +222,8 @@ export class SkiaRenderer { _culledCount = 0 _flashes: Array<{ nodeId: string; startTime: number }> = [] _flashPaint: Paint | null = null + _aiActiveNodes: Set = new Set() + _aiDoneFlashes: Array<{ nodeId: string; startTime: number }> = [] readonly DEFAULT_FONT_SIZE = DEFAULT_FONT_SIZE readonly COMPONENT_SET_BORDER_WIDTH = COMPONENT_SET_BORDER_WIDTH @@ -464,8 +467,37 @@ export class SkiaRenderer { this._flashes.push({ nodeId, startTime: performance.now() }) } + aiMarkActive(nodeIds: string[]): void { + for (const id of nodeIds) this._aiActiveNodes.add(id) + } + + aiMarkDone(nodeIds: string[]): void { + const now = performance.now() + for (const id of nodeIds) { + if (this._aiActiveNodes.delete(id)) { + this._aiDoneFlashes.push({ nodeId: id, startTime: now }) + } + } + } + + aiFlashDone(nodeIds: string[]): void { + const now = performance.now() + for (const id of nodeIds) { + this._aiDoneFlashes.push({ nodeId: id, startTime: now }) + } + } + + aiClearActive(): void { + this._aiActiveNodes.clear() + } + + aiClearAll(): void { + this._aiActiveNodes.clear() + this._aiDoneFlashes = [] + } + get hasActiveFlashes(): boolean { - return this._flashes.length > 0 + return this._flashes.length > 0 || this._aiActiveNodes.size > 0 || this._aiDoneFlashes.length > 0 } hitTestSectionTitle(graph: SceneGraph, canvasX: number, canvasY: number): SceneNode | null { @@ -926,6 +958,7 @@ export class SkiaRenderer { private drawFlashes(canvas: Canvas, graph: SceneGraph): void { drawFlashesFn(this, canvas, graph) + drawAiOverlaysFn(this, canvas, graph) } private drawLayoutInsertIndicator(canvas: Canvas, indicator?: RenderOverlays['layoutInsertIndicator']): void { diff --git a/packages/core/src/renderer/scene.ts b/packages/core/src/renderer/scene.ts index 4707b6273..cb8c269de 100644 --- a/packages/core/src/renderer/scene.ts +++ b/packages/core/src/renderer/scene.ts @@ -566,11 +566,14 @@ export function renderText(r: SkiaRenderer, canvas: Canvas, node: SceneNode): vo return } } - if (r.fontsLoaded && r.fontProvider && r.isNodeFontLoaded(node)) { + if (r.fontsLoaded && r.fontProvider) { const paragraph = r.buildParagraph(node, r.fillPaint.getColor(), { halfLeading: true }) canvas.drawParagraph(paragraph, 0, 0) paragraph.delete() } else if (r.textFont) { + canvas.save() + canvas.clipRect(r.ck.LTRBRect(0, 0, node.width, node.height), r.ck.ClipOp.Intersect, false) canvas.drawText(text, 0, node.fontSize || r.DEFAULT_FONT_SIZE, r.fillPaint, r.textFont) + canvas.restore() } } diff --git a/packages/core/src/renderer/text.ts b/packages/core/src/renderer/text.ts index 7cfba92ea..1f199d19b 100644 --- a/packages/core/src/renderer/text.ts +++ b/packages/core/src/renderer/text.ts @@ -24,7 +24,7 @@ export function measureTextNode( node: SceneNode, maxWidth?: number ): { width: number; height: number } | null { - if (!r.fontsLoaded || !r.fontProvider || !isNodeFontLoaded(r, node)) return null + if (!r.fontsLoaded || !r.fontProvider) return null if (node.type !== 'TEXT' || !node.text) return null const paragraph = buildParagraph(r, node) @@ -42,7 +42,7 @@ export function measureTextNode( } export function buildTextPicture(r: TextRenderer, node: SceneNode): Uint8Array | null { - if (!r.fontsLoaded || !r.fontProvider || !isNodeFontLoaded(r, node)) return null + if (!r.fontsLoaded || !r.fontProvider) return null if (node.type !== 'TEXT' || !node.text) return null const ck = r.ck @@ -170,8 +170,12 @@ export function buildParagraph( const truncateOpts = buildTruncateOpts(node, baseFontSize) - const fontFamilies = (primary: string) => - cjkFallback ? [primary, cjkFallback] : [primary] + const fontFamilies = (primary: string) => { + const families = [primary] + if (primary !== DEFAULT_FONT_FAMILY) families.push(DEFAULT_FONT_FAMILY) + if (cjkFallback) families.push(cjkFallback) + return families + } const paraStyle = new ck.ParagraphStyle({ textAlign: getTextAlign(ck, node.textAlignHorizontal), diff --git a/packages/core/src/scene-graph.ts b/packages/core/src/scene-graph.ts index 10996ec35..0aec517dc 100644 --- a/packages/core/src/scene-graph.ts +++ b/packages/core/src/scene-graph.ts @@ -750,6 +750,7 @@ export class SceneGraph { 'text', 'fontSize', 'fontFamily', 'fontWeight', 'italic', 'textAlignHorizontal', 'textAlignVertical', 'lineHeight', 'letterSpacing', 'textDecoration', 'textCase', 'styleRuns', 'fills', + 'width', 'height', ]) updateNode(id: string, changes: Partial): void { diff --git a/packages/core/src/tools/ai-adapter.ts b/packages/core/src/tools/ai-adapter.ts index a66a7d354..824e946d2 100644 --- a/packages/core/src/tools/ai-adapter.ts +++ b/packages/core/src/tools/ai-adapter.ts @@ -45,7 +45,7 @@ export interface StepBudget { export interface AIAdapterOptions { getFigma: () => FigmaAPI onBeforeExecute?: (def: ToolDef) => void - onAfterExecute?: (def: ToolDef) => void + onAfterExecute?: (def: ToolDef) => Promise | void onFlashNodes?: (nodeIds: string[]) => void onToolLog?: (entry: ToolLogEntry) => void getStepBudget?: () => StepBudget @@ -179,7 +179,7 @@ export function toolsToAI( emitToolLog(options, def, args, startTime, figma, nodeBefore, null, errorMsg) return { error: errorMsg } } finally { - options.onAfterExecute?.(def) + await options.onAfterExecute?.(def) } } } diff --git a/packages/core/src/tools/calc.ts b/packages/core/src/tools/calc.ts index 5960514c1..a43db5b8e 100644 --- a/packages/core/src/tools/calc.ts +++ b/packages/core/src/tools/calc.ts @@ -4,29 +4,45 @@ import { defineTool } from './schema' const parser = new ExprEval.Parser() +function evalExpr(expr: string): { expr: string; result: number } | { expr: string; error: string } { + try { + const result = parser.evaluate(expr) + if (!Number.isFinite(result)) { + return { expr, error: `Produced ${String(result)}` } + } + return { expr, result } + } catch (e) { + return { expr, error: e instanceof Error ? e.message : String(e) } + } +} + export const calc = defineTool({ name: 'calc', description: - 'Arithmetic calculator. ALWAYS use this instead of mental math for layout calculations. ' + - 'Evaluates a math expression and returns the numeric result. ' + + 'Arithmetic calculator. ALWAYS use instead of mental math. ' + + 'Pass one expression or a JSON array of expressions — all evaluated in one call. ' + 'Supports: + - * / % ** ( ) min max floor ceil round abs sqrt pow. ' + - 'Examples: "844 - 56 - 96 - 82" → 610, "floor(390 * 0.6)" → 234', + 'Examples: "844 - 56 - 96 - 82", \'["1440 * 8 / 12", "(952 - 16) / 2", "floor(390 * 0.6)"]\'', params: { expr: { type: 'string', - description: 'Math expression, e.g. "844 - 56 - 96 - 82" or "min(300, 400 - 2 * 20)"', + description: 'Single expression or JSON array of expressions', required: true } }, execute: (_figma, { expr }) => { + let exprs: string[] try { - const result = parser.evaluate(expr) - if (!Number.isFinite(result)) { - return { error: `Expression "${expr}" produced ${String(result)}` } - } - return { expr, result } - } catch (e) { - return { error: e instanceof Error ? e.message : String(e) } + const parsed = JSON.parse(expr) + exprs = Array.isArray(parsed) ? parsed : [expr] + } catch { + exprs = [expr] } + + if (exprs.length === 1) { + return evalExpr(exprs[0]) + } + + return { results: exprs.map(evalExpr) } } }) diff --git a/packages/core/src/tools/create.ts b/packages/core/src/tools/create.ts index ea0782a3a..a2ccd336e 100644 --- a/packages/core/src/tools/create.ts +++ b/packages/core/src/tools/create.ts @@ -53,20 +53,45 @@ export const render = defineTool({ name: 'render', mutates: true, description: - 'Render JSX to design nodes. Primary creation tool — creates entire component trees in one call. Example: Title', + 'Render JSX to design nodes. Use replace_id to swap a skeleton placeholder with real content (keeps position in parent). Example: Title', params: { - jsx: { type: 'string', description: 'JSX string to render', required: true }, + replace_id: { type: 'string', description: 'Node ID to replace — new node takes its position in parent, old node is deleted' }, + parent_id: { type: 'string', description: 'Parent node ID to render into' }, + insert_index: { type: 'number', description: 'Position among siblings (0 = first child). Omit to append at end.' }, x: { type: 'number', description: 'X position of the root node' }, y: { type: 'number', description: 'Y position of the root node' }, - parent_id: { type: 'string', description: 'Parent node ID to render into' } + jsx: { type: 'string', description: 'JSX string to render', required: true }, }, execute: async (figma, args) => { const { renderJSX } = await import('../render/render-jsx.js') + + let parentId = args.parent_id ?? figma.currentPageId + let replaceIndex = -1 + + if (args.replace_id) { + const target = figma.graph.getNode(args.replace_id) + if (target?.parentId) { + parentId = target.parentId + const parent = figma.graph.getNode(parentId) + if (parent) { + replaceIndex = parent.childIds.indexOf(args.replace_id) + } + } + } + const result = await renderJSX(figma.graph, args.jsx, { - parentId: args.parent_id ?? figma.currentPageId, + parentId, x: args.x, y: args.y }) + + if (args.replace_id && replaceIndex >= 0) { + figma.graph.reorderChild(result.id, parentId, replaceIndex) + figma.graph.deleteNode(args.replace_id) + } else if (args.insert_index !== undefined) { + figma.graph.reorderChild(result.id, parentId, args.insert_index) + } + return { id: result.id, name: result.name, type: result.type, children: result.childIds } } }) diff --git a/packages/core/src/tools/describe-issues.ts b/packages/core/src/tools/describe-issues.ts index 5bfd828f0..6a5e688f5 100644 --- a/packages/core/src/tools/describe-issues.ts +++ b/packages/core/src/tools/describe-issues.ts @@ -11,7 +11,10 @@ const LOW_CONTRAST_THRESHOLD = 15 const SHAPE_TYPES = new Set(['RECTANGLE', 'ELLIPSE', 'STAR', 'POLYGON', 'LINE']) const ICON_MAX_SIZE = 48 +export type IssueSeverity = 'error' | 'warning' | 'info' + export interface DescribeIssue { + severity?: IssueSeverity message: string suggestion?: string } @@ -149,6 +152,73 @@ function checkImagePlaceholder(node: SceneNode, graph: SceneGraph, issues: Descr } } +function checkNestedText(node: SceneNode, graph: SceneGraph, issues: DescribeIssue[]): void { + if (node.type !== 'TEXT') return + for (const childId of node.childIds) { + const child = graph.getNode(childId) + if (child?.type === 'TEXT') { + issues.push({ + message: `Nested Text "${child.text.slice(0, 20)}" inside "${node.text.slice(0, 20)}" — causes overflow`, + suggestion: 'Split into separate elements in a flex="row" wrapper' + }) + } + } +} + +function checkZeroPaddingContainer(node: SceneNode, graph: SceneGraph, issues: DescribeIssue[]): void { + if (!CONTAINER_TYPES.has(node.type)) return + if (node.layoutMode === 'NONE') return + if (node.childIds.length < 3) return + const allPad = node.paddingTop + node.paddingRight + node.paddingBottom + node.paddingLeft + if (allPad > 0) return + const hasTextChild = node.childIds.some((id) => { + const c = graph.getNode(id) + return c?.type === 'TEXT' && c.visible + }) + const hasFill = node.fills.some((f) => f.visible) + if (hasTextChild && hasFill) { + issues.push({ + message: `"${node.name}" has text content with fill but no padding`, + suggestion: 'Add p={16} or px={16} py={12}' + }) + } +} + +function checkButtonWithoutPadding(node: SceneNode, _graph: SceneGraph, issues: DescribeIssue[]): void { + if (!looksLikeButton(node)) return + if (node.layoutMode === 'NONE') return + const hPad = node.paddingLeft + node.paddingRight + if (hPad === 0 && node.childIds.length > 0) { + issues.push({ + message: `Button "${node.name}" has no horizontal padding`, + suggestion: 'Add px={16} or px={12}' + }) + } +} + +function checkFlexContainerWithoutAlignment(node: SceneNode, _graph: SceneGraph, issues: DescribeIssue[]): void { + if (!CONTAINER_TYPES.has(node.type)) return + if (node.layoutMode === 'NONE') return + if (node.childIds.length === 0) return + const isRow = node.layoutMode === 'HORIZONTAL' + if (isRow && node.counterAxisAlign === 'MIN' && node.height > 60) { + const pad = node.paddingTop + node.paddingBottom + const contentH = node.height - pad + if (contentH > 50 && node.childIds.length <= 5) { + const maxChildH = node.childIds.reduce((max, id) => { + const c = _graph.getNode(id) + return c?.visible ? Math.max(max, c.height) : max + }, 0) + if (maxChildH > 0 && maxChildH < contentH * 0.6) { + issues.push({ + message: `"${node.name}" row children top-aligned with large empty space below`, + suggestion: 'Add items="center" to vertically center content' + }) + } + } + } +} + function detectVisibilityIssues(node: SceneNode, graph: SceneGraph, issues: DescribeIssue[]): void { for (const fill of node.fills) { if (!fill.visible || fill.type !== 'SOLID') continue @@ -269,6 +339,22 @@ function detectSpacingIssues(node: SceneNode, graph: SceneGraph, gridSize: numbe } } +const ERROR_PATTERNS = [ + /overflow/i, /invisible/i, /no color/i, /collapses/i, /no fill and no stroke/i, + /dark on dark/i, /Touch target too small/i, /Nested Text/i +] +const INFO_PATTERNS = [ + /children named/i, /uppercase at/i, /fill.*matches parent/i, + /radius.*should be/i, /width while siblings/i, /height while siblings/i, + /icon.*width/i +] + +function classifySeverity(message: string): IssueSeverity { + if (ERROR_PATTERNS.some((p) => p.test(message))) return 'error' + if (INFO_PATTERNS.some((p) => p.test(message))) return 'info' + return 'warning' +} + export function detectIssues(node: SceneNode, gridSize: number, graph: SceneGraph): DescribeIssue[] { const issues: DescribeIssue[] = [] detectStructuralIssues(node, gridSize, graph, issues) @@ -277,5 +363,12 @@ export function detectIssues(node: SceneNode, gridSize: number, graph: SceneGrap detectRadiusIssues(node, graph, issues) detectTypographyIssues(node, graph, issues) detectSpacingIssues(node, graph, gridSize, issues) + checkNestedText(node, graph, issues) + checkZeroPaddingContainer(node, graph, issues) + checkButtonWithoutPadding(node, graph, issues) + checkFlexContainerWithoutAlignment(node, graph, issues) + for (const issue of issues) { + issue.severity = classifySeverity(issue.message) + } return issues } diff --git a/packages/core/src/tools/describe-layout-issues.ts b/packages/core/src/tools/describe-layout-issues.ts index 2d4174c4a..4326778ce 100644 --- a/packages/core/src/tools/describe-layout-issues.ts +++ b/packages/core/src/tools/describe-layout-issues.ts @@ -250,6 +250,21 @@ function checkSiblingHeightConsistency(ctx: LayoutContext): void { } } +function checkChildUndersize(ctx: LayoutContext): void { + const { node, graph, issues } = ctx + if (node.layoutMode !== 'NONE') return + for (const childId of node.childIds) { + const child = graph.getNode(childId) + if (!child?.visible) continue + if (child.width > 0 && node.width > 0 && child.width < node.width * 0.3 && child.height >= node.height * 0.5) { + issues.push({ + message: `"${child.name}" is ${Math.round(child.width)}px wide inside ${Math.round(node.width)}px "${node.name}" (no auto-layout)`, + suggestion: `Add flex="col" to "${node.name}" and w="fill" to "${child.name}"` + }) + } + } +} + function checkCrossAxisOverflow(ctx: LayoutContext): void { const { node, isRow, children, issues } = ctx if (node.clipsContent) return @@ -287,26 +302,49 @@ function checkAbsoluteInFlex(_ctx: LayoutContext): void { // Not reported — describe summary already shows "positioned: ABSOLUTE". } +function effectivelyFillsCrossAxis(child: SceneNode, parent: SceneNode, isRow: boolean): boolean { + const childCross = isRow ? child.height : child.width + const parentCrossContent = isRow + ? parent.height - parent.paddingTop - parent.paddingBottom + : parent.width - parent.paddingLeft - parent.paddingRight + return Math.abs(childCross - parentCrossContent) < 2 +} + +function childNeedsFill(child: SceneNode, parent: SceneNode, isRow: boolean): boolean { + if (child.layoutMode === 'NONE') return false + const crossDim = isRow ? child.width : child.height + const crossSizing = isRow ? child.counterAxisSizing : child.primaryAxisSizing + if (crossDim <= 0 && crossSizing !== 'FILL') return false + const mainSizing = isRow ? child.primaryAxisSizing : child.counterAxisSizing + if (mainSizing === 'FIXED') return false + if (effectivelyFillsCrossAxis(child, parent, isRow)) return false + if (child.childIds.length === 0) return false + return isRow + ? child.width < parent.width * 0.3 && child.counterAxisSizing !== 'FILL' && child.layoutGrow <= 0 + : child.height < parent.height * 0.3 && child.primaryAxisSizing !== 'FILL' && child.layoutGrow <= 0 +} + +function hasSiblingWithGrowOrFill(children: SceneNode[], exclude: SceneNode, isRow: boolean): boolean { + return children.some((c) => { + if (c === exclude) return false + if (c.layoutGrow > 0) return true + const sizing = isRow ? c.counterAxisSizing : c.primaryAxisSizing + return sizing === 'FILL' + }) +} + function checkNestedFlexWithoutFill(ctx: LayoutContext): void { const { node, isRow, children, issues } = ctx if (node.layoutMode === 'NONE') return - if (node.primaryAxisAlign === 'SPACE_BETWEEN' || node.primaryAxisAlign === 'CENTER') return + if (node.primaryAxisAlign !== 'MIN') return + if (node.layoutWrap === 'WRAP') return for (const child of children) { - if (child.layoutMode === 'NONE') continue - const crossDim = isRow ? child.width : child.height - const crossSizing = isRow ? child.counterAxisSizing : child.primaryAxisSizing - if (crossDim <= 0 && crossSizing !== 'FILL') continue - const mainSizing = isRow ? child.primaryAxisSizing : child.counterAxisSizing - if (mainSizing === 'FIXED') continue - const needsFill = isRow - ? child.width < node.width * 0.3 && child.counterAxisSizing !== 'FILL' && child.layoutGrow <= 0 - : child.height < node.height * 0.3 && child.primaryAxisSizing !== 'FILL' && child.layoutGrow <= 0 - if (needsFill && child.childIds.length > 0) { - issues.push({ - message: `Nested flex "${child.name}" may collapse — no fill or grow in "${node.name}"`, - suggestion: 'Add w="fill" or grow={1}' - }) - } + if (!childNeedsFill(child, node, isRow)) continue + if (hasSiblingWithGrowOrFill(children, child, isRow)) continue + issues.push({ + message: `Nested flex "${child.name}" may collapse — no fill or grow in "${node.name}"`, + suggestion: 'Add w="fill" or grow={1}' + }) } } @@ -341,6 +379,7 @@ export function detectLayoutIssues(node: SceneNode, graph: SceneGraph, issues: D checkTextVisibility(ctx) checkDuplicateNames(ctx) checkFillWithoutFlex(ctx) + checkChildUndersize(ctx) checkAbsoluteInFlex(ctx) if (node.layoutMode === 'NONE') return diff --git a/packages/core/src/tools/describe.ts b/packages/core/src/tools/describe.ts index 425f325f4..cc6fd45ab 100644 --- a/packages/core/src/tools/describe.ts +++ b/packages/core/src/tools/describe.ts @@ -183,45 +183,83 @@ function describeChild(node: SceneNode, graph: SceneGraph, depth: number, gridSi return result } +function describeOneNode( + figma: { graph: SceneGraph }, + nodeId: string, + depth: number, + gridSize: number +): Record { + const raw = figma.graph.getNode(nodeId) + if (!raw) return { id: nodeId, error: `Node "${nodeId}" not found` } + + const role = detectRole(raw) + const visual = describeVisual(raw) + const layout = describeLayout(raw) + const issues = detectIssues(raw, gridSize, figma.graph) + + const children: ChildDescription[] = [] + for (const childId of raw.childIds) { + const child = figma.graph.getNode(childId) + if (!child || !child.visible) continue + children.push(describeChild(child, figma.graph, depth - 1, gridSize)) + } + + const result: Record = { + id: raw.id, + name: raw.name, + type: raw.type, + role, + size: `${raw.width}×${raw.height}`, + visual, + } + if (layout) result.layout = layout + if (children.length > 0) result.children = children + if (issues.length > 0) result.issues = issues + return result +} + +function countDescendants(graph: SceneGraph, nodeId: string): number { + const node = graph.getNode(nodeId) + if (!node) return 0 + let count = 0 + for (const childId of node.childIds) { + count += 1 + countDescendants(graph, childId) + } + return count +} + +function autoDepth(graph: SceneGraph, nodeId: string): number { + const size = countDescendants(graph, nodeId) + if (size <= 15) return 4 + if (size <= 40) return 3 + if (size <= 100) return 2 + return 1 +} + export const describe = defineTool({ name: 'describe', description: - 'Semantic description of a node: role, visual style, layout, children summary, and design issues. Use depth=2 to see grandchildren (recommended for root frames).', + 'Semantic description of one or more nodes. Pass `id` for a single node, or `ids` for multiple nodes in one call. Omit depth for auto — adapts to subtree size (small block → deeper, large page → shallower).', params: { - id: { type: 'string', description: 'Node ID', required: true }, - depth: { type: 'number', description: 'How many levels of children to include (default: 1, max: 3)' }, + id: { type: 'string', description: 'Node ID (single node)' }, + ids: { type: 'string[]', description: 'Node IDs (multiple nodes in one call)' }, + depth: { type: 'number', description: 'Override depth (auto if omitted, max: 5)' }, grid: { type: 'number', description: 'Grid size for alignment checks (default: 8)' } }, execute: (figma, args) => { const gridSize = args.grid ?? 8 - const depth = Math.min(args.depth ?? 1, 3) - const raw = figma.graph.getNode(args.id) - if (!raw) return { error: `Node "${args.id}" not found` } - const role = detectRole(raw) - const visual = describeVisual(raw) - const layout = describeLayout(raw) - const issues = detectIssues(raw, gridSize, figma.graph) - - const children: ChildDescription[] = [] - for (const childId of raw.childIds) { - const child = figma.graph.getNode(childId) - if (!child || !child.visible) continue - children.push(describeChild(child, figma.graph, depth - 1, gridSize)) + if (args.ids && Array.isArray(args.ids)) { + return { + nodes: args.ids.map((nodeId) => { + const depth = Math.min(args.depth ?? autoDepth(figma.graph, nodeId), 5) + return describeOneNode(figma, nodeId, depth, gridSize) + }) + } } - const result: Record = { - id: raw.id, - name: raw.name, - type: raw.type, - role, - size: `${raw.width}×${raw.height}`, - visual, - } - if (layout) result.layout = layout - if (children.length > 0) result.children = children - if (issues.length > 0) result.issues = issues - - return result + if (!args.id) return { error: 'Provide id (string) or ids (string[])' } + const depth = Math.min(args.depth ?? autoDepth(figma.graph, args.id), 5) + return describeOneNode(figma, args.id, depth, gridSize) } }) diff --git a/packages/core/src/tools/modify.ts b/packages/core/src/tools/modify.ts index e03569fea..cdea45102 100644 --- a/packages/core/src/tools/modify.ts +++ b/packages/core/src/tools/modify.ts @@ -2,22 +2,52 @@ import { parseColor } from '../color' import { DEFAULT_SHADOW_COLOR } from '../constants' import type { CharacterStyleOverride, Effect, StyleRun } from '../scene-graph' +import type { Matrix } from '../types' import { defineTool } from './schema' export const setFill = defineTool({ name: 'set_fill', mutates: true, - description: 'Set the fill color of a node. Accepts hex (#ff0000) or named color.', + description: 'Set fill on a node. Solid: color="#ff0000". Linear gradient: gradient="top-bottom" or "left-right" with color (start) and color_end (end).', params: { id: { type: 'string', description: 'Node ID', required: true }, - color: { type: 'color', description: 'Color value (hex like #ff0000)', required: true } + color: { type: 'color', description: 'Color (hex). For gradient: start color.', required: true }, + color_end: { type: 'color', description: 'End color for gradient (if omitted, solid fill)' }, + gradient: { + type: 'string', + description: 'Gradient direction', + enum: ['top-bottom', 'bottom-top', 'left-right', 'right-left'] + } }, - execute: (figma, { id, color }) => { + execute: (figma, { id, color, color_end, gradient }) => { const node = figma.getNodeById(id) if (!node) return { error: `Node "${id}" not found` } const c = parseColor(color) + + if (gradient && color_end) { + const cEnd = parseColor(color_end) + const transforms: Record = { + 'top-bottom': { m00: 0, m01: 1, m02: 0, m10: -1, m11: 0, m12: 1 }, + 'bottom-top': { m00: 0, m01: -1, m02: 1, m10: 1, m11: 0, m12: 0 }, + 'left-right': { m00: 1, m01: 0, m02: 0, m10: 0, m11: 1, m12: 0 }, + 'right-left': { m00: -1, m01: 0, m02: 1, m10: 0, m11: -1, m12: 1 } + } + node.fills = [{ + type: 'GRADIENT_LINEAR', + color: c, + opacity: 1, + visible: true, + gradientStops: [ + { position: 0, color: c }, + { position: 1, color: cEnd } + ], + gradientTransform: transforms[gradient] ?? transforms['top-bottom'] + }] + return { id, gradient, start: c, end: cEnd } + } + node.fills = [{ type: 'SOLID', color: c, opacity: 1, visible: true }] return { id, color: c } } diff --git a/packages/core/src/tools/read.ts b/packages/core/src/tools/read.ts index d5ce3f46a..137eac9bf 100644 --- a/packages/core/src/tools/read.ts +++ b/packages/core/src/tools/read.ts @@ -49,14 +49,15 @@ export const getPageTree = defineTool({ export const getNode = defineTool({ name: 'get_node', - description: 'Get detailed properties of a node by ID.', + description: 'Get detailed properties of a node by ID. Use depth to limit child recursion (0 = node only, 1 = direct children, etc). Default: unlimited.', params: { - id: { type: 'string', description: 'Node ID', required: true } + id: { type: 'string', description: 'Node ID', required: true }, + depth: { type: 'number', description: 'Max depth of children to include (0 = no children). Default: unlimited' } }, - execute: (figma, { id }) => { + execute: (figma, { id, depth }) => { const node = figma.getNodeById(id) if (!node) return { error: `Node "${id}" not found` } - return nodeToResult(node) + return nodeToResult(node, depth) } }) diff --git a/packages/core/src/tools/registry.ts b/packages/core/src/tools/registry.ts index 3a49fc988..ce8cc3b55 100644 --- a/packages/core/src/tools/registry.ts +++ b/packages/core/src/tools/registry.ts @@ -18,7 +18,8 @@ import { import { deleteNode, cloneNode, renameNode, reparentNode, groupNodes, ungroupNode, flattenNodes, nodeToComponent, nodeBounds, nodeMove, nodeResize, - nodeAncestors, nodeChildren, nodeTree, nodeBindings, nodeReplaceWith, arrangeNodes + nodeAncestors, nodeChildren, nodeTree, nodeBindings, nodeReplaceWith, arrangeNodes, + batchUpdate } from './structure' import { listVariables, listCollections, getVariable, findVariables, @@ -38,6 +39,7 @@ import { import { describe } from './describe' import { designToTokens, designToComponentMap } from './codegen' import { calc } from './calc' +import { stockPhoto } from './stock-photo' /** * Core tools registered by default in AI chat (~30 tools, ~3K schema tokens). @@ -46,38 +48,27 @@ import { calc } from './calc' export const CORE_TOOLS: ToolDef[] = [ // Read getSelection, - getPageTree, getNode, findNodes, - getCurrentPage, - listPages, getJsx, - selectNodes, // Create render, - createShape, - searchIconsTool, - insertIcon, - fetchIconsTool, // Modify - setFill, - setStroke, - setEffects, updateNode, setLayout, setLayoutChild, setRadius, - setOpacity, + setFill, + setStroke, setText, - setFont, setTextProperties, - setVisible, // Structure deleteNode, - cloneNode, reparentNode, - nodeMove, nodeResize, + batchUpdate, + // Stock photos + stockPhoto, // Inspect & utility describe, calc, @@ -91,6 +82,10 @@ export const CORE_TOOLS: ToolDef[] = [ */ export const EXTENDED_TOOLS: ToolDef[] = [ // Read (advanced) + getPageTree, + getCurrentPage, + listPages, + selectNodes, queryNodes, getComponents, switchPage, @@ -98,12 +93,20 @@ export const EXTENDED_TOOLS: ToolDef[] = [ listFonts, diffJsx, // Create (advanced) + createShape, + searchIconsTool, + insertIcon, + fetchIconsTool, createComponent, createInstance, createPage, createVector, createSlice, // Modify (advanced) + setEffects, + setOpacity, + setFont, + setVisible, setConstraints, setRotation, setMinMax, @@ -114,6 +117,8 @@ export const EXTENDED_TOOLS: ToolDef[] = [ setStrokeAlign, setImageFill, // Structure (advanced) + cloneNode, + nodeMove, renameNode, groupNodes, ungroupNode, diff --git a/packages/core/src/tools/schema.ts b/packages/core/src/tools/schema.ts index 246c2565b..7280b6fe7 100644 --- a/packages/core/src/tools/schema.ts +++ b/packages/core/src/tools/schema.ts @@ -56,8 +56,8 @@ export function defineTool

>(def: { return def as unknown as ToolDef } -export function nodeToResult(node: FigmaNodeProxy): Record { - return node.toJSON() +export function nodeToResult(node: FigmaNodeProxy, maxDepth?: number): Record { + return node.toJSON(maxDepth) } export function nodeSummary(node: FigmaNodeProxy): { id: string; name: string; type: string } { diff --git a/packages/core/src/tools/stock-photo.ts b/packages/core/src/tools/stock-photo.ts new file mode 100644 index 000000000..3803682c9 --- /dev/null +++ b/packages/core/src/tools/stock-photo.ts @@ -0,0 +1,257 @@ +import { defineTool } from './schema' + +import type { FigmaAPI } from '../figma-api' + +export interface StockPhotoResult { + url: string + width: number + height: number + photographer: string + sourceId: string +} + +export interface StockPhotoProvider { + name: string + search(query: string, options: { + perPage: number + orientation: 'landscape' | 'portrait' | 'square' + targetDim: number + }): Promise +} + +const providers = new Map() +let activeProviderId: string | null = null + +export function registerStockPhotoProvider(provider: StockPhotoProvider): void { + providers.set(provider.name, provider) + if (!activeProviderId) activeProviderId = provider.name +} + +export function setActiveStockPhotoProvider(name: string | null): void { + activeProviderId = name +} + +export function getStockPhotoProviders(): string[] { + return [...providers.keys()] +} + +function getActiveProvider(): StockPhotoProvider | null { + if (!activeProviderId) return null + return providers.get(activeProviderId) ?? null +} + +// --- Pexels provider --- + +interface PexelsPhoto { + id: number + width: number + height: number + photographer: string + src: { + original: string + large2x: string + large: string + medium: string + small: string + landscape: string + } +} + +function pickPexelsSize(src: PexelsPhoto['src'], targetDim: number): string { + if (targetDim <= 200) return src.small + if (targetDim <= 400) return src.medium + if (targetDim <= 800) return src.large + if (targetDim <= 1600) return src.large2x + return src.original +} + +let pexelsApiKey: string | null = null + +export function setPexelsApiKey(key: string | null): void { + pexelsApiKey = key + if (key) { + registerStockPhotoProvider(pexelsProvider) + setActiveStockPhotoProvider('pexels') + } +} + +const pexelsProvider: StockPhotoProvider = { + name: 'pexels', + async search(query, { perPage, orientation, targetDim }) { + if (!pexelsApiKey) throw new Error('Pexels API key not configured') + const url = `https://api.pexels.com/v1/search?query=${encodeURIComponent(query)}&per_page=${perPage}&orientation=${orientation}` + const resp = await fetch(url, { headers: { Authorization: pexelsApiKey } }) + if (!resp.ok) throw new Error(`Pexels ${resp.status}`) + const data = (await resp.json()) as { photos: PexelsPhoto[] } + return data.photos.map((p) => ({ + url: pickPexelsSize(p.src, targetDim), + width: p.width, + height: p.height, + photographer: p.photographer, + sourceId: String(p.id) + })) + } +} + +// --- Unsplash provider --- + +let unsplashAccessKey: string | null = null + +export function setUnsplashAccessKey(key: string | null): void { + unsplashAccessKey = key + if (key) { + registerStockPhotoProvider(unsplashProvider) + } +} + +interface UnsplashPhoto { + id: string + width: number + height: number + urls: { raw: string; full: string; regular: string; small: string; thumb: string } + user: { name: string } + links: { download_location: string } +} + +function pickUnsplashSize(urls: UnsplashPhoto['urls'], targetDim: number): string { + if (targetDim <= 200) return urls.thumb + if (targetDim <= 400) return urls.small + if (targetDim <= 1080) return urls.regular + return urls.full +} + +const unsplashProvider: StockPhotoProvider = { + name: 'unsplash', + async search(query, { perPage, orientation }) { + if (!unsplashAccessKey) throw new Error('Unsplash access key not configured') + const orient = orientation === 'square' ? 'squarish' : orientation + const url = `https://api.unsplash.com/search/photos?query=${encodeURIComponent(query)}&per_page=${perPage}&orientation=${orient}` + const resp = await fetch(url, { + headers: { + Authorization: `Client-ID ${unsplashAccessKey}`, + 'Accept-Version': 'v1' + } + }) + if (!resp.ok) throw new Error(`Unsplash ${resp.status}`) + const data = (await resp.json()) as { results: UnsplashPhoto[] } + return data.results.map((p) => ({ + url: pickUnsplashSize(p.urls, 1080), + width: p.width, + height: p.height, + photographer: p.user.name, + sourceId: p.id + })) + } +} + +// --- Tool --- + +interface PhotoRequest { + id: string + query: string + index?: number + orientation?: 'landscape' | 'portrait' | 'square' +} + +interface PhotoResult { + id: string + photo?: { sourceId: string; photographer: string; width: number; height: number; provider: string } + error?: string +} + +async function applyPhoto( + figma: FigmaAPI, + provider: StockPhotoProvider, + req: PhotoRequest +): Promise { + const node = figma.getNodeById(req.id) + if (!node) return { id: req.id, error: 'Not found' } + + const children = 'children' in node ? (node as { children: unknown[] }).children : [] + if (children.length > 0) { + return { id: req.id, error: `"${node.name}" has children — use a leaf shape` } + } + + const perPage = Math.min((req.index ?? 0) + 3, 15) + const orientation = req.orientation ?? 'landscape' + const targetDim = Math.max(node.width, node.height) + + let results: StockPhotoResult[] + try { + results = await provider.search(req.query, { perPage, orientation, targetDim }) + } catch (err) { + return { id: req.id, error: err instanceof Error ? err.message : String(err) } + } + + if (results.length === 0) return { id: req.id, error: `No photos for "${req.query}"` } + const photo = results[Math.min(req.index ?? 0, results.length - 1)] + + let imageBytes: Uint8Array + try { + const imgResp = await fetch(photo.url) + if (!imgResp.ok) return { id: req.id, error: `Download ${imgResp.status}` } + imageBytes = new Uint8Array(await imgResp.arrayBuffer()) + } catch (err) { + return { id: req.id, error: `Download: ${err instanceof Error ? err.message : String(err)}` } + } + + const image = figma.createImage(imageBytes) + node.fills = [{ + type: 'IMAGE', + color: { r: 1, g: 1, b: 1, a: 1 }, + imageHash: image.hash, + imageScaleMode: 'FILL', + visible: true, + opacity: 1 + }] + + return { + id: node.id, + photo: { + sourceId: photo.sourceId, + photographer: photo.photographer, + width: photo.width, + height: photo.height, + provider: provider.name + } + } +} + +export const stockPhoto = defineTool({ + name: 'stock_photo', + mutates: true, + description: + 'Search stock photos and apply to nodes. Pass a JSON array — all fetched in parallel. ' + + 'Each item: {id, query, index?, orientation?}. Only works on leaf shapes (Rectangle/Ellipse).', + params: { + requests: { + type: 'string', + description: + 'JSON array: [{"id":"0:5","query":"mountain sunset"},{"id":"0:8","query":"business team","orientation":"square"}]', + required: true + } + }, + execute: async (figma, { requests }) => { + const provider = getActiveProvider() + if (!provider) { + return { + error: `No stock photo provider configured. Ask the user to add an API key in AI chat settings. Available providers: Pexels, Unsplash.` + } + } + + let reqs: PhotoRequest[] + try { + const parsed = JSON.parse(String(requests)) + reqs = Array.isArray(parsed) ? parsed : [parsed] + } catch { + return { error: 'Invalid JSON in requests' } + } + + if (reqs.length === 0) return { error: 'Empty requests array' } + + const results = await Promise.all(reqs.map((r) => applyPhoto(figma, provider, r))) + const ok = results.filter((r) => r.photo).length + + return { applied: ok, failed: results.length - ok, provider: provider.name, results } + } +}) diff --git a/packages/core/src/tools/structure.ts b/packages/core/src/tools/structure.ts index 70de41e90..68f439b7f 100644 --- a/packages/core/src/tools/structure.ts +++ b/packages/core/src/tools/structure.ts @@ -346,3 +346,83 @@ export const arrangeNodes = defineTool({ return { arranged: nodes.length, mode } } }) + + +interface BatchOp { + id: string + props: Record +} + +function str(v: unknown): string { return typeof v === 'string' ? v : '' } +function num(v: unknown): number { return typeof v === 'number' ? v : 0 } + +function applyBatchProps(node: FigmaNodeProxy, p: Record): string[] { + const updated: string[] = [] + + if (p.spacing !== undefined) { node.itemSpacing = num(p.spacing); updated.push('spacing') } + if (p.padding !== undefined) { + const v = num(p.padding) + node.paddingTop = v; node.paddingRight = v; node.paddingBottom = v; node.paddingLeft = v + updated.push('padding') + } + if (p.padding_horizontal !== undefined) { + node.paddingLeft = num(p.padding_horizontal); node.paddingRight = num(p.padding_horizontal) + updated.push('padding_horizontal') + } + if (p.padding_vertical !== undefined) { + node.paddingTop = num(p.padding_vertical); node.paddingBottom = num(p.padding_vertical) + updated.push('padding_vertical') + } + if (p.counter_align !== undefined) { node.counterAxisAlignItems = str(p.counter_align); updated.push('counter_align') } + if (p.align !== undefined) { node.primaryAxisAlignItems = str(p.align); updated.push('align') } + if (p.sizing_horizontal !== undefined) { node.layoutSizingHorizontal = str(p.sizing_horizontal); updated.push('sizing_horizontal') } + if (p.sizing_vertical !== undefined) { node.layoutSizingVertical = str(p.sizing_vertical); updated.push('sizing_vertical') } + if (p.grow !== undefined) { node.layoutGrow = num(p.grow); updated.push('grow') } + if (p.name !== undefined) { node.name = str(p.name); updated.push('name') } + if (p.visible !== undefined) { node.visible = Boolean(p.visible); updated.push('visible') } + if (p.corner_radius !== undefined) { node.cornerRadius = num(p.corner_radius); updated.push('corner_radius') } + if (p.opacity !== undefined) { node.opacity = num(p.opacity); updated.push('opacity') } + if (p.auto_resize !== undefined) { node.textAutoResize = str(p.auto_resize); updated.push('auto_resize') } + if (p.direction !== undefined) { node.layoutMode = str(p.direction) as 'HORIZONTAL' | 'VERTICAL'; updated.push('direction') } + + return updated +} + +export const batchUpdate = defineTool({ + name: 'batch_update', + mutates: true, + description: + 'Execute multiple modifications in one call. Each operation is {id, props} where props can include: spacing, padding, padding_horizontal, padding_vertical, counter_align, sizing_horizontal, sizing_vertical, grow, name, visible, corner_radius, auto_resize (for text), direction. Runs all updates with one layout recompute.', + params: { + operations: { + type: 'string', + description: + 'JSON array: [{"id":"0:5","props":{"spacing":8}},{"id":"0:6","props":{"sizing_horizontal":"FILL","grow":1}}]', + required: true + } + }, + execute: (figma, { operations }) => { + let ops: BatchOp[] + try { + ops = JSON.parse(String(operations)) + } catch { + return { error: 'Invalid JSON in operations' } + } + if (!Array.isArray(ops)) return { error: 'operations must be a JSON array' } + + const results: Array<{ id: string; updated: string[] }> = [] + const errors: string[] = [] + + for (const op of ops) { + const node = figma.getNodeById(op.id) + if (!node) { errors.push(`Node "${op.id}" not found`); continue } + const updated = applyBatchProps(node, op.props) + if (updated.length > 0) results.push({ id: op.id, updated }) + } + + const out: Record = { updated: results.length } + if (results.length > 0) out.results = results + if (errors.length > 0) out.errors = errors + return out + } +}) diff --git a/src/ai/chat-debug.ts b/src/ai/chat-debug.ts index d8b7bfb95..1de759f29 100644 --- a/src/ai/chat-debug.ts +++ b/src/ai/chat-debug.ts @@ -1,7 +1,7 @@ import { getStepUsages, getToolLogEntries } from '@/ai/tools' -import { buildDebugLog } from '@open-pencil/core' +import { buildDebugLog } from '@open-pencil/core/tools' -import type { ToolDebugLog, ToolLogEntry } from '@open-pencil/core' +import type { ToolDebugLog, ToolLogEntry } from '@open-pencil/core/tools' import type { UIMessage } from 'ai' function formatToolPart(part: Record): string { diff --git a/src/ai/system-prompt.md b/src/ai/system-prompt.md index f8cd1735d..e17080445 100644 --- a/src/ai/system-prompt.md +++ b/src/ai/system-prompt.md @@ -4,7 +4,7 @@ After completing a design, give a **2–3 line** summary: frame size, accent col # Rendering -The `render` tool takes JSX and produces design nodes. JavaScript expressions (map, ternaries, Array.from) work inside JSX. +The `render` tool takes JSX and produces design nodes. JavaScript expressions (map, ternaries, Array.from) work inside JSX. **Each render call must have exactly ONE root element.** To add multiple siblings to the same parent, use separate render calls or wrap in a Fragment-like parent Frame. Available elements: Frame, Text, Rectangle, Ellipse, Line, Star, Polygon, Group, Section, Component, Icon. @@ -44,7 +44,7 @@ Nested flex containers need w="fill" at EVERY level to stretch. `grow={1}` insid No margin property. For single-child offset, wrap in Frame with padding. -**Text in fixed containers:** Set `w` or `w="fill"` on multiword Text. For fixed-height rows, add `maxLines={1}`. In wrap layouts, calculate: columns = floor((available + gap) / (child_w + gap)). +**Text wrapping (CRITICAL):** Multiline text MUST have `w="fill"` (not `w={N}`). Use `w="fill"` on Text inside `flex="col"` cards — this stretches text to card width and enables auto-wrapping. Never use fixed `w={N}` on text that should wrap — the width may not match the parent due to font metric differences. For fixed-height rows, add `maxLines={1}`. In wrap layouts, calculate: columns = floor((available + gap) / (child_w + gap)). ## Corner radius @@ -64,7 +64,7 @@ Split into **2–3 render calls**: 2. Fill section A (poster, header) 3. Fill section B (content, details) -🧮 **Use `calc` for ALL layout arithmetic** — never mental math. `calc("844 - 72 - 116 - 87")` → verify before rendering. LLMs make arithmetic errors. +🧮 **Use `calc` for ALL layout arithmetic** — never mental math. Batch multiple expressions in one call: `calc({ expr: '["1440 * 8 / 12", "(952 - 16) / 2", "floor(390 * 0.6)"]' })`. Single expression also works: `calc({ expr: "844 - 72 - 116 - 87" })`. ## Typography @@ -72,6 +72,8 @@ Split into **2–3 render calls**: Hierarchy via one property at a time: size OR weight OR color. Light bg: primary #111827, secondary #6B7280, tertiary #9CA3AF. Dark bg: #FFFFFF, #FFFFFF99, #FFFFFF66. +Fonts are loaded automatically — use any Google Fonts family (Inter, Georgia, Roboto, Playfair Display, etc.). The first render with a new font may take a moment to load. + ## Prohibited No style={{}}, className, CSS. No named colors or rgb(). No percentage values. No TypeScript casts. No Math.random(). No `Math.` prefix in calc — use `floor(x)` not `Math.floor(x)`. No emoji in UI elements (use `` instead) — emoji renders as □. @@ -84,42 +86,134 @@ No style={{}}, className, CSS. No named colors or rgb(). No percentage values. N **Don't mix `w={N}` and `grow={N}`** — grow overrides width. +**Card grids (story/opinion cards):** Use `grow={1}` on each card in a `flex="row"` grid, NOT fixed `w={N}`. Inside each card, use `w="fill"` for images and `w="fill"` for title text. This ensures text wraps within the card regardless of font metrics. Example: `Title`. + **Tab bar / Bottom nav:** Outer frame `flex="row" w="fill" justify="between" px={32}`. Each tab `flex="col" items="center" gap={4}`. Tab items are HUG-width — `justify="between"` distributes them. Don't use `grow` on individual tabs. -**Dividers:** Vertical `w={1} h="fill"` inside flex="row". Horizontal `h={1} w="fill"` inside flex="col". +**Dividers:** Use `` for horizontal dividers inside `flex="col"`. Use `` for vertical dividers inside `flex="row"`. ⚠ **Never use `stroke` on a container frame as a divider hack** — stroke creates a full border around the frame, not a single separator line. Set the parent `gap={0}` and interleave Rectangle dividers between items. + +# Stock Photos + +`stock_photo` places real Pexels images on leaf shapes (Rectangle/Ellipse). Pass a JSON array — **all photos fetched in parallel**: + +``` +stock_photo({ requests: '[{"id":"0:30","query":"wall street trading floor"},{"id":"0:58","query":"AI chip semiconductor"},{"id":"0:65","query":"bank finance credit card"}]' }) +``` + +- Batch all photos in one call — don't call stock_photo 14 times separately +- Only apply to leaf shapes (Rectangle/Ellipse), NOT to Frames with children +- Use descriptive English queries: "aerial city skyline sunset", not "image1" +- Orientation: "landscape" (default), "portrait" for tall cards, "square" for avatars +- If Pexels key is not configured or returns 401, tell the user to add/check it in AI chat settings. Do NOT fall back to `eval` with manual gradients — leave placeholder colors as-is # Workflow (MANDATORY) -1. `calc` — compute section heights -2. `render` — skeleton -3. `describe` root with `depth=2` — verify -4. `render` into section A -5. `describe` root with `depth=2` — verify -6. `render` into section B -7. `describe` root with `depth=2` — check issues -8. Fix ALL issues from `describe` with `set_*` / `update_node` -9. `describe` root with `depth=2` — **MANDATORY final verify after fixes** +## Phase 1 — Plan (text only, no tools) -Typically **3 renders + 3–4 describes**. `describe` the root with `depth=2` — shows sections AND their children in one call. +Write a brief plan as numbered sections: what blocks, rough dimensions, layout approach. Example: -⚠ **Issues from `describe` are bugs.** Fix every issue before finishing. Common ones: +> 1. NavBar 1440×56 dark, row +> 2. Hero 1440×500 with image placeholder + overlay text +> 3. Stories grid: 2×2 cards in wrap row, grow cards +> 4. Sidebar: news feed + stocks widget + newsletter +> 5. Footer 3-col links + +## Phase 2 — Skeleton (visible placeholders for every section) + +Build the ENTIRE page with visible skeleton placeholders. Every section shows gray blocks where content will go — the page looks like a wireframe with correct proportions and spacing. + +1. `calc` — batch all dimension arithmetic +2. **Render 1** — page frame (`h="hug"`, NOT fixed height) + nav bar + ticker (real text content) +3. **Render 2** — hero skeleton: gray image block `` + text placeholder lines `` +4. **Render 3** — stories skeleton: real section header + main story card (gray image + gray text lines) + 3 sub-cards (same pattern) +5. **Render 4** — opinions skeleton (same pattern as stories) +6. **Render 5** — sidebar skeleton: news list (gray text lines), stocks (gray rows), newsletter (dark block with gray input) +7. **Render 6** — footer (final content — simple enough) +8. `describe` root `depth=2` — verify layout, proportions, spacing +9. `batch_update` — fix ALL issues before filling real content + +**Skeleton card pattern:** + +```jsx + + + + + + + + +``` + +After Phase 2 the page looks like a complete wireframe — all sections visible, correct sizes, verified layout. + +## Phase 3 — Fill content (replace skeletons with real content) + +For each skeleton section, use `render` with `replace_id` — the new content takes the skeleton's position and the skeleton is deleted atomically. No separate `delete_node` needed: + +``` +render({ jsx: " - 9:41 - Canvas + + 9:41 + + + Canvas + - + - Untitled Design + + Untitled Design + @@ -159,7 +269,16 @@ This is a **mobile interface app** (390×844) — dark theme, floating panels, t - + @@ -168,11 +287,16 @@ This is a **mobile interface app** (390×844) — dark theme, floating panels, t **Step 3** — describe root depth=2, fix issues (rename duplicate Text nodes, fix spacing). **Step 4** — Fill artboard content into parent "ArtboardOnCanvas": + ```jsx - MOBILE APP - Sample Design Preview + + MOBILE APP + + + Sample Design Preview + @@ -189,6 +313,7 @@ This is a **mobile interface app** (390×844) — dark theme, floating panels, t ``` **Step 5** — Fill bottom dock into parent "BottomDock": + ```jsx @@ -216,10 +341,26 @@ This is a **mobile interface app** (390×844) — dark theme, floating panels, t ``` **Step 6** — Add floating overlays into "CanvasArea" (selection handles, zoom, properties): + ```jsx - + - 75% + + 75% + ``` @@ -227,9 +368,203 @@ This is a **mobile interface app** (390×844) — dark theme, floating panels, t **Step 7** — describe depth=2, fix remaining issues, add shadows, final describe. Key patterns in this example: + - **Every multi-child Frame has `flex`** — no exceptions - **Named all nodes** — Tool_Select, Tool_Move, BrushSizeSlider, etc. - **Floating panels use x/y** — inside non-flex CanvasArea parent - **Procreate aesthetic**: `#2C2C2ECC` semi-transparent panels, `rounded={22}` pill shapes, `shadow` for depth - **Icons with explicit color** — `color="#FFFFFFCC"` or `color="#FFFFFF88"` for hierarchy - **3 renders** (skeleton → content A → content B) + **3 describes** + fix pass + +# Example: desktop business news site + +User prompt: "business media desktop site with real images, 12-col grid, 8 cols main, 4 cols sidebar, breaking news, hero, stories, opinions, sidebar news + stocks + newsletter, footer" + +This is a **desktop media site** (1440px wide, scrollable) — light theme, 12-col grid, card-based layout. + +**Step 1** — calc all grid dimensions in one batch: + +``` +calc({ expr: '["1440 - 48 - 48 - 24", "floor((1320) * 8 / 12)", "floor((1320) * 4 / 12)"]' }) +``` + +→ Content area 1320px, Main 880px, Sidebar 440px. + +**Step 2** — Skeleton render (entire page with gray placeholders): + +```jsx + + {/* NavBar — real content */} + + + + THE MARKETS + + + + Markets + + + Economy + + + Technology + + + + + + + + Subscribe + + + + + + {/* Breaking News — real content */} + + + + BREAKING + + + + Fed signals rate cut — S&P 500 hits record + + + + {/* Content area with skeleton placeholders */} + + + {/* Hero skeleton */} + + + + + + + + + + {/* Stories skeleton */} + + + + + + + + + + + + {Array.from({ length: 3 }, (_, i) => ( + + + + + + + + ))} + + + + {/* Opinions skeleton — same pattern */} + + {/* ... same structure as StoriesSection ... */} + + + {/* Sidebar skeletons */} + + + + + + {Array.from({ length: 6 }, (_, i) => ( + + + + + + + + ))} + + + + + + + + + {/* Footer — real content */} + + {/* ... footer columns ... */} + + +``` + +**Step 3** — `describe` root depth=2, fix layout with `batch_update`. + +**Steps 4–9** — Replace each skeleton with real content using `replace_id`: + +``` +render({ jsx: "` +- **One batch stock_photo** — 17 images in parallel, not 17 sequential calls +- **Footer real content from skeleton** — simple enough to render once +- **Total: 1 calc + 1 skeleton + 6 replace renders + 1 stock_photo + 2 describes + fixes = ~15 steps** diff --git a/src/ai/tools.ts b/src/ai/tools.ts index 1019de6a0..fa7011b86 100644 --- a/src/ai/tools.ts +++ b/src/ai/tools.ts @@ -3,7 +3,14 @@ import { tool } from 'ai' import * as v from 'valibot' import { makeFigmaFromStore } from '@/automation/figma-factory' -import { CORE_TOOLS, computeAllLayouts, toolsToAI } from '@open-pencil/core' +import { + CORE_TOOLS, + collectFontKeys, + computeAllLayouts, + isFontLoaded, + loadFont, + toolsToAI +} from '@open-pencil/core' import type { EditorStore } from '@/stores/editor' import type { SceneNode, StepBudget, ToolLogEntry } from '@open-pencil/core' @@ -80,9 +87,25 @@ export function createAITools(store: EditorStore) { beforeSnapshot = store.snapshotPage() } }, - onAfterExecute: (def) => { + onAfterExecute: async (def) => { if (def.mutates) { - computeAllLayouts(store.graph, store.state.currentPageId) + const pageId = store.state.currentPageId + const pageNode = store.graph.getNode(pageId) + if (pageNode) { + const fontKeys = collectFontKeys(store.graph, pageNode.childIds) + const missing = fontKeys.filter(([family]) => !isFontLoaded(family)) + if (missing.length > 0) { + const results = await Promise.all( + missing.map(([family, style]) => loadFont(family, style)) + ) + if (results.some((r) => r !== null)) { + for (const [, node] of store.graph.nodes) { + if (node.type === 'TEXT' && node.textPicture) node.textPicture = null + } + } + } + } + computeAllLayouts(store.graph, pageId) store.requestRender() if (beforeSnapshot) { const before = beforeSnapshot @@ -97,7 +120,10 @@ export function createAITools(store: EditorStore) { } }, onFlashNodes: (nodeIds) => { - store.flashNodes(nodeIds) + store.renderer?.aiClearActive() + if (nodeIds.length > 0) { + store.aiFlashDone(nodeIds) + } }, onToolLog: (entry) => { runState.toolLog.push(entry) diff --git a/src/components/chat/ProviderSettings.vue b/src/components/chat/ProviderSettings.vue index aa7012cba..7cdb04810 100644 --- a/src/components/chat/ProviderSettings.vue +++ b/src/components/chat/ProviderSettings.vue @@ -24,13 +24,19 @@ const { customBaseURL, customModelID, customAPIType, - maxOutputTokens + maxOutputTokens, + pexelsApiKey, + unsplashAccessKey } = useAIChat() const keyInput = ref('') +const pexelsKeyInput = ref('') +const unsplashKeyInput = ref('') const baseURLInput = ref(customBaseURL.value) const customModelInput = ref(customModelID.value) const hasExistingKey = ref(!!apiKey.value) +const hasExistingPexelsKey = ref(!!pexelsApiKey.value) +const hasExistingUnsplashKey = ref(!!unsplashAccessKey.value) watch(providerID, () => { keyInput.value = '' @@ -45,6 +51,16 @@ function save() { hasExistingKey.value = true keyInput.value = '' } + if (pexelsKeyInput.value.trim()) { + pexelsApiKey.value = pexelsKeyInput.value.trim() + hasExistingPexelsKey.value = true + pexelsKeyInput.value = '' + } + if (unsplashKeyInput.value.trim()) { + unsplashAccessKey.value = unsplashKeyInput.value.trim() + hasExistingUnsplashKey.value = true + unsplashKeyInput.value = '' + } if (providerDef.value.supportsCustomBaseURL) { customBaseURL.value = baseURLInput.value.trim() } @@ -58,6 +74,18 @@ function clearKey() { keyInput.value = '' hasExistingKey.value = false } + +function clearPexelsKey() { + pexelsApiKey.value = '' + pexelsKeyInput.value = '' + hasExistingPexelsKey.value = false +} + +function clearUnsplashKey() { + unsplashAccessKey.value = '' + unsplashKeyInput.value = '' + hasExistingUnsplashKey.value = false +}