From ac5c3c236d00646bf7444305f35c593dbffe99ca Mon Sep 17 00:00:00 2001 From: imrik Date: Thu, 29 Oct 2026 11:58:45 +0200 Subject: [PATCH] feat(sessions): send pasted and attached images, drop tool screenshots Base Code now imports the images a user pasted (base44-dev/apper#35805). Before uploading, sessions import drops images the agent's tools produced (they are most of a big session's size and the server never imports them) and adds an image the user attached as a file path, as apps that wrap Claude Code or Codex do, once the agent opened it. The size limit goes from 25 to 50 MB to match the server. Co-Authored-By: Claude Opus 5.5 --- .../cli/src/cli/commands/sessions/import.ts | 7 +- packages/cli/src/core/session-import/api.ts | 4 +- .../cli/src/core/session-import/images.ts | 193 ++++++++++++++++++ .../cli/tests/cli/sessions_import.spec.ts | 119 ++++++++++- 4 files changed, 316 insertions(+), 7 deletions(-) create mode 100644 packages/cli/src/core/session-import/images.ts diff --git a/packages/cli/src/cli/commands/sessions/import.ts b/packages/cli/src/cli/commands/sessions/import.ts index 65e1b8b7..cf301380 100644 --- a/packages/cli/src/cli/commands/sessions/import.ts +++ b/packages/cli/src/cli/commands/sessions/import.ts @@ -12,6 +12,7 @@ import { import { getBase44ApiUrl } from "@/core/config.js"; import { InvalidInputError } from "@/core/errors.js"; import { MAX_SESSION_BYTES, uploadSession } from "@/core/session-import/api.js"; +import { prepareSessionImages } from "@/core/session-import/images.js"; import { findCurrentSession, type LocalSession, @@ -113,10 +114,12 @@ async function importAction( } const appId = requireApp(ctx).id; - const content = await readSession(ctx, link, options); + const content = await prepareSessionImages( + await readSession(ctx, link, options), + ); if (content.length > MAX_SESSION_BYTES) { throw new InvalidInputError( - "This session is over 25 MB and can't be imported.", + "This session is over 50 MB and can't be imported.", { hints: [] }, ); } diff --git a/packages/cli/src/core/session-import/api.ts b/packages/cli/src/core/session-import/api.ts index f10bf712..a6ff9133 100644 --- a/packages/cli/src/core/session-import/api.ts +++ b/packages/cli/src/core/session-import/api.ts @@ -5,7 +5,7 @@ import { getAppClient } from "@/core/clients/index.js"; import { getBase44ApiUrl } from "@/core/config.js"; import { ApiError, SchemaValidationError } from "@/core/errors.js"; -export const MAX_SESSION_BYTES = 25 * 1024 * 1024; +export const MAX_SESSION_BYTES = 50 * 1024 * 1024; const SessionImportResponseSchema = z .object({ @@ -67,7 +67,7 @@ function messageFor( } if (status === 404) return "Session import isn't available for this app."; if (status === 413) - return "This session is over 25 MB and can't be imported."; + return "This session is over 50 MB and can't be imported."; if (code === "unsupported_file") { return "That file isn't a Claude Code or Codex session."; } diff --git a/packages/cli/src/core/session-import/images.ts b/packages/cli/src/core/session-import/images.ts new file mode 100644 index 00000000..18b45084 --- /dev/null +++ b/packages/cli/src/core/session-import/images.ts @@ -0,0 +1,193 @@ +import { stat } from "node:fs/promises"; +import { extname } from "node:path"; +import { readFile } from "@/core/utils/fs.js"; + +type Entry = Record; + +const MEDIA_TYPES: Record = { + ".png": "image/png", + ".jpg": "image/jpeg", + ".jpeg": "image/jpeg", + ".gif": "image/gif", + ".webp": "image/webp", +}; +// Base Code imports only the newest 20 pasted images. +const MAX_ATTACHED_IMAGES = 20; +const MAX_ATTACHED_IMAGE_BYTES = 10 * 1024 * 1024; +const IMAGE_PATH = + /(?:\/|[A-Za-z]:\\)[^\s"'`<>()]+\.(?:png|jpe?g|gif|webp)\b/gi; + +const asRecord = (value: unknown): Entry => + value && typeof value === "object" ? (value as Entry) : {}; + +const isImage = (value: unknown) => + ["image", "input_image"].includes(String(asRecord(value).type)); + +function withoutImages(value: unknown): unknown { + if (Array.isArray(value)) { + return value.filter((item) => !isImage(item)).map(withoutImages); + } + if (value && typeof value === "object") { + return Object.fromEntries( + Object.entries(value).map(([key, item]) => [key, withoutImages(item)]), + ); + } + return value; +} + +function strings(value: unknown): string[] { + if (typeof value === "string") return [value]; + if (Array.isArray(value)) return value.flatMap(strings); + if (value && typeof value === "object") { + return Object.values(value).flatMap(strings); + } + return []; +} +const payloadOf = (entry: Entry) => asRecord(entry.payload); +const messageOf = (entry: Entry) => asRecord(entry.message); + +/** A message the person typed: the only place an attached file's path counts. */ +function isTypedUserMessage(entry: Entry): boolean { + if (entry.type === "user") { + if (entry.isMeta || entry.isSidechain) return false; + const content = asRecord(entry.message).content; + return ( + typeof content === "string" || + (Array.isArray(content) && + !content.some((block) => asRecord(block).type === "tool_result")) + ); + } + const payload = asRecord(entry.payload); + return ( + entry.type === "response_item" && + payload.type === "message" && + payload.role === "user" + ); +} + +/** The arguments of every tool call: an attached file counts once the agent opened it. */ +function toolInputs(entries: Entry[]): string { + return entries + .flatMap((entry) => { + const payload = asRecord(entry.payload); + if (entry.type === "response_item") { + return String(payload.type).endsWith("_call") + ? strings([payload.input, payload.arguments]) + : []; + } + const content = messageOf(entry).content; + return (Array.isArray(content) ? content : []) + .filter((block) => asRecord(block).type === "tool_use") + .flatMap((block) => strings(asRecord(block).input)); + }) + .join("\n"); +} + +async function imageBlock(path: string, codex: boolean) { + const mediaType = MEDIA_TYPES[extname(path).toLowerCase()]; + const size = await stat(path).then( + (info) => (info.isFile() ? info.size : 0), + () => 0, + ); + if (!mediaType || !size || size > MAX_ATTACHED_IMAGE_BYTES) return undefined; + const data = (await readFile(path)).toString("base64"); + return codex + ? { type: "input_image", image_url: `data:${mediaType};base64,${data}` } + : { + type: "image", + source: { type: "base64", media_type: mediaType, data }, + }; +} + +/** + * Prepares a session file for upload. Images the agent's tools produced + * (screenshots, a Read of a PNG) are dropped: Base Code never imports them, and + * they are most of a big session's size. An image the user attached as a file + * path (as apps that wrap Claude Code or Codex do) and the agent then opened is + * added to the user's message, as if it had been pasted. + */ +export async function prepareSessionImages( + content: Uint8Array, +): Promise { + const lines = Buffer.from(content).toString("utf8").split("\n"); + const entries = lines.map((line): Entry | undefined => { + try { + const entry = JSON.parse(line); + return entry && typeof entry === "object" ? entry : undefined; + } catch { + return undefined; + } + }); + const changed = new Set(); + + entries.forEach((entry, i) => { + if (!entry || !lines[i]?.includes("image")) return; + const message = messageOf(entry); + if ( + entry.type === "user" && + Array.isArray(message.content) && + message.content.some((block) => asRecord(block).type === "tool_result") + ) { + const content = message.content.map((block) => + asRecord(block).type === "tool_result" ? withoutImages(block) : block, + ); + // toolUseResult repeats the tool's output, a Read image's base64 included. + const { toolUseResult: _, ...rest } = entry; + entries[i] = { ...rest, message: { ...message, content } }; + changed.add(i); + } else if ( + entry.type === "response_item" && + String(asRecord(entry.payload).type).endsWith("_output") + ) { + entries[i] = { ...entry, payload: withoutImages(entry.payload) }; + changed.add(i); + } + }); + + const opened = toolInputs(entries.filter((entry) => !!entry) as Entry[]); + let budget = MAX_ATTACHED_IMAGES; + for (let i = entries.length - 1; i >= 0 && budget > 0; i--) { + const entry = entries[i]; + if (!entry || !isTypedUserMessage(entry)) continue; + const codex = entry.type === "response_item"; + const raw = codex ? payloadOf(entry).content : messageOf(entry).content; + const blocks: unknown[] = + typeof raw === "string" + ? [{ type: "text", text: raw }] + : Array.isArray(raw) + ? [...raw] + : []; + const paths = new Set( + strings(blocks.filter((block) => !isImage(block))).flatMap( + (text) => text.match(IMAGE_PATH) ?? [], + ), + ); + const added = []; + for (const path of paths) { + if (budget === 0 || !opened.includes(path)) continue; + const block = await imageBlock(path, codex); + if (block) { + added.push(block); + budget--; + } + } + if (!added.length) continue; + entries[i] = codex + ? { + ...entry, + payload: { ...payloadOf(entry), content: [...blocks, ...added] }, + } + : { + ...entry, + message: { ...messageOf(entry), content: [...blocks, ...added] }, + }; + changed.add(i); + } + + if (!changed.size) return content; + return Buffer.from( + lines + .map((line, i) => (changed.has(i) ? JSON.stringify(entries[i]) : line)) + .join("\n"), + ); +} diff --git a/packages/cli/tests/cli/sessions_import.spec.ts b/packages/cli/tests/cli/sessions_import.spec.ts index 5442e36d..304110fc 100644 --- a/packages/cli/tests/cli/sessions_import.spec.ts +++ b/packages/cli/tests/cli/sessions_import.spec.ts @@ -396,8 +396,8 @@ describe("sessions import command", () => { expect(uploads[0]?.authorization).not.toBe(`Bearer ${TOKEN}`); }); - it("refuses a file over 25 MB before uploading", async () => { - const file = await plant("big.jsonl", "x".repeat(25 * 1024 * 1024 + 1)); + it("refuses a file over 50 MB before uploading", async () => { + const file = await plant("big.jsonl", "x".repeat(50 * 1024 * 1024 + 1)); const result = await t.run( "sessions", @@ -412,11 +412,124 @@ describe("sessions import command", () => { t.expectResult(result).toFail(); t.expectResult(result).toContain( - "This session is over 25 MB and can't be imported.", + "This session is over 50 MB and can't be imported.", ); expect(uploads).toHaveLength(0); }); + const importFile = (file: string) => + t.run( + "sessions", + "import", + "--file", + file, + "--app-id", + APP_ID, + "--token", + TOKEN, + ); + + it("uploads pasted images but not the ones the agent's tools produced", async () => { + const image = (data: string) => ({ + type: "image", + source: { type: "base64", media_type: "image/png", data }, + }); + const file = await plant( + "pasted.jsonl", + [ + { + type: "user", + message: { + role: "user", + content: [{ type: "text", text: "Fix this" }, image("UEFTVEVE")], + }, + }, + { + type: "assistant", + message: { + role: "assistant", + content: [ + { + type: "tool_use", + id: "t1", + name: "Read", + input: { file_path: "/a.png" }, + }, + ], + }, + }, + { + type: "user", + message: { + role: "user", + content: [ + { + type: "tool_result", + tool_use_id: "t1", + content: [image("VE9PTA==")], + }, + ], + }, + toolUseResult: { type: "image", file: { base64: "VE9PTA==" } }, + }, + ] + .map((line) => JSON.stringify(line)) + .join("\n"), + ); + mockUpload(); + + t.expectResult(await importFile(file)).toSucceed(); + + expect(uploads[0]?.body).toContain("UEFTVEVE"); + expect(uploads[0]?.body).toContain('"tool_use_id":"t1"'); + expect(uploads[0]?.body).not.toContain("VE9PTA=="); + }); + + it("adds an image attached as a file path once the agent opened it", async () => { + const opened = await plant("attachments/shot.png", "opened-image"); + const mentioned = await plant("attachments/other.png", "mentioned-only"); + const message = (role: string, text: string) => ({ + type: "response_item", + payload: { + type: "message", + role, + content: [ + { type: role === "user" ? "input_text" : "output_text", text }, + ], + }, + }); + const file = await plant( + "rollout-attached.jsonl", + [ + { type: "session_meta", payload: { cwd: "/repo" } }, + message( + "user", + `What's in this image?\n\n# Attached files\n\n- ${opened}\n- ${mentioned}`, + ), + { + type: "response_item", + payload: { + type: "custom_tool_call", + name: "exec", + input: `tools.view_image({path:"${opened}"})`, + }, + }, + message("assistant", "A screenshot."), + ] + .map((line) => JSON.stringify(line)) + .join("\n"), + ); + mockUpload(); + + t.expectResult(await importFile(file)).toSucceed(); + + const base64 = (text: string) => Buffer.from(text).toString("base64"); + expect(uploads[0]?.body).toContain( + `"image_url":"data:image/png;base64,${base64("opened-image")}"`, + ); + expect(uploads[0]?.body).not.toContain(base64("mentioned-only")); + }); + it("rejects a claude.ai 'Copy link' address", async () => { const result = await t.run( "sessions",