diff --git a/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js b/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js index a5c747364f4..59e24d3c8df 100644 --- a/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js +++ b/api/server/controllers/agents/__tests__/request.resumeMetadata.spec.js @@ -738,6 +738,7 @@ describe('ResumableAgentController resume metadata', () => { model: 'gpt-3.5-turbo', /** The OWNING replica's seal capability, read by the steer route. */ preemptCapable: true, + steerQuotesCapable: true, agent_id: undefined, isTemporary: true, responseMessageId: expect.stringMatching(/^[0-9a-f-]{36}$/), diff --git a/api/server/controllers/agents/client.js b/api/server/controllers/agents/client.js index 320abe0691d..6c1b6a61368 100644 --- a/api/server/controllers/agents/client.js +++ b/api/server/controllers/agents/client.js @@ -58,6 +58,7 @@ const { isSteeringSupported, isSteerPreemptSupported, buildSteerMedia, + collectSteerStampTargets, stampSteerPartMedia, createActivityLabelWiring, createActivityPhaseWiring, @@ -387,6 +388,9 @@ class AgentClient extends BaseClient { ...(item.clientSteerId && { clientSteerId: item.clientSteerId }), createdAt: item.createdAt, ...(item.files?.length && { files: item.files }), + // Persisted separately from the text (mirroring `message.quotes`) so the + // UI renders reference blocks and replay re-merges them per turn. + ...(item.quotes?.length && { quotes: item.quotes }), }; this.contentParts.push(part); this.steerOffsetState.offset += 1; @@ -1892,22 +1896,33 @@ class AgentClient extends BaseClient { payload = formattedMessages; this.modelBoundSteerFileIdsBySourceMessageId = new Map(); - if (this.options.resendFiles) { - /** Persisted steer parts of past turns replay with their attachments: - * one batched owner-scoped fetch, re-encoded per turn and stamped as a - * transient `media` array (same resend semantics as message files). - * The stamp lands after the loop above finalized its counts, so the - * re-encoded media (minus the text part the steer part already counted) - * is folded into the budget here — large steered attachments must - * shrink the window like any other resent media. */ + /** Persisted steer parts of past turns replay with their attachments and + * quotes: one batched owner-scoped fetch, re-encoded per turn and + * stamped as a transient `media` array (same resend semantics as + * message files). Runs regardless of `resendFiles` because quote-bearing + * parts must re-merge their excerpts every turn (mirroring + * `prependQuotes` above); file encoding stays gated on the setting via + * the flag. The stamp lands after the loop above finalized its counts, + * so the re-encoded media (minus the text part the steer part already + * counted) is folded into the budget here — large steered attachments + * and quote blocks must shrink the window like any other resent media. + * The synchronous collection keeps steer-free histories on the + * zero-await path to the parallel context kickoff below, and the + * collected targets feed the stamp directly so the history is scanned + * once. */ + const resendSteerFiles = this.options.resendFiles === true; + const steerStampTargets = collectSteerStampTargets(payload, resendSteerFiles); + if (steerStampTargets.length > 0) { const stamped = await stampSteerPartMedia({ client: this, user: this.options.req?.user, payload, + targets: steerStampTargets, // addPreviousAttachments already fetched steer-part refs in its single // per-turn historical-files query — no second round trip. docsById: this.authorizedHistoricalFiles, getFiles: db.getFiles, + resendFiles: resendSteerFiles, }); for (const { sourceMessageId, fileIds } of stamped) { if (typeof sourceMessageId !== 'string' || sourceMessageId.length === 0) { @@ -1927,8 +1942,8 @@ class AgentClient extends BaseClient { for (const { index, media, steerText } of stamped) { /** Count the FULL stamped content and subtract only the steer body * (already counted inside the assistant message): extracted file - * context prepended into the text part must hit the budget too, or - * large steered documents bypass pruning. */ + * context and merged quote blocks prepended into the text part must + * hit the budget too, or large steered documents bypass pruning. */ const fullTokens = countFormattedMessageTokens({ role: 'user', content: media }, encoding); const bodyTokens = steerText ? countFormattedMessageTokens( @@ -1949,6 +1964,25 @@ class AgentClient extends BaseClient { memoryFormattedMessages[i] ?? buildMemoryFormattedMessage(orderedMessages[i]), ); } + /** The memory copy feeds `processMemory` through the same + * `formatAgentMessages` replay, which reads `part.media`/`part.steer` + * and ignores `part.quotes` — so a steer whose substance lives in its + * quote must be quote-merged here too or memory extraction never sees + * it. Quote merge only (`resendFiles: false`): file media is exactly + * what the memory copy exists to exclude, and text-only stamps touch + * no file fetch or encode. Runs after the fill above so late-built + * copies are stamped too. */ + const memorySteerTargets = collectSteerStampTargets(memoryPayload, false); + if (memorySteerTargets.length > 0) { + await stampSteerPartMedia({ + client: this, + user: this.options.req?.user, + payload: memoryPayload, + targets: memorySteerTargets, + getFiles: db.getFiles, + resendFiles: false, + }); + } } this.memoryPayload = hasFileContext ? memoryPayload : null; messages = orderedMessages; diff --git a/api/server/controllers/agents/client.test.js b/api/server/controllers/agents/client.test.js index 073daddbf0c..9b8d3200e99 100644 --- a/api/server/controllers/agents/client.test.js +++ b/api/server/controllers/agents/client.test.js @@ -3749,6 +3749,60 @@ describe('AgentClient - titleConvo', () => { ); }); + it('quote-merges historical steer parts into the prompt AND the memory copy', async () => { + const previousFileContext = + 'Attached document(s):\n```md\n# "previous.txt"\nPrevious turn file body\n```'; + + const result = await client.buildMessages( + [ + { + messageId: 'msg-1', + parentMessageId: null, + sender: 'User', + text: 'Summarize.', + isCreatedByUser: true, + fileContext: previousFileContext, + }, + { + messageId: 'msg-2', + parentMessageId: 'msg-1', + sender: 'Assistant', + text: '', + isCreatedByUser: false, + content: [ + { type: ContentTypes.TEXT, text: 'working on it' }, + { + type: ContentTypes.STEER, + [ContentTypes.STEER]: 'remember this', + steerId: 's1', + quotes: ['the important fact'], + }, + ], + }, + { + messageId: 'msg-3', + parentMessageId: 'msg-2', + sender: 'User', + text: 'Continue.', + isCreatedByUser: true, + }, + ], + 'msg-3', + {}, + ); + + const merged = '> the important fact\n\nremember this'; + const promptSteer = result.prompt[1].content.find((part) => part.type === ContentTypes.STEER); + expect(promptSteer.media).toEqual([{ type: ContentTypes.TEXT, text: merged }]); + // The memory copy replays through the same formatter, which ignores + // `part.quotes` — it needs its own merged stamp or memory extraction + // never sees the excerpt. + const memorySteer = client.memoryPayload[1].content.find( + (part) => part.type === ContentTypes.STEER, + ); + expect(memorySteer.media).toEqual([{ type: ContentTypes.TEXT, text: merged }]); + }); + it('persists canonical token counts while counting request file context for the prompt', async () => { const { countFormattedMessageTokens } = require('@librechat/api'); const currentFile = makeTextFile('current-file', 'current.txt', 'Current turn file body'); diff --git a/api/server/controllers/agents/request.js b/api/server/controllers/agents/request.js index 0ab98fdb0ce..75a1ee744c6 100644 --- a/api/server/controllers/agents/request.js +++ b/api/server/controllers/agents/request.js @@ -444,7 +444,7 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit const isRecoveredSteerRequest = recoveredSteerId != null; const recoveryUserMessageId = rawOverrideUserMessageId; const recoveredSteerPayload = isRecoveredSteerRequest - ? buildRecoveredSteerPayload(text, req.body?.files) + ? buildRecoveredSteerPayload(text, req.body?.files, req.body?.quotes) : undefined; /** A recovered steer is handed off as a new ordinary user turn. Edit, * regenerate, continue, and arbitrary override-id shapes can reuse an @@ -1106,6 +1106,10 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit // route may land on a different replica whose own SDK probe would // answer for the wrong process during a rolling deploy. preemptCapable: isSteerPreemptSupported(), + // Same owner-recorded pattern: this build's drain merges queued steer + // quotes into the injected turn. Admission on another replica must + // not store/acknowledge quotes an older owner would drop. + steerQuotesCapable: true, // Persist the originating agent so a HITL resume can refuse to rebuild this // paused run on a different agent (see resume.js). agent_id: endpointOption.agent_id ?? req.body?.agent_id, diff --git a/api/server/controllers/agents/resume.js b/api/server/controllers/agents/resume.js index 30e3c8c73c8..5d54b6c96f5 100644 --- a/api/server/controllers/agents/resume.js +++ b/api/server/controllers/agents/resume.js @@ -1222,6 +1222,9 @@ const ResumeAgentController = async (req, res, next, initializeClient, addTitle) pendingAction.actionId, { preemptCapable: isSteerPreemptSupported(), + // The handover owner's quote handling replaces the previous + // replica's flag, mirroring `preemptCapable` above. + steerQuotesCapable: true, providerExecutionId, providerDrained: true, ...(resolvedAskUserQuestion && { resolvedAskUserQuestions }), diff --git a/client/jest.config.cjs b/client/jest.config.cjs index 996a9fb7e6c..94dc2bfa65e 100644 --- a/client/jest.config.cjs +++ b/client/jest.config.cjs @@ -25,6 +25,8 @@ module.exports = { // }, moduleNameMapper: { '\\.(css)$': 'identity-obj-proxy', + /** Mirror the vite resolve.alias so tests parse math with the same tokenizer as production. */ + '^micromark-extension-math$': 'micromark-extension-llm-math', '\\.(jpg|jpeg|png|gif|eot|otf|webp|svg|ttf|woff|woff2|mp4|webm|wav|mp3|m4a|aac|oga)$': 'jest-file-loader', '^test/(.*)$': '/test/$1', diff --git a/client/package.json b/client/package.json index 48de22cdda6..1bbea2c537d 100644 --- a/client/package.json +++ b/client/package.json @@ -91,6 +91,8 @@ "micromark-extension-gfm": "^3.0.0", "micromark-extension-llm-math": "^3.1.0", "micromark-extension-math": "^3.1.0", + "micromark-util-character": "^2.1.0", + "micromark-util-symbol": "^2.0.0", "monaco-editor": "^0.56.0", "qrcode.react": "^4.2.0", "rc-input-number": "^7.4.2", @@ -159,6 +161,7 @@ "jest-environment-jsdom": "^30.2.0", "jest-file-loader": "^1.0.3", "jest-junit": "^17.0.0", + "micromark-util-types": "^2.0.0", "postcss": "^8.5.18", "postcss-preset-env": "^11.2.0", "tailwindcss": "^3.4.1", diff --git a/client/src/components/Chat/Input/InFlightSteers.tsx b/client/src/components/Chat/Input/InFlightSteers.tsx index 633e6250a9a..e959b64693f 100644 --- a/client/src/components/Chat/Input/InFlightSteers.tsx +++ b/client/src/components/Chat/Input/InFlightSteers.tsx @@ -16,6 +16,7 @@ import { import FilePreviewDialog from '~/components/Chat/Messages/Content/FilePreviewDialog'; import { supportsGenerationProtocolV2, useArmSteerMutation } from '~/data-provider'; import { steerOverlayHeightFamily, escalatingSteerFamily } from '~/store/steer'; +import MessageQuotes from '~/components/Chat/Messages/Content/MessageQuotes'; import MarkdownLite from '~/components/Chat/Messages/Content/MarkdownLite'; import FileContainer from '~/components/Chat/Input/Files/FileContainer'; import { useSteerCancel, useSteerReclaim, useLocalize } from '~/hooks'; @@ -482,6 +483,9 @@ const InFlightSteer = memo(function InFlightSteer({ {localize(preempting ? 'com_ui_steer_in_flight_preempt' : 'com_ui_steer_in_flight')}
+ {/* Same reference blocks the applied `SteerPart` shows, outside the + * collapse so the excerpts stay visible while a long steer clips. */} +
-