Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -738,6 +738,7 @@ describe('ResumableAgentController resume metadata', () => {
model: 'gpt-3.5-turbo',
/** The OWNING replica's seal capability, read by the steer route. */
preemptCapable: true,
steerQuotesCapable: true,
agent_id: undefined,
isTemporary: true,
responseMessageId: expect.stringMatching(/^[0-9a-f-]{36}$/),
Expand Down
54 changes: 44 additions & 10 deletions api/server/controllers/agents/client.js
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ const {
isSteeringSupported,
isSteerPreemptSupported,
buildSteerMedia,
collectSteerStampTargets,
stampSteerPartMedia,
createActivityLabelWiring,
createActivityPhaseWiring,
Expand Down Expand Up @@ -387,6 +388,9 @@ class AgentClient extends BaseClient {
...(item.clientSteerId && { clientSteerId: item.clientSteerId }),
createdAt: item.createdAt,
...(item.files?.length && { files: item.files }),
// Persisted separately from the text (mirroring `message.quotes`) so the
// UI renders reference blocks and replay re-merges them per turn.
...(item.quotes?.length && { quotes: item.quotes }),
};
this.contentParts.push(part);
this.steerOffsetState.offset += 1;
Expand Down Expand Up @@ -1892,22 +1896,33 @@ class AgentClient extends BaseClient {

payload = formattedMessages;
this.modelBoundSteerFileIdsBySourceMessageId = new Map();
if (this.options.resendFiles) {
/** Persisted steer parts of past turns replay with their attachments:
* one batched owner-scoped fetch, re-encoded per turn and stamped as a
* transient `media` array (same resend semantics as message files).
* The stamp lands after the loop above finalized its counts, so the
* re-encoded media (minus the text part the steer part already counted)
* is folded into the budget here — large steered attachments must
* shrink the window like any other resent media. */
/** Persisted steer parts of past turns replay with their attachments and
* quotes: one batched owner-scoped fetch, re-encoded per turn and
* stamped as a transient `media` array (same resend semantics as
* message files). Runs regardless of `resendFiles` because quote-bearing
* parts must re-merge their excerpts every turn (mirroring
* `prependQuotes` above); file encoding stays gated on the setting via
* the flag. The stamp lands after the loop above finalized its counts,
* so the re-encoded media (minus the text part the steer part already
* counted) is folded into the budget here — large steered attachments
* and quote blocks must shrink the window like any other resent media.
* The synchronous collection keeps steer-free histories on the
* zero-await path to the parallel context kickoff below, and the
* collected targets feed the stamp directly so the history is scanned
* once. */
const resendSteerFiles = this.options.resendFiles === true;
const steerStampTargets = collectSteerStampTargets(payload, resendSteerFiles);
if (steerStampTargets.length > 0) {
const stamped = await stampSteerPartMedia({
client: this,
user: this.options.req?.user,
payload,
targets: steerStampTargets,
// addPreviousAttachments already fetched steer-part refs in its single
// per-turn historical-files query — no second round trip.
docsById: this.authorizedHistoricalFiles,
getFiles: db.getFiles,
resendFiles: resendSteerFiles,
});
for (const { sourceMessageId, fileIds } of stamped) {
if (typeof sourceMessageId !== 'string' || sourceMessageId.length === 0) {
Expand All @@ -1927,8 +1942,8 @@ class AgentClient extends BaseClient {
for (const { index, media, steerText } of stamped) {
/** Count the FULL stamped content and subtract only the steer body
* (already counted inside the assistant message): extracted file
* context prepended into the text part must hit the budget too, or
* large steered documents bypass pruning. */
* context and merged quote blocks prepended into the text part must
* hit the budget too, or large steered documents bypass pruning. */
const fullTokens = countFormattedMessageTokens({ role: 'user', content: media }, encoding);
const bodyTokens = steerText
? countFormattedMessageTokens(
Expand All @@ -1949,6 +1964,25 @@ class AgentClient extends BaseClient {
memoryFormattedMessages[i] ?? buildMemoryFormattedMessage(orderedMessages[i]),
);
}
/** The memory copy feeds `processMemory` through the same
* `formatAgentMessages` replay, which reads `part.media`/`part.steer`
* and ignores `part.quotes` — so a steer whose substance lives in its
* quote must be quote-merged here too or memory extraction never sees
* it. Quote merge only (`resendFiles: false`): file media is exactly
* what the memory copy exists to exclude, and text-only stamps touch
* no file fetch or encode. Runs after the fill above so late-built
* copies are stamped too. */
const memorySteerTargets = collectSteerStampTargets(memoryPayload, false);
if (memorySteerTargets.length > 0) {
await stampSteerPartMedia({
client: this,
user: this.options.req?.user,
payload: memoryPayload,
targets: memorySteerTargets,
getFiles: db.getFiles,
resendFiles: false,
});
}
}
this.memoryPayload = hasFileContext ? memoryPayload : null;
messages = orderedMessages;
Expand Down
54 changes: 54 additions & 0 deletions api/server/controllers/agents/client.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -3749,6 +3749,60 @@ describe('AgentClient - titleConvo', () => {
);
});

it('quote-merges historical steer parts into the prompt AND the memory copy', async () => {
const previousFileContext =
'Attached document(s):\n```md\n# "previous.txt"\nPrevious turn file body\n```';

const result = await client.buildMessages(
[
{
messageId: 'msg-1',
parentMessageId: null,
sender: 'User',
text: 'Summarize.',
isCreatedByUser: true,
fileContext: previousFileContext,
},
{
messageId: 'msg-2',
parentMessageId: 'msg-1',
sender: 'Assistant',
text: '',
isCreatedByUser: false,
content: [
{ type: ContentTypes.TEXT, text: 'working on it' },
{
type: ContentTypes.STEER,
[ContentTypes.STEER]: 'remember this',
steerId: 's1',
quotes: ['the important fact'],
},
],
},
{
messageId: 'msg-3',
parentMessageId: 'msg-2',
sender: 'User',
text: 'Continue.',
isCreatedByUser: true,
},
],
'msg-3',
{},
);

const merged = '> the important fact\n\nremember this';
const promptSteer = result.prompt[1].content.find((part) => part.type === ContentTypes.STEER);
expect(promptSteer.media).toEqual([{ type: ContentTypes.TEXT, text: merged }]);
// The memory copy replays through the same formatter, which ignores
// `part.quotes` — it needs its own merged stamp or memory extraction
// never sees the excerpt.
const memorySteer = client.memoryPayload[1].content.find(
(part) => part.type === ContentTypes.STEER,
);
expect(memorySteer.media).toEqual([{ type: ContentTypes.TEXT, text: merged }]);
});

it('persists canonical token counts while counting request file context for the prompt', async () => {
const { countFormattedMessageTokens } = require('@librechat/api');
const currentFile = makeTextFile('current-file', 'current.txt', 'Current turn file body');
Expand Down
6 changes: 5 additions & 1 deletion api/server/controllers/agents/request.js
Original file line number Diff line number Diff line change
Expand Up @@ -444,7 +444,7 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit
const isRecoveredSteerRequest = recoveredSteerId != null;
const recoveryUserMessageId = rawOverrideUserMessageId;
const recoveredSteerPayload = isRecoveredSteerRequest
? buildRecoveredSteerPayload(text, req.body?.files)
? buildRecoveredSteerPayload(text, req.body?.files, req.body?.quotes)
: undefined;
/** A recovered steer is handed off as a new ordinary user turn. Edit,
* regenerate, continue, and arbitrary override-id shapes can reuse an
Expand Down Expand Up @@ -1106,6 +1106,10 @@ const ResumableAgentController = async (req, res, next, initializeClient, addTit
// route may land on a different replica whose own SDK probe would
// answer for the wrong process during a rolling deploy.
preemptCapable: isSteerPreemptSupported(),
// Same owner-recorded pattern: this build's drain merges queued steer
// quotes into the injected turn. Admission on another replica must
// not store/acknowledge quotes an older owner would drop.
steerQuotesCapable: true,
// Persist the originating agent so a HITL resume can refuse to rebuild this
// paused run on a different agent (see resume.js).
agent_id: endpointOption.agent_id ?? req.body?.agent_id,
Expand Down
3 changes: 3 additions & 0 deletions api/server/controllers/agents/resume.js
Original file line number Diff line number Diff line change
Expand Up @@ -1222,6 +1222,9 @@ const ResumeAgentController = async (req, res, next, initializeClient, addTitle)
pendingAction.actionId,
{
preemptCapable: isSteerPreemptSupported(),
// The handover owner's quote handling replaces the previous
// replica's flag, mirroring `preemptCapable` above.
steerQuotesCapable: true,
providerExecutionId,
providerDrained: true,
...(resolvedAskUserQuestion && { resolvedAskUserQuestions }),
Expand Down
2 changes: 2 additions & 0 deletions client/jest.config.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@ module.exports = {
// },
moduleNameMapper: {
'\\.(css)$': 'identity-obj-proxy',
/** Mirror the vite resolve.alias so tests parse math with the same tokenizer as production. */
'^micromark-extension-math$': 'micromark-extension-llm-math',
'\\.(jpg|jpeg|png|gif|eot|otf|webp|svg|ttf|woff|woff2|mp4|webm|wav|mp3|m4a|aac|oga)$':
'jest-file-loader',
'^test/(.*)$': '<rootDir>/test/$1',
Expand Down
3 changes: 3 additions & 0 deletions client/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,8 @@
"micromark-extension-gfm": "^3.0.0",
"micromark-extension-llm-math": "^3.1.0",
"micromark-extension-math": "^3.1.0",
"micromark-util-character": "^2.1.0",
"micromark-util-symbol": "^2.0.0",
"monaco-editor": "^0.56.0",
"qrcode.react": "^4.2.0",
"rc-input-number": "^7.4.2",
Expand Down Expand Up @@ -159,6 +161,7 @@
"jest-environment-jsdom": "^30.2.0",
"jest-file-loader": "^1.0.3",
"jest-junit": "^17.0.0",
"micromark-util-types": "^2.0.0",
"postcss": "^8.5.18",
"postcss-preset-env": "^11.2.0",
"tailwindcss": "^3.4.1",
Expand Down
4 changes: 4 additions & 0 deletions client/src/components/Chat/Input/InFlightSteers.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import {
import FilePreviewDialog from '~/components/Chat/Messages/Content/FilePreviewDialog';
import { supportsGenerationProtocolV2, useArmSteerMutation } from '~/data-provider';
import { steerOverlayHeightFamily, escalatingSteerFamily } from '~/store/steer';
import MessageQuotes from '~/components/Chat/Messages/Content/MessageQuotes';
import MarkdownLite from '~/components/Chat/Messages/Content/MarkdownLite';
import FileContainer from '~/components/Chat/Input/Files/FileContainer';
import { useSteerCancel, useSteerReclaim, useLocalize } from '~/hooks';
Expand Down Expand Up @@ -482,6 +483,9 @@ const InFlightSteer = memo(function InFlightSteer({
{localize(preempting ? 'com_ui_steer_in_flight_preempt' : 'com_ui_steer_in_flight')}
</span>
<div className="flex min-w-0 flex-col items-start gap-1">
{/* Same reference blocks the applied `SteerPart` shows, outside the
* collapse so the excerpts stay visible while a long steer clips. */}
<MessageQuotes quotes={steer.quotes} />
<div
ref={contentRef}
id={contentId}
Expand Down
8 changes: 5 additions & 3 deletions client/src/components/Chat/Input/PendingQuoteChips.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,11 @@ const CLOSE_DELAY_MS = 120;
* `autoFocusOnShow` is disabled so opening never pulls focus off the composer;
* `autoFocusOnHide` still returns focus to the trigger when focus was inside.
*
* Reads + writes `pendingQuotesByConvoId` directly; the atom is drained in
* `useChatFunctions.ask` on submit, so chips disappear once the message is sent
* (the excerpts then re-render as `MessageQuotes` on the user bubble).
* Reads + writes `pendingQuotesByConvoId` directly; the atom is drained on
* every submit route — `useChatFunctions.ask` for a fresh send, and the
* during-run steer/queue/interrupt paths in `useSteering` — so chips disappear
* once the message is sent (the excerpts then re-render as `MessageQuotes` on
* the user bubble, or inside the steer bubble for a mid-run injection).
*/
function PendingQuoteChips({ conversationId }: { conversationId: string }) {
const localize = useLocalize();
Expand Down
43 changes: 40 additions & 3 deletions client/src/components/Chat/Input/PendingSteerChips.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import { memo, useMemo, useRef, useState, useCallback } from 'react';
import { useAtomValue } from 'jotai';
import { useRecoilValue } from 'recoil';
import { useToastContext } from '@librechat/client';
import { X, Zap, Send, Clock, Pencil, Trash2, Paperclip, RotateCcw } from 'lucide-react';
import { X, Zap, Send, Clock, Pencil, Trash2, Paperclip, RotateCcw, TextQuote } from 'lucide-react';
import type { TMessage } from 'librechat-data-provider';
import type { SteeringControls, QueuedMessageContext } from '~/hooks/Chat/useSteering';
import type { PendingSteer, QueuedMessage } from '~/store/families';
Expand All @@ -24,19 +24,47 @@ import store from '~/store';
const ROW_CLASS =
'flex w-full items-center gap-2 rounded-xl border border-border-light bg-surface-secondary px-3 py-2 text-sm text-text-primary';

function AttachmentCount({ count, label }: { count: number; label: string }) {
function ContextCount({
icon,
count,
label,
}: {
icon: React.ReactNode;
count: number;
label: string;
}) {
if (count === 0) {
return null;
}
return (
<span className="flex shrink-0 items-center gap-0.5 text-xs text-text-secondary">
<Paperclip className="h-3.5 w-3.5" aria-hidden="true" />
{icon}
{count}
<span className="sr-only">{label}</span>
</span>
);
}

function AttachmentCount({ count, label }: { count: number; label: string }) {
return (
<ContextCount
icon={<Paperclip className="h-3.5 w-3.5" aria-hidden="true" />}
count={count}
label={label}
/>
);
}

function QuoteCount({ count, label }: { count: number; label: string }) {
return (
<ContextCount
icon={<TextQuote className="h-3.5 w-3.5" aria-hidden="true" />}
count={count}
label={label}
/>
);
}

function QueuedRow({
message,
steering,
Expand All @@ -61,6 +89,7 @@ function QueuedRow({
const toggleEntry = useDefaultToggleEntry(steering);
const interruptToggle = useInterruptToggleEntry();
const fileCount = message.files?.length ?? 0;
const quoteCount = message.quotes?.length ?? 0;
const isRecovered = message.recoverySteerId != null;
const actionPendingRef = useRef(false);
const [actionPending, setActionPending] = useState(false);
Expand Down Expand Up @@ -147,6 +176,10 @@ function QueuedRow({
<span className="min-w-0 flex-1 truncate" title={message.text}>
{message.text}
</span>
<QuoteCount
count={quoteCount}
label={localize('com_ui_queued_quote_count', { 0: String(quoteCount) })}
/>
<AttachmentCount
count={fileCount}
label={localize('com_ui_queued_attachment_count', { 0: String(fileCount) })}
Expand Down Expand Up @@ -275,6 +308,10 @@ function FailedSteerRow({
<span className="min-w-0 flex-1 truncate" title={steer.text}>
{steer.text}
</span>
<QuoteCount
count={steer.quotes?.length ?? 0}
label={localize('com_ui_queued_quote_count', { 0: String(steer.quotes?.length ?? 0) })}
/>
<span className="shrink-0 text-xs text-red-500">
{localize(
steer.deliveryUncertain ? 'com_ui_steer_delivery_unconfirmed' : 'com_ui_steer_failed',
Expand Down
5 changes: 1 addition & 4 deletions client/src/components/Chat/Input/QuoteButton.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -2,18 +2,15 @@ import { memo, useRef, useState, useEffect, useCallback, useLayoutEffect } from
import { createPortal } from 'react-dom';
import { TextQuote } from 'lucide-react';
import { useSetRecoilState } from 'recoil';
import { cn, MAX_QUOTE_COUNT } from '~/utils';
import { mainTextareaId } from '~/common';
import { useLocalize } from '~/hooks';
import { cn } from '~/utils';
import store from '~/store';

/** Only selections fully inside a rendered chat message get the popup. */
const MESSAGE_SELECTOR = '.message-render';
/** Max characters captured per excerpt (backend re-caps as defense-in-depth). */
const MAX_QUOTE_LENGTH = 1500;
/** Max excerpts queued at once; mirrors the backend `QUOTE_MAX_COUNT` cap so
* the composer never shows more quotes than the model actually receives. */
const MAX_QUOTE_COUNT = 10;
/** Vertical gap (px) between the selection and the popup. */
const POPUP_OFFSET = 8;
/** Keep the popup this far (px) from the viewport edges. */
Expand Down
13 changes: 13 additions & 0 deletions client/src/components/Chat/Input/__tests__/InFlightSteers.test.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -185,6 +185,19 @@ describe('InFlightSteers', () => {
expect(screen.queryByTestId('in-flight-steers')).toBeNull();
});

it('renders carried quotes as the same reference blocks the applied part shows', () => {
renderSteers([
{
steerId: 's-quoted',
text: 'about the selection',
status: 'pending',
createdAt: 1,
quotes: ['the selected excerpt'],
},
]);
expect(screen.getByTestId('message-quotes')).toHaveTextContent('the selected excerpt');
});

it('shows the menu at rest on every pointer, without hover-gating', () => {
renderSteers([
{ steerId: 's-ack', text: 'waiting on boundary', status: 'pending', createdAt: 1 },
Expand Down
Loading
Loading