From 32f19a8d74fae9b78c5bcab26eff156ad3839ebc Mon Sep 17 00:00:00 2001 From: Valeriy Selitskiy <239034+iamwavecut@users.noreply.github.com> Date: Sat, 10 Oct 2026 13:20:35 +0200 Subject: [PATCH 1/3] Honor explicit image actions and separate readable captions --- crates/openplotva-app/src/agent_tools.rs | 5 + .../openplotva-app/src/dialog_jobs/tests.rs | 115 ++++++++++++++++++ crates/openplotva-app/src/dialog_messages.rs | 74 ++++++++--- crates/openplotva-app/src/dialog_runtime.rs | 3 + crates/openplotva-app/src/dialog_tools.rs | 51 +++++++- .../openplotva-app/src/dialog_turn/session.rs | 39 +++++- crates/openplotva-core/src/lib.rs | 3 + crates/openplotva-dialog/src/dispatch.rs | 1 + crates/openplotva-dialog/src/lib.rs | 35 ++++-- crates/openplotva-dialog/src/transcript.rs | 8 +- crates/openplotva-llm/src/aifarm.rs | 106 +++++++++++++++- crates/openplotva-llm/src/gemini.rs | 2 + crates/openplotva-llm/src/whitecircle.rs | 2 + crates/openplotva-updates/src/lib.rs | 46 ++++++- prompts/aifarm/system.prompt | 4 +- 15 files changed, 453 insertions(+), 41 deletions(-) diff --git a/crates/openplotva-app/src/agent_tools.rs b/crates/openplotva-app/src/agent_tools.rs index 7816e4f7..4fab8460 100644 --- a/crates/openplotva-app/src/agent_tools.rs +++ b/crates/openplotva-app/src/agent_tools.rs @@ -46,6 +46,11 @@ impl DialogToolbox for AgentContextTools { user_id: ctx.user_id, user_full_name: ctx.user_full_name, prompt: step.prompt, + caption: if step.caption.trim().is_empty() { + ctx.message_text + } else { + step.caption + }, thread_id: ctx.thread_id, is_forum: ctx.thread_id.is_some(), }) diff --git a/crates/openplotva-app/src/dialog_jobs/tests.rs b/crates/openplotva-app/src/dialog_jobs/tests.rs index 3d38c539..e3e5cebd 100644 --- a/crates/openplotva-app/src/dialog_jobs/tests.rs +++ b/crates/openplotva-app/src/dialog_jobs/tests.rs @@ -6318,6 +6318,121 @@ fn step_context(request: &openplotva_dialog::ChatStepRequest) -> Vec { .collect() } +#[tokio::test] +async fn explicit_image_request_repairs_text_into_a_native_draw() -> Result<(), Box> { + let now = OffsetDateTime::from_unix_timestamp(1_779_193_800)?; + for queued in [true, false] { + let queue = InMemoryTaskQueue::new(); + let mut params = dialog_params("Плотва, нарисуй рыбу в космосе"); + params.meta = serde_json::json!({"requested_tool": "draw_image"}); + queue.assign(DIALOG_AIFARM_QUEUE_NAME, new_dialog_job_at(params, now)); + let provider = StepProviderStub::with_steps(vec![ + Ok(step_text("Сначала я подумаю, как изобразить эту рыбу.")), + Ok(step_tools( + "", + vec![( + "draw", + openplotva_dialog::ToolStep { + step: "draw_image".into(), + prompt: "A fish floating in space, cinematic lighting".into(), + caption: "Рыба в космосе".into(), + ..Default::default() + }, + )], + )), + Ok(step_text("Сервис рисования сейчас недоступен.")), + ]); + let toolbox = Arc::new(if queued { + SessionToolboxStub::with_queued_draw("image-ticket") + } else { + SessionToolboxStub::default() + }); + let wiring = session_wiring(toolbox.clone(), None); + let effects = EffectsStub::default(); + let outcomes = crate::dialog_turn::DialogTurnObserver::new( + crate::dialog_turn::RuntimeTurnOutcomeBuffer::new(8), + None, + ); + let report = process_dialog_job_once_in_queue_with_materializer_history_and_retry_at( + &queue, + &provider, + &effects, + &BasicDialogInputMaterializer, + &NoopDialogToolCallHistoryStore, + session_options(now, &outcomes, &wiring), + ) + .await; + assert!(report.completed, "{report:?}"); + assert!(!report.failed, "{report:?}"); + assert_eq!(report.regenerations, 1); + let requests = provider.requests(); + assert_eq!(requests[0].required_tool.as_deref(), Some("draw_image")); + assert_eq!(requests[1].required_tool.as_deref(), Some("draw_image")); + assert!( + step_context(&requests[1]) + .iter() + .any(|context| context.contains("explicitly requested draw_image")) + ); + assert!(effects.intermediates().is_empty()); + assert!( + effects + .sent() + .iter() + .all(|(_, text)| !text.contains("Сначала я подумаю")) + ); + let draws = toolbox.draw_requests.lock().expect("draw requests"); + assert_eq!(draws.len(), 1); + assert_eq!(draws[0].caption, "Рыба в космосе"); + assert_eq!( + draws[0].context.message_text, + "Плотва, нарисуй рыбу в космосе" + ); + if !queued { + assert_eq!(requests[2].required_tool, None); + assert_eq!(effects.sent()[0].1, "Сервис рисования сейчас недоступен."); + } + } + Ok(()) +} + +#[tokio::test] +async fn explicit_image_request_never_publishes_a_promise_after_exhausted_repairs() +-> Result<(), Box> { + let now = OffsetDateTime::from_unix_timestamp(1_779_193_800)?; + let queue = InMemoryTaskQueue::new(); + let mut params = dialog_params("Плотва, нарисуй рыбу"); + params.meta = serde_json::json!({"requested_tool": "draw_image"}); + queue.assign(DIALOG_AIFARM_QUEUE_NAME, new_dialog_job_at(params, now)); + let provider = + StepProviderStub::with_steps((0..3).map(|_| Ok(step_text("Сейчас нарисую."))).collect()); + let wiring = session_wiring(Arc::new(SessionToolboxStub::default()), None); + let effects = EffectsStub::default(); + let outcomes = crate::dialog_turn::DialogTurnObserver::new( + crate::dialog_turn::RuntimeTurnOutcomeBuffer::new(8), + None, + ); + let report = process_dialog_job_once_in_queue_with_materializer_history_and_retry_at( + &queue, + &provider, + &effects, + &BasicDialogInputMaterializer, + &NoopDialogToolCallHistoryStore, + session_options(now, &outcomes, &wiring), + ) + .await; + assert!(!report.sent_answer, "{report:?}"); + assert!(effects.sent().is_empty()); + assert!(effects.intermediates().is_empty()); + assert_eq!(provider.requests().len(), 3); + assert!( + report + .empty_answer_error + .as_deref() + .is_some_and(|error| error.contains("requested image tool")) + ); + Ok(()) +} + #[tokio::test] async fn session_returns_rejected_duplicate_draft_to_the_agent() -> Result<(), Box> { let now = OffsetDateTime::from_unix_timestamp(1_779_193_800)?; diff --git a/crates/openplotva-app/src/dialog_messages.rs b/crates/openplotva-app/src/dialog_messages.rs index 5786ef92..1098b9ae 100644 --- a/crates/openplotva-app/src/dialog_messages.rs +++ b/crates/openplotva-app/src/dialog_messages.rs @@ -245,8 +245,10 @@ pub struct DirectDrawApiRequest { pub user_id: i64, /// Caller display name. pub user_full_name: String, - /// Raw shortcut prompt after `%`. + /// Technical generation prompt after `%`. pub prompt: String, + /// Readable caption, independent from generation instructions. + pub caption: String, /// Forum topic ID. pub thread_id: Option, /// Whether the target chat is a forum. @@ -574,7 +576,11 @@ where user_full_name: request.user_full_name.clone(), thread_id: request.thread_id, prompt: request.prompt.clone(), - caption_text: request.prompt.clone(), + caption_text: if request.caption.trim().is_empty() { + request.prompt.clone() + } else { + request.caption.clone() + }, ..ImageGenerationRequest::default() }; let result = match self.generator.generate_image(generation).await { @@ -1794,9 +1800,9 @@ where .as_ref() .is_some_and(context_has_editable_image); let intent = if is_direct_draw_api_shortcut(&first_word_lower) { - Some(format!( - "Requested tool: draw_api. Prompt: {}", - parsed.rest_text + Some(( + "draw_api", + format!("Requested tool: draw_api. Prompt: {}", parsed.rest_text), )) } else if let Some(topic) = direct_song_shortcut_topic( message, @@ -1805,9 +1811,12 @@ where &parsed.rest_text, &bot_username, ) { - Some(format!( - "Requested tool: generate_song. Topic: {}", - resolve_song_topic(message, &topic).unwrap_or_default() + Some(( + "generate_song", + format!( + "Requested tool: generate_song. Topic: {}", + resolve_song_topic(message, &topic).unwrap_or_default() + ), )) } else { direct_image_shortcut( @@ -1818,19 +1827,24 @@ where editable, ) .map(|action| match action { - DirectImageShortcut::Draw(prompt) => { - format!("Requested tool: draw_image. Prompt: {prompt}") - } - DirectImageShortcut::Edit(prompt) => format!( - "Requested tool: draw_image with source attachments. Edit: {}", - resolve_direct_image_edit_prompt( - &prompt, - reply_context.as_ref().unwrap_or(&context) - ) + DirectImageShortcut::Draw(prompt) => ( + "draw_image", + format!("Requested tool: draw_image. Prompt: {prompt}"), + ), + DirectImageShortcut::Edit(prompt) => ( + "draw_image", + format!( + "Requested tool: draw_image with source attachments. Edit: {}", + resolve_direct_image_edit_prompt( + &prompt, + reply_context.as_ref().unwrap_or(&context) + ) + ), ), }) }; - if let Some(intent) = intent { + if let Some((tool, intent)) = intent { + context.meta.requested_tool = tool.to_owned(); context.meta.annotation.push_str(&format!("\n{intent}")); } @@ -2048,8 +2062,17 @@ fn direct_draw_api_photo_request( }, disable_notification: false, photo, - caption: String::new(), - render_as: String::new(), + caption: crate::image_jobs::build_image_generation_caption( + if request.caption.trim().is_empty() { + &request.prompt + } else { + &request.caption + }, + &request.user_full_name, + false, + false, + ), + render_as: "HTML".to_owned(), has_spoiler: false, reply_parameters: Some(ReplyParametersPlan { message_id: i64::from(request.message_id), @@ -5063,6 +5086,7 @@ mod tests { user_id: 99, user_full_name: "Ada".to_owned(), prompt: "neon koi".to_owned(), + caption: String::new(), thread_id: Some(9), is_forum: true, }) @@ -5137,6 +5161,7 @@ mod tests { user_id: 99, user_full_name: "Ada".to_owned(), prompt: "neon koi".to_owned(), + caption: String::new(), thread_id: None, is_forum: false, }) @@ -5181,6 +5206,7 @@ mod tests { user_id: 99, user_full_name: "Ada".to_owned(), prompt: "neon koi".to_owned(), + caption: String::new(), thread_id: None, is_forum: false, }) @@ -5352,6 +5378,7 @@ mod tests { user_id: 99, user_full_name: "Ada".to_owned(), prompt: "neon koi".to_owned(), + caption: "Неоновая рыба".to_owned(), thread_id: Some(9), is_forum: true, }; @@ -5364,6 +5391,9 @@ mod tests { }, ) .expect("bytes photo"); + assert!(bytes.caption.contains("Неоновая рыба")); + assert!(!bytes.caption.contains("neon koi")); + assert_eq!(bytes.render_as, "HTML"); assert_eq!( bytes.photo, PhotoSource::Bytes { @@ -6464,6 +6494,9 @@ mod tests { ("@plotva_bot песня про море", "generate_song"), ("!draw red fish", "draw_image"), ("@plotva_bot нарисуй рыбу", "draw_image"), + ("@plotva_bot пожалуйста, нарисуй рыбу", "draw_image"), + ("@plotva_bot можешь нарисовать рыбу", "draw_image"), + ("@plotva_bot сгенерируй картинку рыбы", "draw_image"), ("@plotva_bot % red fish", "draw_api"), ] { let scheduler = SchedulerStub::default(); @@ -6483,6 +6516,7 @@ mod tests { "{text}: {route:?}" ); assert_eq!(scheduler.calls().len(), 1, "{text}"); + assert_eq!(scheduler.metas()[0].requested_tool, tool, "{text}"); assert!( scheduler.metas()[0].annotation.contains(tool), "{text}: {:?}", diff --git a/crates/openplotva-app/src/dialog_runtime.rs b/crates/openplotva-app/src/dialog_runtime.rs index bfb77c52..80929ee8 100644 --- a/crates/openplotva-app/src/dialog_runtime.rs +++ b/crates/openplotva-app/src/dialog_runtime.rs @@ -1584,6 +1584,7 @@ mod tests { .as_chat_step() .expect("step seam") .run_chat_step(openplotva_dialog::ChatStepRequest { + required_tool: None, preferred_target: None, input: live_dialog_smoke_input(), transcript: Vec::new(), @@ -1953,6 +1954,7 @@ mod tests { fn default_step_request() -> openplotva_dialog::ChatStepRequest { openplotva_dialog::ChatStepRequest { + required_tool: None, preferred_target: None, input: DialogInput::default(), transcript: Vec::new(), @@ -2017,6 +2019,7 @@ mod tests { fn step_request() -> ChatStepRequest { ChatStepRequest { + required_tool: None, preferred_target: None, input: DialogInput::default(), transcript: Vec::new(), diff --git a/crates/openplotva-app/src/dialog_tools.rs b/crates/openplotva-app/src/dialog_tools.rs index fbb0bb22..5364d5ac 100644 --- a/crates/openplotva-app/src/dialog_tools.rs +++ b/crates/openplotva-app/src/dialog_tools.rs @@ -691,6 +691,7 @@ pub struct DrawImageScheduleRequest { pub aspect_ratio: String, /// Sanitized seed. pub seed: String, + /// Readable caption; stored independently from the technical generation prompt. pub original_prompt: String, } @@ -1333,7 +1334,11 @@ impl ImageScheduler for TaskmanDialogToolAdapter { user_id: request.user_id, user_full_name: request.user_full_name, prompt: base_prompt.clone(), - original_text: base_prompt, + original_text: if request.original_prompt.trim().is_empty() { + base_prompt + } else { + sanitize_tool_text(&request.original_prompt) + }, meta, prompt_variants: request.prompt_variants, negative_prompt: request.negative_prompt, @@ -2407,7 +2412,9 @@ where Ok(attachments) => attachments, Err(reason) => return Ok(ToolResult::failed("draw_image_source_invalid", reason)), }; - let original_prompt = if attachments.iter().any(|image| image.kind == "image") + let original_prompt = if !req.caption.trim().is_empty() { + sanitize_tool_text(&req.caption) + } else if !req.context.message_meta.agent_gift && !req.context.message_text.trim().is_empty() { sanitize_tool_text(&req.context.message_text) @@ -4300,6 +4307,7 @@ mod tests { ..context() }, prompt: " neon castle ".to_owned(), + caption: String::new(), file_ids: Vec::new(), negative_prompt: " blur ".to_owned(), aspect_ratio: " 16:9 ".to_owned(), @@ -4352,6 +4360,42 @@ mod tests { Ok(()) } + #[tokio::test] + async fn image_caption_uses_the_request_or_readable_caption_without_changing_generation() + -> Result<(), ToolboxError> { + for (caption, gift, expected) in [ + ("", false, "Плотва, нарисуй рыбу в космосе"), + ("Рыба в космосе", false, "Рыба в космосе"), + ("Подарок: рыба в космосе", true, "Подарок: рыба в космосе"), + ] { + let scheduler = Arc::new(ImageSchedulerStub::successful(DrawImageScheduleResult { + status: "scheduled".to_owned(), + ..Default::default() + })); + let tools = toolbox(None).with_image_scheduler(scheduler.clone()); + let result = tools + .draw_image(DrawRequest { + context: ToolContext { + message_text: "Плотва, нарисуй рыбу в космосе".into(), + message_meta: ChatMessageMeta { + agent_gift: gift, + ..Default::default() + }, + ..context() + }, + prompt: "A fish in space, cinematic lighting".into(), + caption: caption.into(), + ..Default::default() + }) + .await?; + assert_eq!(result.status, TOOL_RESULT_STATUS_QUEUED); + let calls = scheduler.calls(); + assert_eq!(calls[0].prompt, "A fish in space, cinematic lighting"); + assert_eq!(calls[0].original_prompt, expected); + } + Ok(()) + } + #[tokio::test] async fn app_dialog_toolbox_combines_selected_history_images_in_one_edit() -> Result<(), ToolboxError> { @@ -5330,6 +5374,7 @@ mod tests { .draw_image(DrawRequest { context: context(), prompt: " neon\u{200f}\tcastle ".to_owned(), + caption: "Нарисуй неоновый замок".to_owned(), file_ids: Vec::new(), negative_prompt: " blur ".to_owned(), aspect_ratio: " 16:9 ".to_owned(), @@ -5363,7 +5408,7 @@ mod tests { .as_ref() .expect("image job should carry image data"); assert_eq!(image.prompt, "neon castle"); - assert_eq!(image.original_text, "neon castle"); + assert_eq!(image.original_text, "Нарисуй неоновый замок"); assert_eq!(image.author, "Alice"); assert_eq!(image.raw_negative_prompt, "blur"); assert_eq!(image.raw_aspect_ratio, "16:9"); diff --git a/crates/openplotva-app/src/dialog_turn/session.rs b/crates/openplotva-app/src/dialog_turn/session.rs index 6012d13c..49801292 100644 --- a/crates/openplotva-app/src/dialog_turn/session.rs +++ b/crates/openplotva-app/src/dialog_turn/session.rs @@ -303,6 +303,13 @@ fn session_native_tools(allow_finish: bool) -> Result, String> { .collect() } +fn pending_image_tool<'a>(context: &'a ToolContext, calls: &[ToolCall]) -> Option<&'a str> { + let tool = context.message_meta.requested_tool.as_str(); + matches!(tool, "draw_image" | "draw_api") + .then_some(tool) + .filter(|name| !calls.iter().any(|call| call.name == *name)) +} + fn record_rejected_final(agent: &mut openplotva_agent::AgentLoop, text: &str, reason: &str) { agent.transcript.push(SessionMessage::Assistant { text: text.to_owned(), @@ -565,6 +572,9 @@ where extend_context_images(&mut tool_context, params.message_id, meta.attachments); } if params.message_id <= agent.observed_message_id { continue; } + if params.user_id == active_params.user_id { + tool_context.message_meta.requested_tool.clear(); + } agent.observed_message_id = params.message_id; links.record_tool_result(&ToolResult {status:"ok".into(), message:params.message_text.clone(), ..ToolResult::default()}); agent.transcript.push(SessionMessage::InjectedUser { @@ -600,7 +610,8 @@ where ToolsMode::Native((*native_tools).clone()) }; - let input = (*base_input).clone(); + let mut input = (*base_input).clone(); + input.message.meta.requested_tool = tool_context.message_meta.requested_tool.clone(); let mut add_hint = |hint: &str| { let rendered = format!("Runtime guidance: {hint}"); if !agent.transcript.iter().any(|message| matches!(message, SessionMessage::InjectedUser { rendered: prior } if prior == &rendered)) { @@ -619,6 +630,7 @@ where Some(provider_deadline), step_provider.run_chat_step(ChatStepRequest { input, + required_tool: pending_image_tool(&tool_context, &recorded_tool_calls).map(str::to_owned), preferred_target: agent.preferred_target.clone(), transcript: agent.transcript.clone(), tools, @@ -711,7 +723,11 @@ where // stands on its own. let raw_answer = step.text.clone(); let sanitized = links.prepare_response(&raw_answer); - if sanitized.trim().is_empty() { + let image_action_missing = !base_input.disable_tools + && pending_image_tool(&tool_context, &recorded_tool_calls).is_some(); + if sanitized.trim().is_empty() || (image_action_missing + && (force_final || regenerations >= ctx.max_regenerations.max(0) + || budget.remaining(failure_now) < MIN_REGENERATION_BUDGET)) { if !side_effect_tickets.is_empty() { // Silent side-effect finish (should have terminated at // the tool batch already; kept as a safety net). @@ -730,7 +746,12 @@ where disposition: JobDisposition::Fail(error), }); } - let (codes, error) = if raw_answer.trim().is_empty() { + let (codes, error) = if image_action_missing { + ( + SANITIZED_EMPTY_RETRY_CODES, + "dialog provider did not execute the explicitly requested image tool", + ) + } else if raw_answer.trim().is_empty() { ( PROVIDER_EMPTY_RETRY_CODES, "dialog provider returned no answer, response, or queued tool material", @@ -762,6 +783,18 @@ where .await); } + if !base_input.disable_tools && !force_final + && let Some(tool) = pending_image_tool(&tool_context, &recorded_tool_calls) + && regenerations < ctx.max_regenerations.max(0) + && budget.remaining(failure_now) >= MIN_REGENERATION_BUDGET + { + let reason = format!("The user explicitly requested {tool}, but it was not called. Execute the image tool with the user's request; do not replace the action with text."); + regenerations += 1; + report.regenerations = regenerations; + record_rejected_final(&mut agent, &sanitized, &reason); + return next_step!(); + } + if !web_source_urls.is_empty() && !answer_cites_web_source(&sanitized, &web_source_urls) { if search_citation_repairs < MAX_SEARCH_CITATION_REPAIRS diff --git a/crates/openplotva-core/src/lib.rs b/crates/openplotva-core/src/lib.rs index 0df8a545..de4bb10d 100644 --- a/crates/openplotva-core/src/lib.rs +++ b/crates/openplotva-core/src/lib.rs @@ -244,6 +244,9 @@ pub struct ChatMessageMeta { #[serde(default, rename = "type", skip_serializing_if = "String::is_empty")] pub message_type: String, + /// Explicit action recognized by the trusted message router. + #[serde(default, skip_serializing_if = "String::is_empty")] + pub requested_tool: String, /// Optional annotation. #[serde(default, skip_serializing_if = "String::is_empty")] pub annotation: String, diff --git a/crates/openplotva-dialog/src/dispatch.rs b/crates/openplotva-dialog/src/dispatch.rs index 08d3617b..4c8a56b3 100644 --- a/crates/openplotva-dialog/src/dispatch.rs +++ b/crates/openplotva-dialog/src/dispatch.rs @@ -38,6 +38,7 @@ pub async fn dispatch_dialog_tool( .draw_image(DrawRequest { context: meta.clone(), prompt: step.prompt.clone(), + caption: step.caption.clone(), file_ids: step.file_ids.clone(), negative_prompt: step.negative_prompt.clone(), aspect_ratio: step.aspect_ratio.clone(), diff --git a/crates/openplotva-dialog/src/lib.rs b/crates/openplotva-dialog/src/lib.rs index ff8975c5..5bc4b1b3 100644 --- a/crates/openplotva-dialog/src/lib.rs +++ b/crates/openplotva-dialog/src/lib.rs @@ -229,9 +229,12 @@ fn is_false(value: &bool) -> bool { pub struct DrawRequest { /// Tool context. pub context: ToolContext, - /// Image prompt. + /// Technical image prompt; never used as the caption when a source request is available. #[serde(default, skip_serializing_if = "String::is_empty")] pub prompt: String, + /// Optional readable caption in the user's language, separate from generation instructions. + #[serde(default, skip_serializing_if = "String::is_empty")] + pub caption: String, /// Stable image IDs selected from the current dialog context. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub file_ids: Vec, @@ -463,6 +466,7 @@ const INLINE_TOOL_ARG_KEYS: &[&str] = &[ "final_reply", "gift", "prompt", + "caption", "topic", "file_id", "file_ids", @@ -559,7 +563,12 @@ const DRAW_IMAGE_ARGS: &[ToolArgSpec] = &[ ToolArgSpec { name: "prompt", required: true, - description: "Image prompt. Prefer concrete visual instructions.", + description: "Technical visual instructions for generation, separate from the visible caption. Keep the user's requested content.", + }, + ToolArgSpec { + name: "caption", + required: false, + description: "Optional readable caption in the user's language. Use their original request verbatim or a concise faithful adaptation. Omit to show the source message. Never put the technical or optimized generation prompt here.", }, ToolArgSpec { name: "file_ids", @@ -703,11 +712,18 @@ const ALTERNATIVE_DIALOG_TOOL_CATALOG: &[ToolSpec] = &[ when_to_use: "Use for an explicit % image shortcut; draw_image handles ordinary generation and edits.", result: "Returns delivery status. Do not announce success before the result.", continuation: ToolContinuation::RequiresFollowup, - args: &[ToolArgSpec { - name: "prompt", - required: true, - description: "Image prompt after the % shortcut.", - }], + args: &[ + ToolArgSpec { + name: "prompt", + required: true, + description: "Technical generation instructions after the % shortcut.", + }, + ToolArgSpec { + name: "caption", + required: false, + description: "Readable caption: the user's original request or a faithful adaptation in their language. Omit to show the source message; never expose the technical generation prompt.", + }, + ], }, ToolSpec { name: "get_messages", @@ -2401,9 +2417,12 @@ pub struct ToolStep { /// Tool name. #[serde(default, rename = "step")] pub step: String, - /// Image prompt. + /// Technical image prompt; never used as the caption when a source request is available. #[serde(default, skip_serializing_if = "String::is_empty")] pub prompt: String, + /// Optional readable caption in the user's language, separate from generation instructions. + #[serde(default, skip_serializing_if = "String::is_empty")] + pub caption: String, /// Song topic. #[serde(default, skip_serializing_if = "String::is_empty")] pub topic: String, diff --git a/crates/openplotva-dialog/src/transcript.rs b/crates/openplotva-dialog/src/transcript.rs index 6a783ba0..9643a539 100644 --- a/crates/openplotva-dialog/src/transcript.rs +++ b/crates/openplotva-dialog/src/transcript.rs @@ -22,7 +22,11 @@ pub fn dialog_tool_context(input: &DialogInput) -> ToolContext { message_id: input.message.id, user_id: input.user.id, user_full_name: input.user.full_name.clone(), - message_text: input.message.text.clone(), + message_text: if input.message.original_text.trim().is_empty() { + input.message.text.clone() + } else { + input.message.original_text.clone() + }, message_meta: input.message.meta.clone(), image_attachments: dialog_image_attachments(input), image_reference_ids: dialog_image_reference_ids(input), @@ -158,6 +162,8 @@ pub struct ChatStepRequest { /// In-session messages so far; empty on the first iteration. pub transcript: Vec, pub tools: ToolsMode, + /// Require this native function for an explicit action that has not been attempted. + pub required_tool: Option, /// Continue with this provider/model pair when it remains eligible and live. pub preferred_target: Option<(String, String)>, /// 1-based session iteration, used for trace/telemetry tagging. diff --git a/crates/openplotva-llm/src/aifarm.rs b/crates/openplotva-llm/src/aifarm.rs index f08b055e..67d55be8 100644 --- a/crates/openplotva-llm/src/aifarm.rs +++ b/crates/openplotva-llm/src/aifarm.rs @@ -2968,7 +2968,7 @@ where ) -> Result { let history = build_session_history_with_limit(&request.input, self.cfg.max_history); let iteration = request.iteration.max(1); - let completion_request = self + let mut completion_request = self .step_request_with_history( &request.input, &history, @@ -2977,6 +2977,17 @@ where iteration, ) .map_err(|error| Box::new(error) as CompletionError)?; + if let Some(name) = request.required_tool.as_deref() + && !request.input.disable_tools + && matches!(request.tools, ToolsMode::Native(_)) + && completion_request + .tools + .iter() + .any(|tool| tool["function"]["name"].as_str() == Some(name)) + { + completion_request.tool_choice = + Some(json!({"type":"function", "function":{"name":name}})); + } let model = completion_request.model.clone(); let guard = reply_leak_guard(&completion_request.messages, &request.input); let traced = self @@ -5800,6 +5811,17 @@ fn saturating_i32(value: u128) -> i32 { pub fn build_runtime_context(input: &DialogInput) -> String { let mut out = String::new(); out.push_str("\n"); + let requested_tool = input.message.meta.requested_tool.as_str(); + if !input.disable_tools && matches!(requested_tool, "draw_image" | "draw_api" | "generate_song") + { + write_text_element( + &mut out, + "explicit_tool_request", + &format!( + "The current user explicitly requested {requested_tool}. Execute this action through the tool; a text description or promise does not fulfill it. A recorded tool result fulfills the request, including an actual failure." + ), + ); + } write_text_element( &mut out, "bot_name", @@ -8607,6 +8629,29 @@ mod tests { Ok(()) } + #[test] + fn explicit_image_intent_is_in_the_first_dynamic_message_not_only_history() + -> Result<(), AifarmMessageError> { + let mut input = base_input(); + input.message.meta.requested_tool = "draw_image".into(); + input.message.text = "Плотва, нарисуй рыбу".into(); + let messages = build_default_initial_messages(&input)?; + assert_eq!(messages[0].role, "system"); + assert!(!messages[0].content.contains("")); + assert_eq!(messages[1].role, "user"); + assert!( + messages[1].content.contains( + "The current user explicitly requested draw_image." + ) + ); + let explicit_system = messages[0].content.clone(); + input.message.meta.requested_tool.clear(); + let ordinary = build_default_initial_messages(&input)?; + assert_eq!(ordinary[0].content, explicit_system); + assert!(!ordinary[1].content.contains("")); + Ok(()) + } + #[test] fn runtime_context_uses_names_custom_persona_and_raw_shield() { let input = DialogInput { @@ -10056,6 +10101,54 @@ mod tests { assert_eq!(history[history.len() - 1].message_id, 101); } + #[tokio::test] + async fn chat_step_forces_only_an_available_native_tool_for_an_explicit_action() + -> Result<(), CompletionError> { + for (required, disabled, final_step, forced) in [ + ("draw_image", false, false, true), + ("draw_image", true, false, false), + ("draw_image", false, true, false), + ("not_advertised", false, false, false), + ] { + let (provider, transport, _) = direct_dialog_provider( + json!({"choices": [{"message": {"role": "assistant", "content": "answer"}}]}), + AifarmDialogConfig::default(), + ); + let mut input = base_input(); + input.disable_tools = disabled; + crate::ChatStepProvider::run_chat_step( + &provider, + ChatStepRequest { + required_tool: Some(required.into()), + preferred_target: None, + input, + transcript: Vec::new(), + tools: if final_step { + ToolsMode::FinalOnly + } else { + ToolsMode::Native(native_tool_values(&["draw_image"]).expect("draw tool")) + }, + iteration: 2, + }, + ) + .await?; + let requests = transport.requests(); + let body: Value = serde_json::from_slice(&requests[0].body).expect("request JSON"); + assert_eq!( + body["tool_choice"].is_object(), + forced, + "{required}, disabled={disabled}, final={final_step}: {body}" + ); + if forced { + assert_eq!( + body["tool_choice"], + json!({"type": "function", "function": {"name": "draw_image"}}) + ); + } + } + Ok(()) + } + #[tokio::test] async fn chat_step_parses_native_tool_calls_without_publishing_adjacent_text() -> Result<(), CompletionError> { @@ -10079,6 +10172,7 @@ mod tests { AifarmDialogConfig::default(), ); let request = openplotva_dialog::ChatStepRequest { + required_tool: None, preferred_target: None, input: base_input(), transcript: Vec::new(), @@ -10136,6 +10230,7 @@ mod tests { let output = crate::ChatStepProvider::run_chat_step( &provider, openplotva_dialog::ChatStepRequest { + required_tool: None, preferred_target: None, input: base_input(), transcript: Vec::new(), @@ -10183,6 +10278,7 @@ mod tests { let output = crate::ChatStepProvider::run_chat_step( &provider, openplotva_dialog::ChatStepRequest { + required_tool: None, preferred_target: None, input: base_input(), transcript: Vec::new(), @@ -10225,6 +10321,7 @@ mod tests { let error = crate::ChatStepProvider::run_chat_step( &provider, openplotva_dialog::ChatStepRequest { + required_tool: None, preferred_target: None, input: base_input(), transcript: Vec::new(), @@ -10262,6 +10359,7 @@ mod tests { let output = crate::ChatStepProvider::run_chat_step( &provider, openplotva_dialog::ChatStepRequest { + required_tool: None, preferred_target: None, input: base_input(), transcript: Vec::new(), @@ -10306,6 +10404,7 @@ mod tests { crate::ChatStepProvider::run_chat_step( &native_provider, openplotva_dialog::ChatStepRequest { + required_tool: None, preferred_target: None, input: base_input(), transcript: transcript.clone(), @@ -10322,6 +10421,7 @@ mod tests { let output = crate::ChatStepProvider::run_chat_step( &final_provider, openplotva_dialog::ChatStepRequest { + required_tool: None, preferred_target: None, input: base_input(), transcript, @@ -10399,6 +10499,7 @@ mod tests { let error = crate::ChatStepProvider::run_chat_step( &provider, openplotva_dialog::ChatStepRequest { + required_tool: None, preferred_target: None, input: base_input(), transcript: Vec::new(), @@ -10491,6 +10592,7 @@ mod tests { crate::ChatStepProvider::run_chat_step( &provider, openplotva_dialog::ChatStepRequest { + required_tool: None, preferred_target: None, input, transcript: Vec::new(), @@ -10521,6 +10623,7 @@ mod tests { crate::ChatStepProvider::run_chat_step( &provider, openplotva_dialog::ChatStepRequest { + required_tool: None, preferred_target: None, input: base_input(), transcript: Vec::new(), @@ -10557,6 +10660,7 @@ mod tests { let error = crate::ChatStepProvider::run_chat_step( &provider, openplotva_dialog::ChatStepRequest { + required_tool: None, preferred_target: None, input: base_input(), transcript: Vec::new(), diff --git a/crates/openplotva-llm/src/gemini.rs b/crates/openplotva-llm/src/gemini.rs index 48c5b1ec..8232c408 100644 --- a/crates/openplotva-llm/src/gemini.rs +++ b/crates/openplotva-llm/src/gemini.rs @@ -3191,6 +3191,7 @@ mod tests { fn gemini_step_request() -> openplotva_dialog::ChatStepRequest { openplotva_dialog::ChatStepRequest { + required_tool: None, preferred_target: None, input: openplotva_dialog::DialogInput { context: openplotva_dialog::DialogContext { @@ -3511,6 +3512,7 @@ mod tests { let output = provider .run_step(openplotva_dialog::ChatStepRequest { + required_tool: None, preferred_target: None, input, transcript: Vec::new(), diff --git a/crates/openplotva-llm/src/whitecircle.rs b/crates/openplotva-llm/src/whitecircle.rs index c99d5de2..18c7c8d4 100644 --- a/crates/openplotva-llm/src/whitecircle.rs +++ b/crates/openplotva-llm/src/whitecircle.rs @@ -1562,6 +1562,7 @@ mod tests { // first step; the audit must still dispatch for that turn. let first = seam .run_chat_step(ChatStepRequest { + required_tool: None, preferred_target: None, input: step_check_input(), transcript: vec![openplotva_dialog::SessionMessage::InjectedUser { @@ -1577,6 +1578,7 @@ mod tests { let second = seam .run_chat_step(ChatStepRequest { + required_tool: None, preferred_target: None, input: step_check_input(), transcript: vec![openplotva_dialog::SessionMessage::Assistant { diff --git a/crates/openplotva-updates/src/lib.rs b/crates/openplotva-updates/src/lib.rs index a94ab758..fcef1c9f 100644 --- a/crates/openplotva-updates/src/lib.rs +++ b/crates/openplotva-updates/src/lib.rs @@ -1811,9 +1811,25 @@ pub fn resolve_draw_prompt_from_message( first_word_lower: &str, rest_text: &str, ) -> Option { - DRAW_VERB_ALIASES - .contains(&first_word_lower) - .then(|| draw_prompt_with_reply_context(rest_text, Some(message))) + let mut verb = first_word_lower.trim_end_matches(',').to_owned(); + let mut prompt = rest_text.trim().to_owned(); + for _ in 0..2 { + if !matches!(verb.as_str(), "пожалуйста" | "прошу" | "можешь" | "можете") + { + break; + } + (verb, prompt) = cut_first_word(&prompt); + verb = verb.trim_end_matches(',').to_lowercase(); + } + let image_generation = matches!( + verb.as_str(), + "сгенерируй" | "сгенерь" | "создай" | "сделай" | "generate" + ) && matches!( + cut_first_word(&prompt.to_lowercase()).0.as_str(), + "картинку" | "изображение" | "рисунок" | "картину" | "image" | "picture" + ); + (DRAW_VERB_ALIASES.contains(&verb.as_str()) || verb == "нарисовать" || image_generation) + .then(|| draw_prompt_with_reply_context(&prompt, Some(message))) } #[must_use] @@ -3209,7 +3225,14 @@ fn addressed_bot_response_bucket(chat_id: i64, message_id: i64) -> u64 { const BOT_ADDRESSED_RESPONSE_SAMPLING_DIVISOR: u64 = 3; -const DRAW_VERB_ALIASES: &[&str] = &["нарисуй", "draw", "рисуй"]; +const DRAW_VERB_ALIASES: &[&str] = &[ + "нарисуй", + "нарисуйте", + "draw", + "рисуй", + "рисуйте", + "изобрази", +]; const GUEST_DRAW_BANG_ALIASES: &[&str] = &["!рис", "!draw"]; @@ -6376,6 +6399,21 @@ mod tests { resolve_draw_prompt_from_message(&text_reply, "song", "cat"), None ); + for (first, rest) in [ + ("не", "нарисуй рыбу"), + ("расскажи", "как нарисовать рыбу"), + ("можешь", "объяснить рисование"), + ("сгенерируй", "песню про рыбу"), + ] { + assert_eq!( + resolve_draw_prompt_from_message(&text_reply, first, rest), + None + ); + } + assert_eq!( + resolve_draw_prompt_from_message(&text_reply, "пожалуйста,", "можешь Нарисовать рыбу"), + Some("рыбу".into()) + ); Ok(()) } diff --git a/prompts/aifarm/system.prompt b/prompts/aifarm/system.prompt index 33dcf0cf..c1c199ab 100644 --- a/prompts/aifarm/system.prompt +++ b/prompts/aifarm/system.prompt @@ -7,7 +7,9 @@ Подарок draw_image/generate_song с gift=true возможен только при выданном runtime разрешении и по твоей инициативе. Просьбу участника или переделку нельзя выдавать за подарок. Проверяй права через get_user_status; неизвестный статус не означает отсутствие VIP. Отказ объясняй коротко своим голосом, без рекламного хвоста. В случайном включении можно поставить реакцию или вызвать finish_turn без текста. При прямом обращении ответь или уточни. - Команды и разговорные шорткаты задают намерение пользователя: выполни соответствующее действие через инструмент. Не заменяй просьбу нарисовать словесным описанием. + Команды и разговорные шорткаты задают намерение пользователя: выполни соответствующее действие через инструмент. + Прямая просьба нарисовать, сгенерировать, переделать или отредактировать изображение требует вызова draw_image (draw_api для %). Не заменяй действие рассуждением, описанием, обещанием или вопросом «начать?». По известному запросу сразу вызови инструмент; уточняй только недостающую деталь, без которой результат нельзя получить. Обычное рисование не требует предварительного get_user_status: исполнитель сам проверит доступ и лимиты. Действующие ограничения контента и результат инструмента сохраняют силу. + В draw_image.prompt передавай инструкции генератору, в caption — дословный запрос пользователя или его читаемую адаптацию на языке пользователя. Технический и оптимизированный промпт не являются подписью. Если caption не нужен, не передавай его: runtime покажет исходный запрос. {{#ifEquals toolMode "native"}} From b148c9626b18f43722e95c7b6b8ec8ff953430c2 Mon Sep 17 00:00:00 2001 From: Valeriy Selitskiy <239034+iamwavecut@users.noreply.github.com> Date: Sat, 10 Oct 2026 13:24:02 +0200 Subject: [PATCH 2/3] Avoid forcing drawings for standalone infinitive discussion --- crates/openplotva-updates/src/lib.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/crates/openplotva-updates/src/lib.rs b/crates/openplotva-updates/src/lib.rs index fcef1c9f..7b561ea9 100644 --- a/crates/openplotva-updates/src/lib.rs +++ b/crates/openplotva-updates/src/lib.rs @@ -1813,11 +1813,13 @@ pub fn resolve_draw_prompt_from_message( ) -> Option { let mut verb = first_word_lower.trim_end_matches(',').to_owned(); let mut prompt = rest_text.trim().to_owned(); + let mut request_prefix = false; for _ in 0..2 { if !matches!(verb.as_str(), "пожалуйста" | "прошу" | "можешь" | "можете") { break; } + request_prefix = true; (verb, prompt) = cut_first_word(&prompt); verb = verb.trim_end_matches(',').to_lowercase(); } @@ -1828,7 +1830,9 @@ pub fn resolve_draw_prompt_from_message( cut_first_word(&prompt.to_lowercase()).0.as_str(), "картинку" | "изображение" | "рисунок" | "картину" | "image" | "picture" ); - (DRAW_VERB_ALIASES.contains(&verb.as_str()) || verb == "нарисовать" || image_generation) + (DRAW_VERB_ALIASES.contains(&verb.as_str()) + || (request_prefix && verb == "нарисовать") + || image_generation) .then(|| draw_prompt_with_reply_context(&prompt, Some(message))) } @@ -6404,6 +6408,7 @@ mod tests { ("расскажи", "как нарисовать рыбу"), ("можешь", "объяснить рисование"), ("сгенерируй", "песню про рыбу"), + ("нарисовать", "такое сложно?"), ] { assert_eq!( resolve_draw_prompt_from_message(&text_reply, first, rest), From 586ddd9951c2aaacbe419e9007ba7f1d20b1317f Mon Sep 17 00:00:00 2001 From: Valeriy Selitskiy <239034+iamwavecut@users.noreply.github.com> Date: Sat, 10 Oct 2026 13:33:08 +0200 Subject: [PATCH 3/3] Preserve explicit intent for newer initiator commands --- .../openplotva-app/src/dialog_jobs/tests.rs | 125 +++++++++++++++++- .../openplotva-app/src/dialog_turn/session.rs | 42 ++++-- 2 files changed, 156 insertions(+), 11 deletions(-) diff --git a/crates/openplotva-app/src/dialog_jobs/tests.rs b/crates/openplotva-app/src/dialog_jobs/tests.rs index e3e5cebd..8135c79a 100644 --- a/crates/openplotva-app/src/dialog_jobs/tests.rs +++ b/crates/openplotva-app/src/dialog_jobs/tests.rs @@ -3115,6 +3115,7 @@ impl DialogToolCallHistoryStore for ToolHistoryStub { #[derive(Clone, Default)] struct MaterializerStub { history: Vec, + observed: Arc>>, } #[derive(Clone, Copy, Default)] @@ -3161,11 +3162,37 @@ impl DialogInputMaterializer for FailingInjectedMaterializer { impl MaterializerStub { fn with_history(history: Vec) -> Self { - Self { history } + Self { + history, + ..Default::default() + } } } impl DialogInputMaterializer for MaterializerStub { + fn observe_dialog_messages<'a>( + &'a self, + _params: &'a DialogJobParams, + after: i32, + ) -> std::pin::Pin< + Box, String>> + Send + 'a>, + > { + Box::pin(async move { + Ok(self + .observed + .lock() + .expect("observed messages") + .iter() + .filter(|message| { + message["message_id"] + .as_i64() + .is_some_and(|id| id > i64::from(after)) + }) + .cloned() + .collect()) + }) + } + fn materialize_dialog_input<'a>( &'a self, params: &'a DialogJobParams, @@ -6433,6 +6460,102 @@ async fn explicit_image_request_never_publishes_a_promise_after_exhausted_repair Ok(()) } +#[tokio::test] +async fn newer_initiator_image_intent_replaces_the_previous_attempt() -> Result<(), Box> +{ + let now = OffsetDateTime::from_unix_timestamp(1_779_193_800)?; + for observed_first in [false, true] { + for (sender, requested, expected) in [ + (7, "draw_image", Some("draw_image")), + (7, "", None), + (8, "draw_image", None), + ] { + let registry = Arc::new(crate::dialog_turn::DialogSessionRegistry::new()); + let key = crate::dialog_turn::SessionKey::new(42, Some(9)); + let toolbox = Arc::new(SessionToolboxStub::default()); + let materializer = MaterializerStub::default(); + let observed_messages = materializer.observed.clone(); + let wiring = crate::dialog_turn::SessionWorkerWiring { + registry: registry.clone(), + ..session_wiring(toolbox.clone(), None) + }; + let queue = InMemoryTaskQueue::new(); + let mut params = dialog_params("нарисуй рыбу"); + params.meta = serde_json::json!({"requested_tool": "draw_image"}); + let job = queue.assign(DIALOG_AIFARM_QUEUE_NAME, new_dialog_job_at(params, now)); + let draw = |id, prompt: &str| { + step_tools( + "", + vec![( + id, + openplotva_dialog::ToolStep { + step: "draw_image".into(), + prompt: prompt.into(), + ..Default::default() + }, + )], + ) + }; + let provider = StepProviderStub::with_steps(vec![ + Ok(draw("old", "fish")), + Ok(if expected.is_some() { + draw("new", "cat") + } else { + step_text("Приняла новое сообщение.") + }), + Ok(step_text("Рисование сейчас недоступно.")), + ]) + .with_on_call(Box::new(move |index| { + if index != 1 { + return; + } + let mut params = dialog_params("нарисуй кота"); + params.message_id += 1; + params.user_id = sender; + params.meta = serde_json::json!({"requested_tool": requested}); + if observed_first { + observed_messages + .lock() + .expect("observed messages") + .push(serde_json::json!({ + "message_id": params.message_id, "user_id": sender, + "text": params.message_text, "meta": {} + })); + } + assert!(registry.inject(key, job, crate::dialog_turn::InjectedMessage { params })); + })); + let effects = EffectsStub::default(); + let outcomes = crate::dialog_turn::DialogTurnObserver::new( + crate::dialog_turn::RuntimeTurnOutcomeBuffer::new(8), + None, + ); + let report = process_dialog_job_once_in_queue_with_materializer_history_and_retry_at( + &queue, + &provider, + &effects, + &materializer, + &NoopDialogToolCallHistoryStore, + session_options(now, &outcomes, &wiring), + ) + .await; + assert!(report.completed, "{report:?}"); + let requests = provider.requests(); + assert_eq!(requests[0].required_tool.as_deref(), Some("draw_image")); + assert_eq!( + requests[1].required_tool.as_deref(), + expected, + "sender={sender}, requested={requested}" + ); + if expected.is_some() { + let draws = toolbox.draw_requests.lock().expect("draws"); + assert_eq!(draws.len(), 2); + assert_eq!(draws[1].context.message_text, "нарисуй кота"); + } + } + } + Ok(()) +} + #[tokio::test] async fn session_returns_rejected_duplicate_draft_to_the_agent() -> Result<(), Box> { let now = OffsetDateTime::from_unix_timestamp(1_779_193_800)?; diff --git a/crates/openplotva-app/src/dialog_turn/session.rs b/crates/openplotva-app/src/dialog_turn/session.rs index 49801292..675ec075 100644 --- a/crates/openplotva-app/src/dialog_turn/session.rs +++ b/crates/openplotva-app/src/dialog_turn/session.rs @@ -303,11 +303,20 @@ fn session_native_tools(allow_finish: bool) -> Result, String> { .collect() } -fn pending_image_tool<'a>(context: &'a ToolContext, calls: &[ToolCall]) -> Option<&'a str> { +fn pending_image_tool<'a>( + context: &'a ToolContext, + calls: &[ToolCall], + call_offset: usize, +) -> Option<&'a str> { let tool = context.message_meta.requested_tool.as_str(); matches!(tool, "draw_image" | "draw_api") .then_some(tool) - .filter(|name| !calls.iter().any(|call| call.name == *name)) + .filter(|name| { + !calls + .iter() + .skip(call_offset) + .any(|call| call.name == *name) + }) } fn record_rejected_final(agent: &mut openplotva_agent::AgentLoop, text: &str, reason: &str) { @@ -341,6 +350,8 @@ pub(crate) struct SessionRunContext<'a> { #[derive(serde::Serialize, serde::Deserialize)] struct SessionState { + #[serde(default)] + requested_tool_call_offset: usize, tool_context: ToolContext, gift_used: bool, budget: SessionBudget, @@ -475,6 +486,7 @@ where let max_iterations = cfg.max_iterations.max(1); let mut state = restored.unwrap_or(SessionState { + requested_tool_call_offset: 0, tool_context: meta, gift_used: false, budget, @@ -549,15 +561,22 @@ where if let Err(error) = persist_agent_event(queue, effects, ctx.item_id, "agent_checkpoint", BTreeMap::from([("checkpoint".into(), serde_json::to_string(&(base_input, &state)).expect("serializable agent state"))]), ctx.now).await { return std::ops::ControlFlow::Break(agent_persistence_failed(error)); } - let SessionState { mut tool_context, mut gift_used, mut budget, mut agent, mut sent, mut side_effect_tickets, mut recorded_tool_calls, mut regenerations, mut anti_loop, mut repeated_final_repair, mut requires_novel_final, mut draws_scheduled, mut songs_scheduled, mut reacted_message_ids, mut tool_result_cache, mut media_reference_aliases, mut successful_web_search, mut web_source_urls, mut links, mut search_citation_repairs } = state; - macro_rules! next_step { () => { std::ops::ControlFlow::Continue((SessionState { tool_context, gift_used, budget, agent, sent, side_effect_tickets, recorded_tool_calls, regenerations, anti_loop, repeated_final_repair, requires_novel_final, draws_scheduled, songs_scheduled, reacted_message_ids, tool_result_cache, media_reference_aliases, successful_web_search, web_source_urls, links, search_citation_repairs }, report)) }; } + let SessionState { mut requested_tool_call_offset, mut tool_context, mut gift_used, mut budget, mut agent, mut sent, mut side_effect_tickets, mut recorded_tool_calls, mut regenerations, mut anti_loop, mut repeated_final_repair, mut requires_novel_final, mut draws_scheduled, mut songs_scheduled, mut reacted_message_ids, mut tool_result_cache, mut media_reference_aliases, mut successful_web_search, mut web_source_urls, mut links, mut search_citation_repairs } = state; + macro_rules! next_step { () => { std::ops::ControlFlow::Continue((SessionState { requested_tool_call_offset, tool_context, gift_used, budget, agent, sent, side_effect_tickets, recorded_tool_calls, regenerations, anti_loop, repeated_final_repair, requires_novel_final, draws_scheduled, songs_scheduled, reacted_message_ids, tool_result_cache, media_reference_aliases, successful_web_search, web_source_urls, links, search_citation_repairs }, report)) }; } + let previously_observed_message_id = agent.observed_message_id; if let Ok(Ok(messages)) = tokio::time::timeout(std::time::Duration::from_secs(2), materializer.observe_dialog_messages(active_params, agent.observed_message_id)).await { for mut message in messages { if let Some(id) = message.get("message_id").and_then(Value::as_i64).and_then(|id| i32::try_from(id).ok()) { agent.observed_message_id = agent.observed_message_id.max(id); } message["can_change_task"] = Value::Bool(message.get("user_id").and_then(Value::as_i64) == Some(active_params.user_id)); + if message["can_change_task"] == true { + tool_context.message_id = message["message_id"].as_i64().and_then(|id| i32::try_from(id).ok()).unwrap_or(tool_context.message_id); + tool_context.message_meta.requested_tool = message["meta"]["requested_tool"].as_str().unwrap_or_default().to_owned(); + requested_tool_call_offset = recorded_tool_calls.len(); + if let Some(text) = message["text"].as_str() { tool_context.message_text = text.to_owned(); } + } remember_context_images(&mut tool_context, &message); links.record_tool_result(&ToolResult {status:"ok".into(), data:Some(message.clone()), ..ToolResult::default()}); agent.transcript.push(SessionMessage::InjectedUser { rendered:message.to_string() }); @@ -571,10 +590,13 @@ where if let Ok(meta) = serde_json::from_value::(params.meta.clone()) { extend_context_images(&mut tool_context, params.message_id, meta.attachments); } - if params.message_id <= agent.observed_message_id { continue; } - if params.user_id == active_params.user_id { - tool_context.message_meta.requested_tool.clear(); + if params.message_id > previously_observed_message_id && params.message_id >= tool_context.message_id && params.user_id == active_params.user_id { + tool_context.message_id = params.message_id; + tool_context.message_meta.requested_tool = params.meta["requested_tool"].as_str().unwrap_or_default().to_owned(); + requested_tool_call_offset = recorded_tool_calls.len(); + tool_context.message_text = if params.original_text.trim().is_empty() { params.message_text.clone() } else { params.original_text.clone() }; } + if params.message_id <= agent.observed_message_id { continue; } agent.observed_message_id = params.message_id; links.record_tool_result(&ToolResult {status:"ok".into(), message:params.message_text.clone(), ..ToolResult::default()}); agent.transcript.push(SessionMessage::InjectedUser { @@ -630,7 +652,7 @@ where Some(provider_deadline), step_provider.run_chat_step(ChatStepRequest { input, - required_tool: pending_image_tool(&tool_context, &recorded_tool_calls).map(str::to_owned), + required_tool: pending_image_tool(&tool_context, &recorded_tool_calls, requested_tool_call_offset).map(str::to_owned), preferred_target: agent.preferred_target.clone(), transcript: agent.transcript.clone(), tools, @@ -724,7 +746,7 @@ where let raw_answer = step.text.clone(); let sanitized = links.prepare_response(&raw_answer); let image_action_missing = !base_input.disable_tools - && pending_image_tool(&tool_context, &recorded_tool_calls).is_some(); + && pending_image_tool(&tool_context, &recorded_tool_calls, requested_tool_call_offset).is_some(); if sanitized.trim().is_empty() || (image_action_missing && (force_final || regenerations >= ctx.max_regenerations.max(0) || budget.remaining(failure_now) < MIN_REGENERATION_BUDGET)) { @@ -784,7 +806,7 @@ where } if !base_input.disable_tools && !force_final - && let Some(tool) = pending_image_tool(&tool_context, &recorded_tool_calls) + && let Some(tool) = pending_image_tool(&tool_context, &recorded_tool_calls, requested_tool_call_offset) && regenerations < ctx.max_regenerations.max(0) && budget.remaining(failure_now) >= MIN_REGENERATION_BUDGET {