Track Eredu main 465cf519 (planned-session API port)

Cargo.toml

@@ -18,9 +18,9 @@ serde = { version = "1", features = ["derive"] }
 serde_json = "1"
 tokio = { version = "1", features = ["full"] }
 mesh-llm-plugin = { git = "https://github.com/Mesh-LLM/mesh-llm.git", rev = "bc215ce5b880e21c6c3840b8574ab64f83278821" }
-eredu = { git = "https://github.com/jbg/eredu", rev = "d5683bb3a454a627b1023713a053d0a0d5a3c74a", default-features = false, optional = true }
-eredu-core = { git = "https://github.com/jbg/eredu", rev = "d5683bb3a454a627b1023713a053d0a0d5a3c74a", optional = true }
-eredu-backend-mlx = { git = "https://github.com/jbg/eredu", rev = "d5683bb3a454a627b1023713a053d0a0d5a3c74a", default-features = false, optional = true }
+eredu = { git = "https://github.com/jbg/eredu", rev = "465cf519fc5e0f90a40072320aa1d546103fbcc8", default-features = false, optional = true }
+eredu-core = { git = "https://github.com/jbg/eredu", rev = "465cf519fc5e0f90a40072320aa1d546103fbcc8", optional = true }
+eredu-backend-mlx = { git = "https://github.com/jbg/eredu", rev = "465cf519fc5e0f90a40072320aa1d546103fbcc8", default-features = false, optional = true }
 
 [lints.rust]
 unsafe_code = "forbid"

README.md

@@ -9,7 +9,11 @@ checkpoint-native function calls through OpenAI JSON/SSE. Tools require an Eredu
 registered native-tool template; unsupported templates fail closed.
 
 Tool requests support `tools`, `tool_choice` (`auto`, `none`, `required`),
-`parallel_tool_calls` (default false), and `enable_thinking`. Named-function
+`parallel_tool_calls` (default false), and thinking control through
+`enable_thinking`, `reasoning_effort` (`none` disables thinking) or
+`chat_template_kwargs.enable_thinking` — the form Mesh forwards, so these knobs
+work through Mesh too. Other `chat_template_kwargs` reach the checkpoint template
+unchanged. Named-function
 `tool_choice` objects are not supported and are rejected, not silently ignored.
 The plugin returns calls for your client to execute; it never executes tools.
 Replay the assistant message and a `role: "tool"` message with its `tool_call_id`

src/main.rs

@@ -41,9 +41,9 @@ fn check_model(path: PathBuf) -> Result<()> {
 
     let owner = std::thread::spawn(move || -> Result<()> {
         let plan = ExecutionPlan::fully_resident(local_device_plan(default_local_device())?);
-        let planned =
+        let mut planned =
             LoadedModel::load_execution_plan(&MlxBackendFactory::default(), &path, &plan)?;
-        let (mut model, _) = planned.into_parts();
+        let model = planned.model_mut();
         model.reset()?;
         model.synchronize()?;
         println!("native load succeeded: {}", path.display());

src/native.rs

@@ -6,8 +6,8 @@ use crate::{
 use anyhow::{Result, bail};
 use eredu::{
     api::{
-        LoadedModel, PreparedChatGenerationRequest, PreparedChatGenerationSettings,
-        PreparedChatInput, default_local_device, local_device_plan,
+        GenerationRequest, LoadedModel, NoDrafting, OutputMode, PlannedModel,
+        PreparedChatGenerationSettings, PreparedChatInput, default_local_device, local_device_plan,
     },
     runtime::chat::{ChatTemplateRequest, NativeToolSupport, ParallelToolCallPolicy, ToolChoice},
 };
@@ -20,14 +20,16 @@ use eredu_core::{
 use std::path::PathBuf;
 use tokio::sync::mpsc;
 
-pub struct Native(LoadedModel<<MlxBackendFactory as ExecutionPlanBackendFactory>::Backend>);
+type MlxBackend = <MlxBackendFactory as ExecutionPlanBackendFactory>::Backend;
+
+/// Target-only generation owner: the plugin never selects drafting resources.
+pub struct Native(PlannedModel<MlxBackend, NoDrafting>);
 
 impl Native {
     pub fn load(path: PathBuf) -> Result<Self> {
         let plan = ExecutionPlan::fully_resident(local_device_plan(default_local_device())?);
         let planned = LoadedModel::load_execution_plan(&MlxBackendFactory::default(), path, &plan)?;
-        let (model, _) = planned.into_parts();
-        Ok(Self(model))
+        Ok(Self(planned.into_target_only()))
     }
 }
 
@@ -43,13 +45,13 @@ impl Backend for Native {
                 .messages
                 .iter()
                 .any(|m| m.get("tool_calls").is_some() || m["role"] == "tool");
-        self.0.reset()?;
+        self.0.model_mut().reset()?;
         let tool_choice = match request.tool_choice.as_deref() {
             Some("none") => ToolChoice::None,
             Some("required") => ToolChoice::Required,
             _ => ToolChoice::Auto,
         };
-        let prepared = self.0.prepare_chat(ChatTemplateRequest {
+        let prepared = self.0.model_mut().prepare_chat(ChatTemplateRequest {
             messages: request.messages,
             tools: request.tools,
             tool_choice,
@@ -59,72 +61,74 @@ impl Backend for Native {
                 ParallelToolCallPolicy::Disabled
             },
             enable_thinking: request.enable_thinking,
+            extra_template_kwargs: request.chat_template_kwargs,
             add_generation_prompt: true,
             ..Default::default()
         })?;
         if needs_tools
-            && let NativeToolSupport::Unsupported { reason } = prepared.native_tool_support()
+            && let NativeToolSupport::Unsupported { rejection } = prepared.native_tool_support()
         {
-            bail!("native tools unavailable for this template: {reason}");
+            bail!("native tools unavailable for this template: {rejection}");
         }
         let mut started_calls = 0usize;
         let mut ended_calls = 0usize;
         let cancellation = GenerationCancellationToken::new();
         let mut delivery_failed = false;
-        let result = self
-            .0
-            .generate_prepared_chat(PreparedChatGenerationRequest {
-                input: PreparedChatInput::rendered_prompt(&prepared),
-                settings: PreparedChatGenerationSettings {
-                    overrides: GenerationConfigOverrides {
-                        max_new_tokens: Some(request.max_tokens.unwrap_or(256)),
-                        temperature: request.temperature,
-                        do_sample: request.temperature.map(|t| t > 0.0),
-                        ..Default::default()
-                    },
+        let result = self.0.generate(GenerationRequest {
+            input: PreparedChatInput::rendered_prompt(&prepared),
+            output_mode: OutputMode::Semantic,
+            control_handle: None,
+            settings: PreparedChatGenerationSettings {
+                overrides: GenerationConfigOverrides {
+                    max_new_tokens: Some(request.max_tokens.unwrap_or(256)),
+                    temperature: request.temperature,
+                    do_sample: request.temperature.map(|t| t > 0.0),
                     ..Default::default()
                 },
-                caller_stop_sequences: &[],
-                cancellation: cancellation.clone(),
-                on_event: |event| {
-                    if work.is_cancelled() {
-                        cancellation.cancel();
-                        return;
+                ..Default::default()
+            },
+            caller_stop_sequences: &[],
+            cancellation: cancellation.clone(),
+            on_event: |event| {
+                if work.is_cancelled() {
+                    cancellation.cancel();
+                    return;
+                }
+                let event = match event {
+                    SemanticEvent::TextDelta(text) => Some(Event::Text(text)),
+                    SemanticEvent::ToolCallStart { index, id, name } => {
+                        started_calls += 1;
+                        Some(Event::ToolStart { index, id, name })
                     }
-                    let event = match event {
-                        SemanticEvent::TextDelta(text) => Some(Event::Text(text)),
-                        SemanticEvent::ToolCallStart { index, id, name } => {
-                            started_calls += 1;
-                            Some(Event::ToolStart { index, id, name })
-                        }
-                        SemanticEvent::ToolArgumentsDelta {
-                            index,
-                            json_fragment,
-                        } => Some(Event::ToolArguments {
-                            index,
-                            fragment: json_fragment,
-                        }),
-                        SemanticEvent::ToolCallEnd => {
-                            ended_calls += 1;
-                            None
-                        }
-                        _ => None,
-                    };
-                    if let Some(event) = event
-                        && publish(output, event).is_err()
-                    {
-                        delivery_failed = true;
-                        cancellation.cancel();
+                    SemanticEvent::ToolArgumentsDelta {
+                        index,
+                        json_fragment,
+                    } => Some(Event::ToolArguments {
+                        index,
+                        fragment: json_fragment,
+                    }),
+                    // Only a schema-validated, complete call closes a tool call.
+                    SemanticEvent::ToolCallCompleted { .. } => {
+                        ended_calls += 1;
+                        None
                     }
-                },
-            })?;
+                    _ => None,
+                };
+                if let Some(event) = event
+                    && publish(output, event).is_err()
+                {
+                    delivery_failed = true;
+                    cancellation.cancel();
+                }
+            },
+        })?;
         if delivery_failed || work.is_cancelled() {
             bail!("generation cancelled by consumer");
         }
         if started_calls != ended_calls {
             bail!("generation ended with incomplete tool calls");
         }
-        let reason = match result.finish_reason {
+        let reason = match result.finish_reason() {
             FinishReason::MaxTokens => "length",
             FinishReason::Cancelled => bail!("generation cancelled"),
             _ if started_calls > 0 => "tool_calls",
@@ -134,8 +138,9 @@ impl Backend for Native {
     }
 
     fn settle(&mut self) -> Result<()> {
-        self.0.synchronize()?;
-        self.0.reset()?;
+        let model = self.0.model_mut();
+        model.synchronize()?;
+        model.reset()?;
         Ok(())
     }
 }