From 45d14b1e1d2fa77ca34de6c299f03679abb56930 Mon Sep 17 00:00:00 2001 From: Null <1708213363@qq.com> Date: Tue, 6 Oct 2026 11:24:20 +0800 Subject: [PATCH 001/121] fix(cognitive): retry display name sync after index failures --- docs/cognitive-memory.md | 2 + src/Undefined/cognitive/service/service.py | 13 ++-- tests/test_cognitive_service.py | 89 ++++++++++++++++++++++ 3 files changed, 98 insertions(+), 6 deletions(-) diff --git a/docs/cognitive-memory.md b/docs/cognitive-memory.md index 73cb01a3..c1e87df1 100644 --- a/docs/cognitive-memory.md +++ b/docs/cognitive-memory.md @@ -98,6 +98,8 @@ end → pending(持久化入队顺序) 等待顺位的任务留在磁盘队列,不创建等待协程、不占执行名额。群侧写与发送者侧写分别调度,某个用户目标被阻塞时,同一任务已经轮到的群目标可以先执行。不同实体的 LLM 合并可以同时进行;Chroma 的实际数据库访问仍使用下文的共享单 worker 调度。昵称刷新和侧写恢复复用同一实体锁,避免基于旧副本覆盖正文。 +昵称刷新先更新侧写向量,再提交 Markdown 文件中的新名称。向量暂时不可写时,文件仍保留旧名称,后续同名消息可继续重试,重启后也不会因名称已更新而跳过。文件写入失败时向量可能已更新,但下一次刷新仍按实际文件重试;名称相同且无需刷新时不重复写入。 + **进度与恢复**:任务保存 `_enqueue_order` 以及 `_historian_progress`(已改写文本、已入库事件数、已完成侧写目标)。自动重试只处理未完成部分,阶段切换不增加 `_retry_count`。文件、向量和目标进度均提交后才放行下一个同实体更新;明确 `skip=true` 也会完成目标。没有有效更新或明确跳过的模型结束按失败处理。 失败不会自动回滚已经写入的侧写文件。向量写入或进度保存失败时,重试会重新读取当前实际文件;重试耗尽后,后续任务同样基于当前实际版本继续。正常停止等待在途阶段结束;外层取消或停止超时触发取消时,事件改写和侧写合并的模型调用可立即中断,只等待已经开始的文件、向量、对应进度及队列状态提交收敛,再释放实体占用。取消后尚未完成队列切换的任务保留 `processing` 与已保存进度,后续恢复时不重做已提交部分。启动及轮询恢复超过 `stale_job_timeout_seconds` 的遗留 `processing`,排除活跃任务;未达到恢复阈值的遗留任务仍占其未完成实体的顺位,不阻塞其他实体。 diff --git a/src/Undefined/cognitive/service/service.py b/src/Undefined/cognitive/service/service.py index 841a65c9..82f55335 100644 --- a/src/Undefined/cognitive/service/service.py +++ b/src/Undefined/cognitive/service/service.py @@ -144,12 +144,6 @@ async def _sync() -> bool: updated_markdown = _serialize_profile_markdown( frontmatter, summary, evaluation=evaluation, roast=roast ) - await self._profile_storage.write_profile( - normalized_entity_type, - normalized_entity_id, - updated_markdown, - ) - profile_doc, profile_metadata = _build_profile_vector_payload( entity_type=normalized_entity_type, entity_id=normalized_entity_id, @@ -166,6 +160,13 @@ async def _sync() -> bool: profile_metadata, priority=CHROMA_PRIORITY_FOREGROUND, ) + # 文件中的名称决定是否需要重试;索引成功后再提交名称,避免 + # 一次向量写入失败后,被「名称未变化」永久跳过(重启后也一样)。 + await self._profile_storage.write_profile( + normalized_entity_type, + normalized_entity_id, + updated_markdown, + ) logger.info( "[认知服务] 已刷新侧写展示名: entity_type=%s entity_id=%s old=%s new=%s", normalized_entity_type, diff --git a/tests/test_cognitive_service.py b/tests/test_cognitive_service.py index 0a8e7cfb..59f9cf53 100644 --- a/tests/test_cognitive_service.py +++ b/tests/test_cognitive_service.py @@ -862,6 +862,95 @@ async def test_sync_profile_display_name_noops_when_name_unchanged() -> None: assert vector_store.last_upsert_profile is None +@pytest.mark.asyncio +@pytest.mark.parametrize("entity_type", ["user", "group"]) +async def test_sync_profile_display_name_retries_after_vector_failure( + entity_type: str, +) -> None: + existing_profile = "---\nname: 旧名称\ntags: []\n---\n既有侧写内容\n" + + class FailingOnceVectorStore(_FakeVectorStore): + attempts: int = 0 + + async def upsert_profile( + self, + profile_id: str, + document: str, + metadata: dict[str, Any], + **kwargs: Any, + ) -> None: + self.attempts += 1 + if self.attempts == 1: + raise RuntimeError("temporary vector failure") + await super().upsert_profile(profile_id, document, metadata, **kwargs) + + vectors = FailingOnceVectorStore() + storage = _FakeProfileStorage(existing_profile) + service = CognitiveService( + lambda: SimpleNamespace(enabled=True), vectors, _FakeJobQueue(), storage + ) + with pytest.raises(RuntimeError, match="temporary vector failure"): + await service.sync_profile_display_name( + entity_type=entity_type, entity_id="12345", preferred_name="新名称" + ) + assert storage.profile == existing_profile + assert storage.last_write is None + + # 重建服务模拟重启:重试不能依赖进程内的脏标记。 + service = CognitiveService( + lambda: SimpleNamespace(enabled=True), vectors, _FakeJobQueue(), storage + ) + assert await service.sync_profile_display_name( + entity_type=entity_type, entity_id="12345", preferred_name="新名称" + ) + assert vectors.attempts == 2 + assert vectors.last_upsert_profile is not None + assert vectors.last_upsert_profile[2]["name"] == "新名称" + assert storage.profile is not None + assert "name: 新名称" in storage.profile + assert "既有侧写内容" in storage.profile + assert not await service.sync_profile_display_name( + entity_type=entity_type, entity_id="12345", preferred_name="新名称" + ) + assert vectors.attempts == 2 + + +@pytest.mark.asyncio +async def test_sync_profile_display_name_retries_after_file_failure() -> None: + existing_profile = "---\nname: 旧名称\ntags: []\n---\n既有侧写内容\n" + + class FailingOnceStorage(_FakeProfileStorage): + attempts: int = 0 + + async def write_profile( + self, entity_type: str, entity_id: str, content: str + ) -> None: + self.attempts += 1 + if self.attempts == 1: + raise OSError("temporary file failure") + await super().write_profile(entity_type, entity_id, content) + + vectors = _FakeVectorStore() + storage = FailingOnceStorage(existing_profile) + service = CognitiveService( + lambda: SimpleNamespace(enabled=True), vectors, _FakeJobQueue(), storage + ) + with pytest.raises(OSError, match="temporary file failure"): + await service.sync_profile_display_name( + entity_type="user", entity_id="12345", preferred_name="新名称" + ) + assert storage.profile == existing_profile + assert await service.sync_profile_display_name( + entity_type="user", entity_id="12345", preferred_name="新名称" + ) + assert storage.attempts == 2 + assert storage.profile is not None + assert "name: 新名称" in storage.profile + assert "既有侧写内容" in storage.profile + assert vectors.last_upsert_profile is not None + assert vectors.last_upsert_profile[2]["name"] == "新名称" + + def test_parse_profile_markdown_old_two_part_has_empty_evaluation() -> None: from Undefined.cognitive.service.helpers import _parse_profile_markdown From e4b6d16cc58f0dace25472c47d3a082422267054 Mon Sep 17 00:00:00 2001 From: Null <1708213363@qq.com> Date: Tue, 6 Oct 2026 11:24:58 +0800 Subject: [PATCH 002/121] fix(batching): finish completed speculative replies before new input --- docs/message-batching.md | 1 + .../services/message_batcher/scheduler.py | 17 +++++++--- tests/test_message_batcher.py | 32 +++++++++++++++++++ 3 files changed, 46 insertions(+), 4 deletions(-) diff --git a/docs/message-batching.md b/docs/message-batching.md index 3393e245..77557b0a 100644 --- a/docs/message-batching.md +++ b/docs/message-batching.md @@ -133,6 +133,7 @@ allow_cancel_after_send = false - inflight **尚未发消息** → 调 `inflight_task.cancel()`,桶回到 `TYPING`,新消息追加进去,重置 T1/T2;inflight 协程在 `RequestContext` 里清理后退出,**不写入回复历史**。 - T2 已经把请求入队但 coordinator 还没注册 inflight → 取消旧 `BatchDispatchToken`;旧请求即使稍后被队列取出,也会在 `execute_reply` 入口跳过,新消息继续合并进重新计时的 batch。 - inflight **已经发过消息** 且 `allow_cancel_after_send=False`(默认安全) → 不取消 inflight,**新消息开新 batch**(旧桶在 inflight 自然结束后清理)。 + - 投机调用已经回复并完成 → 立即结束旧 batch;窗口内后续消息也会开启新 batch,不再把已回答的旧消息合并重放。此时任务已经完成,`allow_cancel_after_send` 不改变该行为。尚未回复就完成的投机仍保留原消息,允许后续补充重新合并。 - **FINALIZING**:旧 batch 已到 T1,若此时又来新消息,直接开新桶,不阻塞旧 inflight 收尾。 - `allow_cancel_after_send=True` 会在 inflight 已发过消息后仍取消,可能造成半截回复、重复回复或上下文撕裂,仅极端场景启用。 diff --git a/src/Undefined/services/message_batcher/scheduler.py b/src/Undefined/services/message_batcher/scheduler.py index 45b0cf93..30dcd4a0 100644 --- a/src/Undefined/services/message_batcher/scheduler.py +++ b/src/Undefined/services/message_batcher/scheduler.py @@ -340,11 +340,20 @@ def unregister_inflight( state.phase.value, ) return + if ( + state.phase is BatchPhase.SPECULATING + and state.inflight is not None + and was_message_sent(state.inflight.request_context) + ): + # 已回复且模型已结束,旧批次无需再等待 T1;完成后不能再取消, + # 新消息必须开新桶,不能重放旧 items(与 allow_cancel_after_send 无关)。 + self._cancel_t1(state) + self._cancel_t2(state) + state.phase = BatchPhase.FINALIZING + self._buckets.pop(key, None) state.inflight = None - # 若 phase 是 SPECULATING 且 T1 已经 fire 过(FINALIZING 才 unregister), - # 此时 inflight 自然结束 → 桶已经在 _on_t1_timer 中弹出,无需再做事 - # 若仍在 SPECULATING(T1 未到):inflight 已结束但仍可能有新消息进来; - # 保持 SPECULATING,新消息会按 SPECULATING 分支处理(已发消息开新 batch / 未发追加) + # 未发回复的投机仍保留 items,后续补充消息可以重新合并;T1 已到的桶 + # 则交给 _handle_t1 收尾,避免注销旧任务时误删后来创建的新桶。 logger.debug( "[MessageBatcher] 注销 inflight 任务: scope=%s sender=%s phase=%s", scope, diff --git a/tests/test_message_batcher.py b/tests/test_message_batcher.py index 09048d0f..5970bb74 100644 --- a/tests/test_message_batcher.py +++ b/tests/test_message_batcher.py @@ -429,6 +429,38 @@ async def flush(items: list[BufferedMessage]) -> None: assert not inflight_task.cancelled() +@pytest.mark.asyncio +@pytest.mark.parametrize("already_sent", [False, True]) +@pytest.mark.parametrize("allow_cancel_after_send", [False, True]) +async def test_completed_speculation_does_not_reprocess_answered_items( + already_sent: bool, allow_cancel_after_send: bool +) -> None: + cfg = MessageBatcherConfig( + enabled=True, + window_seconds=100.0, + pre_send_seconds=50.0, + allow_cancel_after_send=allow_cancel_after_send, + ) + rec = _Recorder() + batcher = MessageBatcher(cfg, rec) + fake_ctx = _FakeRequestContext() + fake_ctx.set_resource("message_sent_this_turn", already_sent) + try: + await batcher.submit(_make_item(text="m1")) + await batcher._handle_t2(("group:1", 100)) + inflight = asyncio.create_task(asyncio.sleep(0)) + batcher.register_inflight("group:1", 100, inflight, fake_ctx) + await inflight + batcher.unregister_inflight("group:1", 100, inflight) + + await batcher.submit(_make_item(text="m2")) + await batcher.flush_sender("group:1", 100) + expected = [["m1"], ["m2"]] if already_sent else [["m1"], ["m1", "m2"]] + assert [[item.text for item in items] for items in rec.batches] == expected + finally: + await batcher.flush_all() + + @pytest.mark.asyncio async def test_speculative_cancelled_when_already_sent_with_allow_flag() -> None: """``allow_cancel_after_send=True`` 时即便已发过消息也强制取消 inflight。 From 7a8202db0677eb857ca5b7e43684bbf3f216ad0a Mon Sep 17 00:00:00 2001 From: Null <1708213363@qq.com> Date: Tue, 6 Oct 2026 11:26:45 +0800 Subject: [PATCH 003/121] fix(jm): honor session paths and isolate malformed inputs and chapters --- config.toml.example | 4 +- docs/configuration.md | 6 ++- src/Undefined/jm/client.py | 3 ++ src/Undefined/jm/downloader.py | 60 +++++++++++------------ src/Undefined/jm/parser.py | 10 ++-- tests/test_jm_downloader.py | 87 +++++++++++++++++++++++++++++++++- tests/test_jm_parser.py | 43 +++++++++++++++++ tests/test_jm_tool.py | 16 +++++++ 8 files changed, 192 insertions(+), 37 deletions(-) diff --git a/config.toml.example b/config.toml.example index a556f52a..21a67eae 100644 --- a/config.toml.example +++ b/config.toml.example @@ -1611,8 +1611,8 @@ request_timeout = 20.0 # zh: 图片下载超时(秒)。<=0 回退 60,>600 截断到 600。 # en: Image download timeout (seconds). <=0 falls back to 60, >600 is clamped to 600. image_timeout = 60.0 -# zh: jmcpy 的配置目录(登录态与会话文件所在目录,可指向已登录的 JMCPY_HOME)。留空使用平台默认目录。 -# en: jmcpy config directory (login session files; point it at an existing JMCPY_HOME). Empty uses the platform default directory. +# zh: jmcpy 的配置目录(登录态与会话文件所在目录)。显式值优先;留空读取 JMCPY_HOME,两者均为空时使用平台默认目录。 +# en: jmcpy config directory (login session files). An explicit value takes precedence; empty falls back to JMCPY_HOME, then the platform default directory. session_dir = "" # zh: Code Delivery Agent 配置(代码交付 Agent,在 Docker 容器中编写代码并打包上传)。 diff --git a/docs/configuration.md b/docs/configuration.md index 2f55a95e..a3939cdd 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1131,16 +1131,18 @@ api_key = "replace-with-your-key" | `download_concurrency` | `8` | 图片下载并发数 | `<1` 回退 `8`,`>32` 截断到 `32` | | `request_timeout` | `20.0` | 接口请求超时(秒) | `<=0` 回退 `20`,`>120` 截断到 `120` | | `image_timeout` | `60.0` | 图片下载超时(秒) | `<=0` 回退 `60`,`>600` 截断到 `600` | -| `session_dir` | `""` | jmcpy 配置目录(登录态会话文件所在目录) | 空时用 jmcpy 的平台默认目录 | +| `session_dir` | `""` | jmcpy 配置目录(登录态会话文件所在目录) | 非空值优先;空时使用 `JMCPY_HOME`,两者均为空时使用平台默认目录 | 触发规则: - 匹配 `JM` 前缀加 5–8 位车号:`JM1114751`、`jm 1114751`、`jm:1114751`、`jm-1114751`;前缀前必须是边界(`xxjm1234567` 不触发),数字后不跟数字(9 位以上不会被截成 8 位)。 - 匹配主机名含 `18comic` 或 `jmcomic` 的链接,车号取自 `/album/`、`/photo/` 或 `?id=`。 - 裸数字不触发,避免群号、时间戳等误报;QQ 分享卡片(`json` 消息段)里的链接同样会解析。 +- 畸形 URL 只跳过该链接,仍会继续解析同一条消息或后续分享卡片中的合法车号。 自动提取行为: - 命中后发送一次外层合并转发,固定两个节点:**本子信息**(车号、标题、作者、章节数、标签、观看/点赞、页数、zip 大小、被跳过的章节数、简介预览,末尾是可直接复制的 `JM<车号>`)、**PDF 解密密码**(每次随机生成 8 位,字母表去掉 `0/O/1/l/I` 等形近字符,适用于 zip 内所有 PDF)。打包好的 zip **不进转发**,而是紧随转发之后单独作为一条文件消息发送。 - 多章节本子会把全部章节按顺序下载(`max_chapters` 可限制章数),**每章各出一份 AES-256 加密 PDF**(都用同一个密码),再打包成一个**无密码** zip:`JM<车号> <标题>.zip`,内部条目是 `JM<车号> <序号>[ <章节标题>].pdf`,序号补零所以解压后按文件名就是阅读顺序。zip 不需要密码,密码在每个章节 PDF 内部由阅读器询问。 +- 服务端章节序号重复或缺失导致冲突时,下载目录与章节 PDF 的产物序号会顺延,保持唯一且符合阅读顺序;`uid` 模式也使用独立章节目录,避免复用同标题的上一章图片。下载结果中的章节元数据仍保留服务端原始序号。 - 每页按 `pdf_dpi` 统一换算物理尺寸(`页宽 = 像素宽 ÷ DPI × 72`),下载与合成都不缩放像素:PDF 里的页就是站点源图的原始分辨率,画质上限由站点自己提供的图决定。 - 每页只做一次有损编码:取服务端原始字节(无损落盘)→ 解扰 → JPEG(`image_quality`,4:4:4)→ 直接作为 PDF 的图像数据,中间不再多一代重编码。需要更高画质时可把 `image_quality` 提到 `98`/`100`(约 `+30%`/`+50%` 体积),源图本身的分辨率无法通过配置提升。 - 单章 PDF 按 `chapter_max_file_size` 判定:组装时累计**编码后字节**(JPEG 重编码 4:4:4 / q95 可能比源 WebP 大),超过上限就中止这一章、不落盘,其余章节照常打包,信息节点会写明「N 章未打包」。这是**单章**上限,zip 总体积随章节数增长、不受配置约束;旧键 `[jm].max_file_size` 已失效(回退默认 100),请改用 `chapter_max_file_size`。 @@ -1155,7 +1157,7 @@ api_key = "replace-with-your-key" - 除了自动提取,`jm_book` 工具让 AI 也能按需处理车号:`output_mode=send`(默认,等价自动提取)、`uid`(只下载并**把所有章节合成一份未加密 PDF** 注册为附件 UID,交给 `file_analysis_agent` 用 `extract_pdf` / `describe_pdf_page` 解析,不发送消息、不打包 zip)、`info`(只返回本子信息,不下载)。 - `info_agent` 的 `jm_search` 工具查询本子列表,`mode` 三种:`search`(关键词检索,可用 `target` 限定站内/作品/作者/标签/角色,也可直接输入纯数字车号)、`browse`(按分类浏览,不需要关键词)与 `ranking`(日/周/月排行,用 `time_range` 选日榜/周榜/月榜,固定按观看数排序)。三种模式共用 `genre` 大分类、`page` 翻页与 `n`(默认 5、上限 20);`search` / `browse` 另支持 `sort`(最新/观看/图片/点赞/评分/评论)与 `time_range` 时间范围;`sub_genre` 副分类必须与 `genre` 同时给出,且只有网页端接口支持,网页端被反爬拦截时会直接返回失败原因。只返回车号、标题、作者、分类与更新时间,不下载也不发送。查询复用同一份 `[jm]` 配置(代理、`request_timeout`、`session_dir`),且**不受** `auto_extract_enabled` 影响——该开关只约束自动提取管线。 - jmcpy 的 HTTP 请求是异步的,但图片解码与 PDF 合成是同步 CPU 工作;这些步骤在线程中执行,不阻塞事件循环。副作用是单次自动提取不可取消,由 `request_timeout` / `image_timeout` 与 jmcpy 的多端点重试兜底。 -- 默认不登录即可使用;需要在 jmcpy 中登录(例如访问账号可见内容)时,用 jmcpy 自己的方式保存会话,再把 `session_dir` 指向同一配置目录(或设置 `JMCPY_HOME`)。 +- 默认不登录即可使用;需要在 jmcpy 中登录(例如访问账号可见内容)时,用 jmcpy 自己的方式保存会话,再把 `session_dir` 指向同一配置目录,或在 `session_dir` 留空时设置 `JMCPY_HOME`。目录值会去除首尾空白,显式 `session_dir` 优先于环境变量。 - 请遵守所在地法律法规与目标站点服务条款,只处理你有权获取的内容。 --- diff --git a/src/Undefined/jm/client.py b/src/Undefined/jm/client.py index 4d73d5e9..8bf22368 100644 --- a/src/Undefined/jm/client.py +++ b/src/Undefined/jm/client.py @@ -6,6 +6,7 @@ from __future__ import annotations +import os from pathlib import Path from typing import Any @@ -20,6 +21,8 @@ def build_settings(config: Any) -> Settings: """把 ``[jm]`` 配置映射成 jmcpy 的 :class:`~jmcpy.Settings`。""" home = str(getattr(config, "jm_session_dir", "") or "").strip() + if not home: + home = os.environ.get("JMCPY_HOME", "").strip() return Settings( proxy=get_request_proxy(_PROXY_PROBE_URL, proxy_scope="jm", config=config), timeout=float(getattr(config, "jm_request_timeout", 20.0)), diff --git a/src/Undefined/jm/downloader.py b/src/Undefined/jm/downloader.py index 6e9deaaa..487e53e2 100644 --- a/src/Undefined/jm/downloader.py +++ b/src/Undefined/jm/downloader.py @@ -26,7 +26,7 @@ from __future__ import annotations -from collections.abc import Callable, Sequence +from collections.abc import Callable, Iterator, Sequence from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass, field import asyncio @@ -165,11 +165,33 @@ def _book_file_name(book: Book, suffix: str) -> str: return f"{_BOOK_ID_PREFIX}{book.book_id} {title}{suffix}" +def _chapter_artifact_indices( + chapters: Sequence[tuple[int, int, str]], +) -> Iterator[str]: + """生成递增且唯一的章节产物序号,保留正常序号与阅读顺序。""" + previous_index: int | None = None + for chapter_id, chapter_order, _title in chapters: + index = ( + chapter_order + if previous_index is None + else max(chapter_order, previous_index + 1) + ) + if index != chapter_order: + logger.warning( + "[JM] 章节产物序号冲突,顺延避免覆盖: chapter=%s order=%s → %s", + chapter_id, + chapter_order, + index, + ) + previous_index = index + yield f"{index:0{_CHAPTER_INDEX_DIGITS}d}" + + def _chapter_pdf_name(stem: str, title: str) -> str: """章节 PDF 名:``<产物名主体>[ <章节标题>].pdf``。 - ``stem`` 由 :func:`_download_archive_sync` 内的 ``artifact_stem`` 生成,已经保证 - 同一次下载内唯一。章节标题写在文件名里便于挑选章节;服务端没给标题时只留序号。 + ``stem`` 使用 :func:`_chapter_artifact_indices` 的唯一序号。章节标题写在文件名里 + 便于挑选章节;服务端没给标题时只留序号。 """ chapter_title = sanitize_filename(title) return ( @@ -319,40 +341,19 @@ def _download_archive_sync( """逐章下载并各出一份加密 PDF,最后打包成无密码 zip。""" limit_bytes = _chapter_limit_bytes(chapter_max_file_size_mb) built: list[_ChapterPdf] = [] - used_orders: set[int] = set() downloaded_chapters = 0 skipped_chapters = 0 failed_pages = 0 page_count = 0 - def artifact_index(chapter_order: int, local_index: int, chapter_id: int) -> str: - """章节产物的序号(补零字符串),用于下载目录名与 zip 条目名。 - - 优先用服务端章节序号(与阅读顺序一致);服务端返回重复序号(异常数据)时退回 - 本次下载的局部序号,否则两条章节会撞同一个下载目录与图片文件名 - (``overwrite=False`` 会拿旧图),也会撞同一个 zip 条目名而丢掉一份 PDF。 - """ - index = chapter_order - if index in used_orders: - index = local_index - logger.warning( - "[JM] 服务端返回重复章节序号,改用局部序号避免覆盖: book=%s chapter=%s order=%s → %s", - book_id, - chapter_id, - chapter_order, - index, - ) - used_orders.add(index) - return f"{index:0{_CHAPTER_INDEX_DIGITS}d}" - with Client(settings) as client: book = client.get_book(book_id) # 声明总数取本子自身的章节列表,不受 max_chapters 截断影响 declared_chapters = len(book.chapters) or 1 chapters = _ordered_chapters(book, max_chapters=max_chapters) - for local_index, (chapter_id, chapter_order, chapter_title) in enumerate( - chapters, start=1 + for (chapter_id, chapter_order, chapter_title), index in zip( + chapters, _chapter_artifact_indices(chapters), strict=True ): chapter = client.get_chapter(chapter_id) if not len(chapter): @@ -360,7 +361,6 @@ def artifact_index(chapter_order: int, local_index: int, chapter_id: int) -> str "[JM] 章节没有图片,跳过: book=%s chapter=%s", book_id, chapter_id ) continue - index = artifact_index(chapter_order, local_index, chapter_id) # 每章独立的目录:同名目录复用同名目录时,overwrite=False 会拿旧图。 # decode=False 取服务端原始字节(无损),解扰与编码在写 PDF 时做一次。 result = client.download( @@ -481,7 +481,9 @@ def _download_plain_sync( failed_pages = 0 downloaded_chapters = 0 - for chapter_id, chapter_order, _title in chapters: + for (chapter_id, _chapter_order, _title), index in zip( + chapters, _chapter_artifact_indices(chapters), strict=True + ): chapter = client.get_chapter(chapter_id) if not len(chapter): logger.info( @@ -491,7 +493,7 @@ def _download_plain_sync( result = client.download( chapter, output=ExportFormat.PATH, - dest=task_dir / f"c{chapter_order:0{_CHAPTER_INDEX_DIGITS}d}", + dest=task_dir / f"c{index}", decode=False, concurrency=settings.concurrency, ) diff --git a/src/Undefined/jm/parser.py b/src/Undefined/jm/parser.py index 8ad2334b..238e8259 100644 --- a/src/Undefined/jm/parser.py +++ b/src/Undefined/jm/parser.py @@ -42,7 +42,11 @@ def _strip_wrapper_chars(value: str) -> str: return html.unescape(value).strip().rstrip(_URL_WRAPPERS) -def _is_jm_host(host: str) -> bool: +def _is_jm_url(value: str) -> bool: + try: + host = urlsplit(value).hostname or "" + except ValueError: + return False return any(keyword in host for keyword in _JM_HOST_KEYWORDS) @@ -53,7 +57,7 @@ def normalize_jm_id(value: int | str) -> str | None: (普通数字串与非禁漫链接都不会被误判成车号)。 """ if isinstance(value, str) and "://" in value: - if not _is_jm_host(urlsplit(value).hostname or ""): + if not _is_jm_url(value): return None try: digits = normalize_book_id(value) @@ -83,7 +87,7 @@ def _iter_candidates(text: str) -> Iterator[tuple[int, str]]: """产出 ``(位置, 候选串)``:链接与 ``JM`` 车号混排时也按原文顺序排列。""" for match in _URL_REGEX.finditer(text): candidate = _strip_wrapper_chars(match.group(0)) - if _is_jm_host(urlsplit(candidate).hostname or ""): + if _is_jm_url(candidate): yield match.start(), candidate for match in _JM_TOKEN_REGEX.finditer(text): yield match.start(), match.group(1) diff --git a/tests/test_jm_downloader.py b/tests/test_jm_downloader.py index b180aa14..429de1f2 100644 --- a/tests/test_jm_downloader.py +++ b/tests/test_jm_downloader.py @@ -148,7 +148,9 @@ def download( paths: list[Path] = [] for name in chapter.pictures: path = dest / name - path.write_bytes(b"page") + # jmcpy 默认不覆盖同名页面:保留章节身份以验证不会复用上一章图片。 + if not path.exists(): + path.write_bytes(str(chapter.chapter_id).encode()) paths.append(path) self.downloads.append( { @@ -479,6 +481,60 @@ async def test_download_book_keeps_every_chapter_when_orders_collide( assert sorted(path.name for path in task_dir.glob("*.pdf")) == [] +@pytest.mark.asyncio +@pytest.mark.parametrize("password", [None, "Ab3xK9Qm"]) +@pytest.mark.parametrize( + ("orders", "expected_indices"), + [ + ((2, 2), ("002", "003")), + ((2, 2, 3), ("002", "003", "004")), + ((1, 3, 3, 4), ("001", "003", "004", "005")), + ((3, 3, 3), ("003", "004", "005")), + ], +) +async def test_download_book_preserves_chapter_pages_when_orders_collide( + monkeypatch: pytest.MonkeyPatch, + orders: tuple[int, ...], + expected_indices: tuple[str, ...], + password: str | None, +) -> None: + chapter_ids = [111 * index for index in range(1, len(orders) + 1)] + _install_client( + monkeypatch, + chapters=[ + (chapter_id, order, "序章") + for chapter_id, order in zip(chapter_ids, orders, strict=True) + ], + ) + captured = _capture_write_pdf(monkeypatch) + + result, _ = await download_book("1114751", config=_config(), password=password) + + assert result.ok + assert result.page_count == len(orders) * 2 + assert [item["dest"].name for item in _FakeClient.instances[0].downloads] == [ + f"c{index}" for index in expected_indices + ] + sources = [ + path.read_bytes() for call in captured["calls"] for path in call["sources"] + ] + assert sources == [ + str(chapter_id).encode() for chapter_id in chapter_ids for _ in range(2) + ] + if password is not None: + archive = _archive(result) + assert archive.zip_path is not None + assert _zip_entries(archive.zip_path) == [ + f"JM1114751 {index} 序章.pdf" for index in expected_indices + ] + assert [pdf.order for pdf in archive.pdfs] == list(orders) + assert [pdf.chapter_id for pdf in archive.pdfs] == chapter_ids + else: + plain = _plain(result) + assert plain.pdf_path is not None + assert len(captured["calls"]) == 1 + + @pytest.mark.asyncio async def test_download_book_skips_chapter_whose_pages_all_failed( monkeypatch: pytest.MonkeyPatch, @@ -613,6 +669,7 @@ def test_build_settings_leaves_home_unset_when_session_dir_empty( monkeypatch: pytest.MonkeyPatch, ) -> None: monkeypatch.setattr(jm_client, "get_request_proxy", lambda *_args, **_kwargs: None) + monkeypatch.delenv("JMCPY_HOME", raising=False) settings = jm_client.build_settings(_config(jm_session_dir=" ")) @@ -620,6 +677,34 @@ def test_build_settings_leaves_home_unset_when_session_dir_empty( assert settings.proxy is None +@pytest.mark.parametrize( + ("session_dir", "env_home", "expected_home"), + [ + ("", "/tmp/jmcpy-env", Path("/tmp/jmcpy-env")), + (" ", " /tmp/jmcpy-env ", Path("/tmp/jmcpy-env")), + (" /tmp/jmcpy-explicit ", "/tmp/jmcpy-env", Path("/tmp/jmcpy-explicit")), + ("", " ", None), + ], +) +def test_build_settings_resolves_session_dir_before_jmcpy_home( + monkeypatch: pytest.MonkeyPatch, + session_dir: str, + env_home: str, + expected_home: Path | None, +) -> None: + monkeypatch.setenv("JMCPY_HOME", env_home) + monkeypatch.setattr(jm_client, "get_request_proxy", lambda *_args, **_kwargs: None) + + settings = jm_client.build_settings(_config(jm_session_dir=session_dir)) + + assert settings.home == expected_home + assert ( + settings.resolved_session_path + == (expected_home if expected_home is not None else settings.resolved_home) + / "session.json" + ) + + def _page_png(path: Path, size: tuple[int, int], *, band: int = 0) -> Path: image = Image.new("RGB", size, (255, 255, 255)) for index in range(0, size[1], 7): diff --git a/tests/test_jm_parser.py b/tests/test_jm_parser.py index 55f03e0a..0e292c64 100644 --- a/tests/test_jm_parser.py +++ b/tests/test_jm_parser.py @@ -3,6 +3,8 @@ import json from typing import Any +import pytest + from Undefined.jm.parser import ( extract_from_json_message, extract_jm_ids, @@ -27,6 +29,25 @@ def test_normalize_jm_id_rejects_out_of_range_and_garbage() -> None: assert normalize_jm_id("https://example.com/album/1114751") is None +@pytest.mark.parametrize( + "value", + [ + "https://[18comic.vip/album/1114751", + "https://[invalid]/album/1114751", + ], +) +def test_normalize_jm_id_rejects_malformed_url(value: str) -> None: + assert normalize_jm_id(value) is None + + +def test_extract_jm_ids_skips_malformed_urls_and_keeps_text_order() -> None: + assert extract_jm_ids( + "JM1111111 https://[example.com/anything " + "https://[18comic.vip/album/3333333 " + "https://18comic.vip/album/2222222 JM4444444" + ) == ["1111111", "2222222", "4444444"] + + def test_extract_jm_ids_accepts_token_forms() -> None: assert extract_jm_ids("jm1234567") == ["1234567"] assert extract_jm_ids("看看 JM 1114751 吧") == ["1114751"] @@ -141,3 +162,25 @@ def test_extract_from_json_message_skips_broken_json_body() -> None: segments = [{"type": "json", "data": {"data": "{not json"}}] assert extract_from_json_message(segments) == [] + + +def test_extract_from_json_message_keeps_ids_after_malformed_url() -> None: + segments: list[dict[str, Any]] = [ + { + "type": "json", + "data": { + "data": json.dumps( + { + "bad_url": "https://[18comic.vip/album/1114751", + "text": "JM2222222", + } + ) + }, + }, + { + "type": "json", + "data": {"data": json.dumps({"url": "https://18comic.vip/album/3333333"})}, + }, + ] + + assert extract_from_json_message(segments) == ["2222222", "3333333"] diff --git a/tests/test_jm_tool.py b/tests/test_jm_tool.py index a89ee514..66f5823b 100644 --- a/tests/test_jm_tool.py +++ b/tests/test_jm_tool.py @@ -58,6 +58,22 @@ async def test_jm_book_info_mode_rejects_bad_book_id( fetch.assert_not_awaited() +@pytest.mark.asyncio +async def test_jm_book_rejects_malformed_url_without_fetching( + monkeypatch: pytest.MonkeyPatch, +) -> None: + fetch = AsyncMock() + monkeypatch.setattr(jm_tool, "jm_book_info_text", fetch) + + result = await execute( + {"book_id": "https://[18comic.vip/album/350234", "output_mode": "info"}, + _context(), + ) + + assert "无法解析禁漫车号" in result + fetch.assert_not_awaited() + + @pytest.mark.asyncio async def test_jm_book_uid_mode_registers_attachment( monkeypatch: pytest.MonkeyPatch, From d70d4cba62b0aff774ae7ca4c49ae834a058db13 Mon Sep 17 00:00:00 2001 From: Null <1708213363@qq.com> Date: Tue, 6 Oct 2026 11:27:22 +0800 Subject: [PATCH 004/121] fix(deploy): defer writes and map interpreter files for host Docker Accept UTF-8 BOM configuration and preserve cancellation without side effects. --- docs/deployment.md | 2 + docs/docker-deploy.md | 6 +- src/Undefined/deploy/config_patch.py | 2 +- src/Undefined/deploy/generate.py | 11 +++ src/Undefined/deploy/runner.py | 55 ++++++------ .../deploy/templates/compose.bot.yaml | 4 + .../skills/tools/python_interpreter/README.md | 17 +++- .../tools/python_interpreter/handler.py | 51 ++++++++--- tests/test_deploy_config_patch.py | 21 +++++ tests/test_deploy_generate.py | 37 ++++++++ tests/test_deploy_runner.py | 60 +++++++++++++ tests/test_python_interpreter_handler.py | 90 +++++++++++++++++++ 12 files changed, 314 insertions(+), 42 deletions(-) diff --git a/docs/deployment.md b/docs/deployment.md index ae437ef1..5c75d6b2 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -201,6 +201,8 @@ uv run deploy up 部署结束后,按终端显示的地址和密码进入 WebUI,填写模型 API 与 QQ 身份配置,到 NapCat 扫码登录后,再返回 WebUI 点击“启动机器人”。`container` 与 `host` 模式都会将 `[webui].autostart_bot` 设为 `false`、`[onebot].file_send_mode` 设为 `"stream"`。**环境准备、首次配置、远程访问和常见问题请直接看 [Docker 一键部署指南](docker-deploy.md)**。 +向导在最终确认前只读取配置,取消不会生成文件或初始化子模块;已有 `config.toml` 支持 UTF-8 BOM 编码。`container` 模式还会为 Python 解释器自动映射已共享的 `deploy/data/cache/python_interpreter/`,让宿主 Docker 从正确的宿主路径读取执行文件。手工挂载 Docker socket 的容器需按 [Python 解释器工具说明](../src/Undefined/skills/tools/python_interpreter/README.md#容器内使用宿主-docker)配置共享路径。 + --- ## pip/uv tool 部署(快速体验) diff --git a/docs/docker-deploy.md b/docs/docker-deploy.md index 1843e3ce..2f612f3a 100644 --- a/docs/docker-deploy.md +++ b/docs/docker-deploy.md @@ -56,7 +56,7 @@ uv run deploy up | 发布端口的绑定地址 | 保持 `127.0.0.1`;在服务器上部署时,访问方法见下文[远程访问](#在自己的电脑上访问服务器) | | 是否拉取 NagaAgent 子模块 | 不需要 NagaAgent 代码问答时选“否” | -向导随后会展示配置变更,确认后开始拉取镜像和启动容器。首次运行需要下载依赖与镜像,请等待命令完成。 +向导随后会展示配置变更,最终确认前只读取配置和模板;选择取消不会创建 `config.toml` 或拉取 NagaAgent 子模块。确认后才生成配置、按需初始化子模块、拉取镜像并启动容器。首次运行需要下载依赖与镜像,请等待命令完成。 如需跳过向导,可用 `uv run deploy up --yes` 代替;首次执行采用默认配置,之后执行会沿用上次的部署选择。只想提前查看变更时,可用 `uv run deploy up --dry-run`,它不会写文件或启动容器。 @@ -85,7 +85,7 @@ uv run deploy status 3. 配置 `models.chat`、`models.vision`、`models.agent` 的 API 地址、API Key 和模型名称;其他已启用功能所需的模型也应按配置提示填写。完整要求见 [配置必填项](configuration.md#3-严格模式stricttrue必填项)。 4. 保存配置并检查校验结果;字段缺失或无效时,按提示补齐。 -也可以直接编辑仓库根目录的 `config.toml`。部署工具已填写 OneBot 连接和所选服务的地址,首次部署通常无需再改这些项。**模型 API 和 QQ 身份信息需要你自己填写,容器启动不代表这些配置已完成。** +也可以直接编辑仓库根目录的 `config.toml`,部署工具与 Bot 一样支持 UTF-8 和 UTF-8 BOM 编码。部署工具已填写 OneBot 连接和所选服务的地址,首次部署通常无需再改这些项。**模型 API 和 QQ 身份信息需要你自己填写,容器启动不代表这些配置已完成。** ### 扫码登录并测试回复 @@ -227,6 +227,8 @@ uv run deploy up --port bot_webui=18787 --port napcat_webui=16099 重新运行向导可以调整模式和可选服务。重复部署会重新生成 Compose 与服务配置,并复用已有有效凭据;对生成文件的手工修改可能被覆盖。修改过 WebUI 密码时,以 `config.toml` 中的当前值为准。 +`container` 模式的 Python 解释器使用 `deploy/data/cache/python_interpreter/` 中独立的临时工作目录。部署工具通过容器环境变量传入这份共享目录在 Bot 内和宿主机上的不同路径,使宿主 Docker 能读取脚本、依赖清单和生成的文件;不会共享整个宿主 `/tmp`。`host` 模式继续使用系统临时目录。手工部署容器时的设置方法见 [Python 解释器工具说明](../src/Undefined/skills/tools/python_interpreter/README.md#容器内使用宿主-docker)。 + ### 更新版本 先备份配置与数据,并将仓库更新到要部署的已发布版本,再执行: diff --git a/src/Undefined/deploy/config_patch.py b/src/Undefined/deploy/config_patch.py index 25bc229c..2bec12cf 100644 --- a/src/Undefined/deploy/config_patch.py +++ b/src/Undefined/deploy/config_patch.py @@ -107,7 +107,7 @@ def load_toml(path: Path) -> dict[str, Any]: if not path.is_file(): return {} try: - return tomllib.loads(path.read_text(encoding="utf-8")) + return tomllib.loads(path.read_text(encoding="utf-8-sig")) except tomllib.TOMLDecodeError as exc: raise ConfigError(f"{path} 解析失败:{exc}") from exc except OSError as exc: diff --git a/src/Undefined/deploy/generate.py b/src/Undefined/deploy/generate.py index 5c519a9d..058e62a9 100644 --- a/src/Undefined/deploy/generate.py +++ b/src/Undefined/deploy/generate.py @@ -67,6 +67,10 @@ #: 挂进本体容器的宿主机 Docker socket(DooD)。 DOCKER_SOCKET_ENV = "UNDEFINED_DEPLOY_DOCKER_SOCKET" +#: Python 解释器在已挂载 data/ 下共享文件;两端路径允许不同。 +PYTHON_INTERPRETER_TMP_SUBDIR = Path("cache") / "python_interpreter" +PYTHON_INTERPRETER_LOCAL_DIR_ENV = "UNDEFINED_DEPLOY_PYTHON_INTERPRETER_LOCAL_DIR" +PYTHON_INTERPRETER_HOST_DIR_ENV = "UNDEFINED_DEPLOY_PYTHON_INTERPRETER_HOST_DIR" class GenerateError(RuntimeError): @@ -264,6 +268,13 @@ def prev(name: str) -> str | None: "UNDEFINED_DEPLOY_LXMUSIC2API_UID": str(_current_uid()), "UNDEFINED_DEPLOY_LXMUSIC2API_GID": str(_current_gid()), } + if ctx.mode == catalog.MODE_CONTAINER: + env[PYTHON_INTERPRETER_LOCAL_DIR_ENV] = str( + Path(CONTAINER_REPO_PATH) / "data" / PYTHON_INTERPRETER_TMP_SUBDIR + ) + env[PYTHON_INTERPRETER_HOST_DIR_ENV] = str( + ctx.layout.data_dir / PYTHON_INTERPRETER_TMP_SUBDIR + ) # 端口与绑定:默认只绑回环,远程访问需显式改 --port-bind for key, spec in catalog.port_specs().items(): diff --git a/src/Undefined/deploy/runner.py b/src/Undefined/deploy/runner.py index baee85a3..4ca053e5 100644 --- a/src/Undefined/deploy/runner.py +++ b/src/Undefined/deploy/runner.py @@ -486,37 +486,21 @@ def run_up(options: dict[str, Any]) -> int: _previous_port_bind(previous_state, previous_env) ) or _previous_port_bind(previous_state, previous_env) - # 子模块必须先就绪:能力开关与目标代码由同一问决定 - if nagaagent_enabled and not dry_run: - try: - result = nagaagent.ensure_submodule(repo) - except RuntimeError as exc: - print(f"错误:{exc}") - return EXIT_ERROR - print( - "NagaAgent 子模块:" - + ("已就绪" if result == "already-ready" else "拉取完成") - ) - # 端口与绑定地址必须像凭据一样跨次复用:否则重跑 up 会静默把 --port / # --port-bind 退回默认值(远程访问会无声中断)。 ports = _resolve_ports(previous_state, previous_env) ports.update(options.get("port_overrides") or {}) - # 全新部署时必须先落一份完整配置:否则只渲染被 patch 的几个键, - # 生成出来的 config.toml 没有 [models]/[core],用户无从填写模型端。 - # dry-run 不落盘,因此跳过创建,只按现有(可能为空)配置推导。 - if not dry_run: - try: - if config_patch.ensure_config_file( - repo / "config.toml", repo / "config.toml.example" - ): - print("config.toml:已从 config.toml.example 生成") - except config_patch.ConfigError as exc: - print(f"错误:{exc}") - return EXIT_ERROR - - existing_config = config_patch.load_toml(repo / "config.toml") + # 确认前只读现有配置或模板:取消和 dry-run 都不创建配置、不拉取子模块。 + config_path = repo / "config.toml" + example_path = repo / "config.toml.example" + try: + existing_config = config_patch.load_toml( + config_path if config_path.is_file() else example_path + ) + except config_patch.ConfigError as exc: + print(f"错误:{exc}") + return EXIT_ERROR ctx = generate.GenerateContext( repo=repo, @@ -564,6 +548,25 @@ def run_up(options: dict[str, Any]) -> int: print("已取消,未改动任何文件。") return EXIT_OK + if not dry_run: + # 子模块先就绪,再开启能力;所有写入都必须在最终确认之后。 + if nagaagent_enabled: + try: + result = nagaagent.ensure_submodule(repo) + except RuntimeError as exc: + print(f"错误:{exc}") + return EXIT_ERROR + print( + "NagaAgent 子模块:" + + ("已就绪" if result == "already-ready" else "拉取完成") + ) + try: + if config_patch.ensure_config_file(config_path, example_path): + print("config.toml:已从 config.toml.example 生成") + except config_patch.ConfigError as exc: + print(f"错误:{exc}") + return EXIT_ERROR + project_name = ( previous_state.project_name if previous_state else catalog.COMPOSE_PROJECT_NAME ) diff --git a/src/Undefined/deploy/templates/compose.bot.yaml b/src/Undefined/deploy/templates/compose.bot.yaml index 43f0091b..ce81f8b8 100644 --- a/src/Undefined/deploy/templates/compose.bot.yaml +++ b/src/Undefined/deploy/templates/compose.bot.yaml @@ -19,6 +19,10 @@ services: networks: - undefined-deploy working_dir: /data/Undefined + environment: + # Docker bind mount 来源由宿主 daemon 解析;不可把 Bot 容器的 /tmp 直接传给它。 + UNDEFINED_PYTHON_INTERPRETER_LOCAL_DIR: ${UNDEFINED_DEPLOY_PYTHON_INTERPRETER_LOCAL_DIR} + UNDEFINED_PYTHON_INTERPRETER_HOST_DIR: ${UNDEFINED_DEPLOY_PYTHON_INTERPRETER_HOST_DIR} ports: - "${UNDEFINED_DEPLOY_PORT_BOT_WEBUI_BIND}:${UNDEFINED_DEPLOY_PORT_BOT_WEBUI}:8787" - "${UNDEFINED_DEPLOY_PORT_BOT_API_BIND}:${UNDEFINED_DEPLOY_PORT_BOT_API}:8788" diff --git a/src/Undefined/skills/tools/python_interpreter/README.md b/src/Undefined/skills/tools/python_interpreter/README.md index f934f487..92ada344 100644 --- a/src/Undefined/skills/tools/python_interpreter/README.md +++ b/src/Undefined/skills/tools/python_interpreter/README.md @@ -4,7 +4,7 @@ 限制说明: - 主执行容器**无法访问网络**且只读(`--network none --read-only`) -- 无法访问宿主机文件系统(`send_files` 除外,见下) +- 仅挂载本次执行的临时工作目录到 `/tmp`;`send_files` 只能回传该目录内的产物 ## 参数 @@ -14,6 +14,21 @@ | `libraries` | list[string] | 否 | 需要的第三方库;指定后容器会先联网执行 `pip install`,再在无网环境运行代码 | | `send_files` | list[string] | 否 | 需要回传的容器内 `/tmp/` 产物路径;执行后作为附件返回 | +## 容器内使用宿主 Docker + +源码或 `host` 模式下,Bot 与 Docker daemon 使用同一宿主文件系统,工具默认在系统临时目录中创建工作目录。 + +Bot 本身在容器中且挂载宿主 Docker socket 时,Docker 的 bind mount 来源由宿主 daemon 解析,需要显式映射共享目录的两端路径。`uv run deploy up` 的 `container` 模式自动复用已有 `data/` 挂载,并设置: + +| Bot 环境变量 | 一键部署中的值 | +|---|---| +| `UNDEFINED_PYTHON_INTERPRETER_LOCAL_DIR` | `/data/Undefined/data/cache/python_interpreter`,Bot 可读写的共享目录 | +| `UNDEFINED_PYTHON_INTERPRETER_HOST_DIR` | 当前仓库的 `deploy/data/cache/python_interpreter` 绝对路径,宿主 Docker 可见的同一目录 | + +手工部署时,同时设置这两个变量为对应的绝对路径,并保证指向同一份共享目录。工具在 Bot 侧创建独立工作目录、写入脚本,向 Docker 传入对应宿主路径,再从 Bot 侧读取输出并清理。不会挂载整个宿主 `/tmp`。更换仓库位置后应重新生成部署配置,避免保留旧的宿主绝对路径。 + +执行目录在完成后清理;需要回传文件时会保留 30 秒,让附件发送完成。 + 目录结构: - `config.json`:工具定义 - `handler.py`:执行逻辑 diff --git a/src/Undefined/skills/tools/python_interpreter/handler.py b/src/Undefined/skills/tools/python_interpreter/handler.py index f23365db..7fdb57e9 100644 --- a/src/Undefined/skills/tools/python_interpreter/handler.py +++ b/src/Undefined/skills/tools/python_interpreter/handler.py @@ -18,6 +18,8 @@ TIMEOUT = 480 # 8 分钟 TIMEOUT_WITH_LIBS = 600 # 10 分钟(pip 安装需要更多时间) OUTPUT_FILE_RETENTION_SECONDS = 30.0 +SHARED_LOCAL_DIR_ENV = "UNDEFINED_PYTHON_INTERPRETER_LOCAL_DIR" +SHARED_HOST_DIR_ENV = "UNDEFINED_PYTHON_INTERPRETER_HOST_DIR" # 图片扩展名(内联发送而非文件附件) _IMAGE_EXTENSIONS = frozenset({".png", ".jpg", ".jpeg", ".gif", ".bmp", ".webp"}) @@ -27,6 +29,26 @@ _PENDING_CLEANUP_TASKS: set[asyncio.Task[None]] = set() +def _create_execution_dirs() -> tuple[str, str]: + """返回 Bot 本地工作目录与 Docker daemon 可见的 bind mount 来源。""" + local_dir = os.getenv(SHARED_LOCAL_DIR_ENV, "").strip() + host_dir = os.getenv(SHARED_HOST_DIR_ENV, "").strip() + if not local_dir and not host_dir: + tmpdir = tempfile.mkdtemp(prefix="pyinterp_") + return tmpdir, tmpdir + if not local_dir or not host_dir: + raise ValueError( + f"必须同时配置 {SHARED_LOCAL_DIR_ENV} 和 {SHARED_HOST_DIR_ENV}" + ) + local_root = Path(local_dir) + host_root = Path(host_dir) + if not local_root.is_absolute() or not host_root.is_absolute(): + raise ValueError("Python 解释器共享目录必须是绝对路径") + local_root.mkdir(parents=True, exist_ok=True) + tmpdir = tempfile.mkdtemp(prefix="pyinterp_", dir=local_root) + return tmpdir, str(host_root / Path(tmpdir).name) + + def _get_output_relative_path(container_path: str) -> PurePosixPath | None: try: relative = PurePosixPath(container_path).relative_to("/tmp") @@ -172,25 +194,27 @@ async def execute(args: Dict[str, Any], context: Dict[str, Any]) -> str: memory = MEMORY_LIMIT_WITH_LIBS if has_libs else MEMORY_LIMIT timeout = TIMEOUT_WITH_LIBS if has_libs else TIMEOUT - host_tmpdir = tempfile.mkdtemp(prefix="pyinterp_") + execution_dir: str | None = None defer_cleanup = False try: + local_tmpdir, docker_tmpdir = await asyncio.to_thread(_create_execution_dirs) + execution_dir = local_tmpdir for fpath in send_files: - if _resolve_output_host_path(fpath, host_tmpdir) is None: + if _resolve_output_host_path(fpath, local_tmpdir) is None: return f"错误: 输出文件路径必须位于容器 /tmp 目录内: '{fpath}'" - script_path = os.path.join(host_tmpdir, "_script.py") + script_path = os.path.join(local_tmpdir, "_script.py") deadline = time.monotonic() + timeout if has_libs: # 单独安装依赖,再以无网络/只读根文件系统执行用户代码。 - req_path = os.path.join(host_tmpdir, "_requirements.txt") + req_path = os.path.join(local_tmpdir, "_requirements.txt") with open(req_path, "w", encoding="utf-8") as f: for lib in libraries: f.write(lib + "\n") install_timeout = max(deadline - time.monotonic(), 1.0) - install_cmd = _build_install_cmd(host_tmpdir, memory) + install_cmd = _build_install_cmd(docker_tmpdir, memory) install_code, install_stdout, install_stderr = await _run_docker_command( install_cmd, timeout=install_timeout, @@ -207,7 +231,7 @@ async def execute(args: Dict[str, Any], context: Dict[str, Any]) -> str: exec_timeout = max(deadline - time.monotonic(), 1.0) cmd = _build_exec_cmd( - host_tmpdir, + docker_tmpdir, memory, pythonpath="/tmp/_site_packages", ) @@ -215,7 +239,7 @@ async def execute(args: Dict[str, Any], context: Dict[str, Any]) -> str: # 将代码写入脚本文件(避免 shell 引号转义问题) with open(script_path, "w", encoding="utf-8") as f: f.write(code) - cmd = _build_exec_cmd(host_tmpdir, memory) + cmd = _build_exec_cmd(docker_tmpdir, memory) exec_timeout = timeout logger.info( @@ -243,7 +267,9 @@ async def execute(args: Dict[str, Any], context: Dict[str, Any]) -> str: ) if send_files and exit_code == 0: - file_result = await _send_output_files(send_files, host_tmpdir, context) + file_result = await _send_output_files( + send_files, local_tmpdir, context + ) if file_result: parts.append(file_result) defer_cleanup = True @@ -257,10 +283,11 @@ async def execute(args: Dict[str, Any], context: Dict[str, Any]) -> str: logger.exception("[Python解释器] 执行出错: %s", e) return "执行出错,请检查代码或重试" finally: - if defer_cleanup: - _schedule_output_dir_cleanup(host_tmpdir) - else: - shutil.rmtree(host_tmpdir, ignore_errors=True) + if execution_dir is not None: + if defer_cleanup: + _schedule_output_dir_cleanup(execution_dir) + else: + await asyncio.to_thread(shutil.rmtree, execution_dir, True) async def _send_output_files( diff --git a/tests/test_deploy_config_patch.py b/tests/test_deploy_config_patch.py index f00b4640..e10362c5 100644 --- a/tests/test_deploy_config_patch.py +++ b/tests/test_deploy_config_patch.py @@ -8,6 +8,7 @@ import pytest from Undefined.deploy import config_patch +from Undefined.config.toml_io import load_toml_data from Undefined.webui.utils.comment import parse_comment_map #: 故意写成「已有注释 + 已有用户自定义值 + 嵌套表」的形态, @@ -63,6 +64,26 @@ def _plan(config_path: Path, desired: dict[str, object]) -> config_patch.PatchPl ) +def test_apply_accepts_bom_config_and_preserves_original_backup( + config_dir: tuple[Path, Path, Path], +) -> None: + config, example, backup_dir = config_dir + config.write_text(EXAMPLE, encoding="utf-8-sig") + original = config.read_bytes() + assert config_patch.load_toml(config) == load_toml_data(config, strict=True) + + outcome = config_patch.apply_plan( + _plan(config, {"onebot.ws_url": "ws://napcat:3001"}), + example_path=example, + backup_dir=backup_dir, + ) + + assert outcome.written is True + assert outcome.backup_path is not None + assert outcome.backup_path.read_bytes() == original + assert config_patch.load_toml(config)["onebot"]["ws_url"] == "ws://napcat:3001" + + def test_plan_items_report_current_and_about( config_dir: tuple[Path, Path, Path], ) -> None: diff --git a/tests/test_deploy_generate.py b/tests/test_deploy_generate.py index 86f41a83..d43e7d04 100644 --- a/tests/test_deploy_generate.py +++ b/tests/test_deploy_generate.py @@ -128,6 +128,43 @@ def test_container_mode_publishes_bot_ports(tmp_path: Path) -> None: assert env["UNDEFINED_DEPLOY_PORT_BOT_WEBUI_BIND"] == catalog.DEFAULT_PORT_BIND +def test_container_maps_python_interpreter_paths_to_existing_data_mount( + tmp_path: Path, +) -> None: + ctx = _ctx(tmp_path) + config = generate.build(ctx) + bot = _compose_services(config.compose_text)["undefined-bot"] + environment = bot["environment"] + local_root = Path(config.env[generate.PYTHON_INTERPRETER_LOCAL_DIR_ENV]) + host_root = Path(config.env[generate.PYTHON_INTERPRETER_HOST_DIR_ENV]) + + assert local_root != host_root + # 真实 host 来源必须由 data/ 挂载及同一相对路径推出,不能传 Bot 的绝对路径。 + data_mount = next( + volume for volume in bot["volumes"] if volume.endswith(":/data/Undefined/data") + ) + source, target = data_mount.split(":") + assert host_root == (ctx.layout.root / source).resolve() / local_root.relative_to( + target + ) + assert environment["UNDEFINED_PYTHON_INTERPRETER_LOCAL_DIR"] == ( + "${" + generate.PYTHON_INTERPRETER_LOCAL_DIR_ENV + "}" + ) + assert environment["UNDEFINED_PYTHON_INTERPRETER_HOST_DIR"] == ( + "${" + generate.PYTHON_INTERPRETER_HOST_DIR_ENV + "}" + ) + assert not any(volume.endswith(":/tmp") for volume in bot["volumes"]) + + +def test_host_mode_leaves_python_interpreter_temp_paths_unchanged( + tmp_path: Path, +) -> None: + config = generate.build(_ctx(tmp_path, mode=catalog.MODE_HOST)) + assert generate.PYTHON_INTERPRETER_LOCAL_DIR_ENV not in config.env + assert generate.PYTHON_INTERPRETER_HOST_DIR_ENV not in config.env + assert "undefined-bot" not in _compose_services(config.compose_text) + + def test_unselected_service_ports_are_not_published(tmp_path: Path) -> None: env = generate.build(_ctx(tmp_path)).env assert "UNDEFINED_DEPLOY_PORT_SEARXNG" not in env diff --git a/tests/test_deploy_runner.py b/tests/test_deploy_runner.py index bd12e63d..8b551e6a 100644 --- a/tests/test_deploy_runner.py +++ b/tests/test_deploy_runner.py @@ -628,6 +628,66 @@ def test_up_creates_full_config_from_example_on_fresh_clone( assert parsed["webui"]["autostart_bot"] is False +@pytest.mark.parametrize("existing_config", (False, True)) +def test_cancel_up_writes_nothing_and_does_not_initialize_submodule( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], + existing_config: bool, +) -> None: + repo = tmp_path / "repo" + repo.mkdir() + (repo / "config.toml.example").write_text(CONFIG_TOML, encoding="utf-8") + if existing_config: + (repo / "config.toml").write_text(CONFIG_TOML, encoding="utf-8") + before = {path.name: path.read_bytes() for path in repo.iterdir()} + answers = iter(("", "", "n")) + monkeypatch.setattr(runner, "repo_root", lambda: repo) + monkeypatch.setattr(runner, "wizard_enabled", lambda options: True) + monkeypatch.setattr(runner.Wizard, "_ask", lambda self, prompt: next(answers)) + + def unexpected_side_effect(*args: object, **kwargs: object) -> None: + raise AssertionError("取消前不能写入子模块或调用 Docker") + + monkeypatch.setattr(nagaagent, "ensure_submodule", unexpected_side_effect) + monkeypatch.setattr(docker_cli, "check_availability", unexpected_side_effect) + + assert runner.run_up(_yes_options(yes=False, dry_run=False, nagaagent=True)) == 0 + assert {path.name: path.read_bytes() for path in repo.iterdir()} == before + assert "已取消,未改动任何文件。" in capsys.readouterr().out + + +def test_up_accepts_bom_config( + fake_repo: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + config_path = fake_repo / "config.toml" + config_path.write_text(CONFIG_TOML, encoding="utf-8-sig") + _stub_docker(monkeypatch) + + assert runner.run_up(_yes_options(dry_run=False)) == 0 + assert ( + tomllib.loads(config_path.read_text())["onebot"]["file_send_mode"] == "stream" + ) + + +def test_up_reports_invalid_config_before_any_write( + fake_repo: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + config_path = fake_repo / "config.toml" + config_path.write_text("[broken", encoding="utf-8") + + def unexpected_submodule(*args: object, **kwargs: object) -> None: + raise AssertionError("无效配置不能触发子模块写入") + + monkeypatch.setattr(nagaagent, "ensure_submodule", unexpected_submodule) + assert runner.run_up(_yes_options(dry_run=False, nagaagent=True)) != 0 + assert config_path.read_text() == "[broken" + assert not (fake_repo / "deploy").exists() + assert "解析失败" in capsys.readouterr().out + + @pytest.mark.parametrize("mode", catalog.DEPLOY_MODES) @pytest.mark.parametrize("previous_file_mode", ("local", "url")) def test_up_replaces_legacy_startup_and_file_transport_settings( diff --git a/tests/test_python_interpreter_handler.py b/tests/test_python_interpreter_handler.py index 6683eb8d..6ba24d52 100644 --- a/tests/test_python_interpreter_handler.py +++ b/tests/test_python_interpreter_handler.py @@ -4,6 +4,7 @@ import os import tempfile from pathlib import Path +from typing import Any from unittest.mock import AsyncMock, MagicMock import pytest @@ -11,6 +12,12 @@ from Undefined.skills.tools.python_interpreter import handler as python_handler +@pytest.fixture(autouse=True) +def clear_shared_directory_environment(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv(python_handler.SHARED_LOCAL_DIR_ENV, raising=False) + monkeypatch.delenv(python_handler.SHARED_HOST_DIR_ENV, raising=False) + + class _FakeProcess: def __init__(self, *, returncode: int = 0) -> None: self.returncode = returncode @@ -76,6 +83,13 @@ async def _fake_create_subprocess_exec( *args: str, **kwargs: object ) -> _FakeProcess: calls.append(tuple(args)) + source = Path(args[args.index("-v") + 1].removesuffix(":/tmp")) + assert source.is_dir() + if len(calls) == 1: + assert (source / "_requirements.txt").read_text() == "requests\n" + assert not (source / "_script.py").exists() + else: + assert (source / "_script.py").read_text() == "print('hello')" return _FakeProcess() real_mkdtemp = tempfile.mkdtemp @@ -162,3 +176,79 @@ async def test_cleanup_output_dir_later_removes_directory(tmp_path: Path) -> Non await python_handler._cleanup_output_dir_later(str(host_tmpdir), 0) assert not host_tmpdir.exists() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("with_libraries", (False, True)) +async def test_shared_directory_uses_daemon_path_and_reads_outputs_locally( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, with_libraries: bool +) -> None: + # 两个名字模拟 bind mount 两端:Bot 和宿主 daemon 路径不同,底层目录相同。 + host_root = tmp_path / "daemon" / "shared" + host_root.mkdir(parents=True) + local_root = tmp_path / "bot" / "shared" + local_root.parent.mkdir() + local_root.symlink_to(host_root, target_is_directory=True) + monkeypatch.setenv(python_handler.SHARED_LOCAL_DIR_ENV, str(local_root)) + monkeypatch.setenv(python_handler.SHARED_HOST_DIR_ENV, str(host_root)) + mounts: list[Path] = [] + cleanup_paths: list[str] = [] + + async def run_docker(cmd: list[str], *, timeout: float) -> tuple[int, str, str]: + source = Path(cmd[cmd.index("-v") + 1].removesuffix(":/tmp")) + mounts.append(source) + assert source.parent == host_root + if "--network" not in cmd: + assert (source / "_requirements.txt").read_text() == "requests\n" + assert not (source / "_script.py").exists() + return 0, "", "" + assert (source / "_script.py").read_text() == "print('shared')" + (source / "result.txt").write_text("shared output", encoding="utf-8") + return 0, "shared\n", "" + + async def send_outputs( + files: list[str], execution_dir: str, context: dict[str, Any] + ) -> str: + assert Path(execution_dir).parent == local_root + output = python_handler._resolve_output_host_path(files[0], execution_dir) + assert output is not None + assert output.read_text() == "shared output" + return "已发送: result.txt" + + monkeypatch.setattr(python_handler, "_run_docker_command", run_docker) + monkeypatch.setattr(python_handler, "_send_output_files", send_outputs) + monkeypatch.setattr( + python_handler, "_schedule_output_dir_cleanup", cleanup_paths.append + ) + + result = await python_handler.execute( + { + "code": "print('shared')", + "libraries": ["requests"] if with_libraries else [], + "send_files": ["/tmp/result.txt"], + }, + {}, + ) + + assert result == "shared\n\n已发送: result.txt" + assert len(mounts) == (2 if with_libraries else 1) + assert len(set(mounts)) == 1 + assert len(cleanup_paths) == 1 + await python_handler._cleanup_output_dir_later(cleanup_paths[0], 0) + assert not mounts[0].exists() + + +@pytest.mark.parametrize("missing", ("local", "host")) +def test_shared_directory_requires_both_paths( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, missing: str +) -> None: + name = ( + python_handler.SHARED_HOST_DIR_ENV + if missing == "local" + else python_handler.SHARED_LOCAL_DIR_ENV + ) + monkeypatch.setenv(name, str(tmp_path / "shared")) + + with pytest.raises(ValueError, match="必须同时配置"): + python_handler._create_execution_dirs() + assert not (tmp_path / "shared").exists() From 5f134c90bef80f3e5d9297806f44cf85c474b1b6 Mon Sep 17 00:00:00 2001 From: Null <1708213363@qq.com> Date: Tue, 6 Oct 2026 11:29:54 +0800 Subject: [PATCH 005/121] docs(development): document the CI coverage gate --- AGENTS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AGENTS.md b/AGENTS.md index 45ed82a9..5cc69a56 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -63,7 +63,7 @@ Both main prompt variants and `res/IMPORTANT/each.md` guide replies to add usefu `group_analysis.member_activity` distinguishes recency (`member_list`), retrieved window message counts (`history`), and weighted indicators (`hybrid`, default). History-mode timestamps must come from messages inside the selected window. History aggregation uses `try_parse_message_time` and skips invalid timestamps before counting or ranking; `parse_message_time` retains its current-time fallback for existing display callers. Missing/zero `last_sent_time` means unknown, not proof of inactivity or never speaking; last-message time does not indicate current online status. See [the toolset documentation](src/Undefined/skills/toolsets/group_analysis/README.md) for reporting limits. ## Testing Guidelines -Write tests as `tests/test_.py`. Async tests use `pytest-asyncio`. Add or update coverage for behavior changes in APIs, config loading/hot reload, cognitive memory, meme or knowledge flows, and WebUI/runtime routes. If you touch `apps/undefined-console/` or `src/Undefined/webui/static/js/`, run `npm run check` in `apps/undefined-console/` in addition to the Python checks; if you touch `apps/undefined-chat/`, run `npm run check` in `apps/undefined-chat/` (it bundles Vitest unit/e2e suites, so cover changed behavior there). No fixed coverage threshold is configured, so cover touched paths well. +Write tests as `tests/test_.py`. Async tests use `pytest-asyncio`. Add or update coverage for behavior changes in APIs, config loading/hot reload, cognitive memory, meme or knowledge flows, and WebUI/runtime routes. If you touch `apps/undefined-console/` or `src/Undefined/webui/static/js/`, run `npm run check` in `apps/undefined-console/` in addition to the Python checks; if you touch `apps/undefined-chat/`, run `npm run check` in `apps/undefined-chat/` (it bundles Vitest unit/e2e suites, so cover changed behavior there). CI runs `pytest --cov --cov-report=term` with the 65% minimum configured in `pyproject.toml`; cover touched paths well in addition to satisfying this overall gate. ## Commit & Pull Request Guidelines Recent history follows Conventional Commits with optional scopes, for example `fix(webui): refine launcher return flow` and `feat(commands): add /version (/v) slash command`. Keep commit subjects imperative and concise. Keep `code/NagaAgent/` syncs separate from local feature work when possible. If you are bumping release versions, prefer `uv run python scripts/bump_version.py ` so `pyproject.toml`, `src/Undefined/__init__.py`, and the `package.json` + Tauri config of both `apps/undefined-console/` and `apps/undefined-chat/` stay in sync. For pull requests, include a short impact summary, linked issues, and the commands you ran; attach screenshots for WebUI or Tauri UI changes. From 9743776079d9bfc8bde88c794f86706b7014ed30 Mon Sep 17 00:00:00 2001 From: Null <1708213363@qq.com> Date: Tue, 6 Oct 2026 11:32:21 +0800 Subject: [PATCH 006/121] fix(chat): publish history readiness after session release --- docs/openapi.md | 2 + src/Undefined/api/routes/chat.py | 3 +- tests/test_runtime_api_chat_jobs.py | 73 +++++++++++++++++++++++++++++ 3 files changed, 77 insertions(+), 1 deletion(-) diff --git a/docs/openapi.md b/docs/openapi.md index 46d635e9..febb77dc 100644 --- a/docs/openapi.md +++ b/docs/openapi.md @@ -584,6 +584,8 @@ Undefined Chat 使用同一组 Runtime Chat 端点作为权威合同。客户端 `events` 只包含 `after` 之后的持久事件以及当前运行阶段快照。快照使用当前 `last_seq`,便于刷新或断线后以 `job_id + seq` 轮询续接,但不会推进序号或重复写入历史。 +`done`、`error` 或 `cancelled` 终态事件可能早于历史落盘与任务收尾完成。客户端应继续查询 job,直到快照的 `history_finalized = true`,再重载历史、释放会话发送锁;该字段只有在历史已完成且任务已解除会话占用时才为 `true`,与 Runtime 接受下一任务的门槛一致。详情接口与 JSON 事件接口使用相同的完成信号。 + Runtime 在同一个 job 条件锁内维护事件、顶层阶段、Agent 阶段和耗时快照,因此 JSON 查询和兼容 SSE 都应看到一致的 job 状态。 - `GET /api/v1/chat/jobs/{job_id}/events?after=` 加请求头 `Accept: text/event-stream`:兼容 SSE 订阅;SSE 帧包含 `id: `,长时间无事件时会发送 keep-alive 注释帧。 - `POST /api/v1/chat/jobs/{job_id}/cancel`:取消运行中的 job。 diff --git a/src/Undefined/api/routes/chat.py b/src/Undefined/api/routes/chat.py index 93349c4b..abc3efd6 100644 --- a/src/Undefined/api/routes/chat.py +++ b/src/Undefined/api/routes/chat.py @@ -280,7 +280,8 @@ def snapshot(self) -> dict[str, Any]: "messages": list(self.outputs), "current_agent_stages": self.current_agent_stage_snapshots(now), "current_tool_calls": self.current_tool_call_snapshots(now), - "history_finalized": self.history_finalized, + # 对外完成信号与新任务/历史修改的放行门槛一致。 + "history_finalized": self.history_finalized and self.done.is_set(), "waiting_input": None, } diff --git a/tests/test_runtime_api_chat_jobs.py b/tests/test_runtime_api_chat_jobs.py index 0da2ee17..5ebac482 100644 --- a/tests/test_runtime_api_chat_jobs.py +++ b/tests/test_runtime_api_chat_jobs.py @@ -106,6 +106,79 @@ async def _last_webchat_record(server: RuntimeAPIServer) -> dict[str, Any]: return cast(dict[str, Any], messages[-1]) +@pytest.mark.asyncio +async def test_history_finalized_is_published_only_after_job_releases_session( + monkeypatch: pytest.MonkeyPatch, +) -> None: + async def run_chat(ctx: RuntimeAPIContext, *, text: str, send_output: Any) -> str: + await send_output(42, "final") + return "chat" + + title_entered = asyncio.Event() + release_title = asyncio.Event() + + async def schedule_title(conversation_id: str) -> None: + title_entered.set() + await release_title.wait() + + server = RuntimeAPIServer(_context(), host="127.0.0.1", port=8788) + manager = server._chat_job_manager + monkeypatch.setattr(runtime_api_chat, "run_webui_chat", run_chat) + monkeypatch.setattr(manager, "maybe_schedule_title_generation", schedule_title) + job = await manager.create_job("first") + detail_request = cast( + web.Request, + cast(Any, _DummyRequest(match_info={"job_id": job.job_id}, query={})), + ) + events_request = cast( + web.Request, + cast( + Any, + _DummyRequest( + match_info={"job_id": job.job_id}, + query={"format": "json"}, + headers={"Accept": "application/json"}, + ), + ), + ) + try: + await asyncio.wait_for(title_entered.wait(), timeout=2.0) + # 真实 job 已经落盘,但最终收尾仍占用会话。 + assert job.status == "done" + assert job.history_finalized is True + assert not job.done.is_set() + assert await manager.has_running_job(job.conversation_id) + assert (await manager.snapshot(job))["history_finalized"] is False + response = await server._chat_job_detail_handler(detail_request) + assert json.loads(response.text or "{}")["history_finalized"] is False + events_response = await server._chat_job_events_handler(events_request) + assert isinstance(events_response, web.Response) + payload = json.loads(events_response.text or "{}") + assert payload["job"]["history_finalized"] is False + assert any(event["event"] == "done" for event in payload["events"]) + with pytest.raises(RuntimeError, match="still running"): + await manager.create_job("too early", job.conversation_id) + + release_title.set() + assert job.task is not None + await asyncio.wait_for(job.task, timeout=2.0) + assert job.done.is_set() + assert not await manager.has_running_job(job.conversation_id) + response = await server._chat_job_detail_handler(detail_request) + assert json.loads(response.text or "{}")["history_finalized"] is True + events_response = await server._chat_job_events_handler(events_request) + assert isinstance(events_response, web.Response) + assert ( + json.loads(events_response.text or "{}")["job"]["history_finalized"] is True + ) + next_job = await manager.create_job("next", job.conversation_id) + assert next_job.task is not None + await asyncio.wait_for(next_job.task, timeout=2.0) + finally: + release_title.set() + await manager.stop() + + async def _store_runtime_attachment( attachment_id: str = "attachment123", *, From 3ffaaf4beddbfadec608de67e98a81d38c94422f Mon Sep 17 00:00:00 2001 From: Null <1708213363@qq.com> Date: Tue, 6 Oct 2026 11:34:28 +0800 Subject: [PATCH 007/121] fix(chat): align native pagination and preserve job recovery state --- .../src-tauri/src/native_tests.rs | 24 +- .../src-tauri/src/runtime_client.rs | 5 +- .../src/chat-store/runtime-contract.test.ts | 472 ++++++++++++++++++ .../src/chat-store/store.test.ts | 2 + apps/undefined-chat/src/chat-store/store.ts | 174 +++++-- docs/undefined-chat.md | 6 +- 6 files changed, 627 insertions(+), 56 deletions(-) create mode 100644 apps/undefined-chat/src/chat-store/runtime-contract.test.ts diff --git a/apps/undefined-chat/src-tauri/src/native_tests.rs b/apps/undefined-chat/src-tauri/src/native_tests.rs index 9c81f7f4..78e50909 100644 --- a/apps/undefined-chat/src-tauri/src/native_tests.rs +++ b/apps/undefined-chat/src-tauri/src/native_tests.rs @@ -9,7 +9,7 @@ use crate::preview::{ }; use crate::runtime_client::{ build_runtime_url, job_events_url, parse_sse_chunks, runtime_health_from_body_result, - RuntimeRequestInput, StartJobEventStreamInput, Utf8ChunkDecoder, + ConversationHistoryInput, RuntimeRequestInput, StartJobEventStreamInput, Utf8ChunkDecoder, }; use crate::secret::{ api_key_status_from_storage, classify_secret_storage, derive_stronghold_key, @@ -27,6 +27,28 @@ use crate::{ }, }; +#[test] +fn history_input_accepts_runtime_numeric_cursors() { + for before in [0, 70] { + let input: ConversationHistoryInput = serde_json::from_value(serde_json::json!({ + "conversationId": "default", + "limit": 50, + "before": before, + })) + .unwrap(); + assert_eq!(input.conversation_id, "default"); + assert_eq!(input.limit, Some(50)); + assert_eq!(input.before, Some(before)); + } + for input in [ + serde_json::json!({"conversationId": "default"}), + serde_json::json!({"conversationId": "default", "before": null}), + ] { + let input: ConversationHistoryInput = serde_json::from_value(input).unwrap(); + assert_eq!(input.before, None); + } +} + #[test] fn normalize_runtime_url_removes_trailing_slashes() { let value = normalize_runtime_url("http://127.0.0.1:8788///").unwrap(); diff --git a/apps/undefined-chat/src-tauri/src/runtime_client.rs b/apps/undefined-chat/src-tauri/src/runtime_client.rs index 920efcdb..5f129464 100644 --- a/apps/undefined-chat/src-tauri/src/runtime_client.rs +++ b/apps/undefined-chat/src-tauri/src/runtime_client.rs @@ -92,7 +92,7 @@ pub struct ConversationHistoryInput { #[serde(default)] pub limit: Option, #[serde(default)] - pub before: Option, + pub before: Option, } #[derive(Debug, Clone, Deserialize)] @@ -495,8 +495,7 @@ pub async fn get_history( path.push_str(&format!("&limit={limit}")); } if let Some(before) = input.before { - path.push_str("&before="); - path.push_str(&urlencoding::encode(&before)); + path.push_str(&format!("&before={before}")); } request_json_command( &app, diff --git a/apps/undefined-chat/src/chat-store/runtime-contract.test.ts b/apps/undefined-chat/src/chat-store/runtime-contract.test.ts new file mode 100644 index 00000000..15d822ef --- /dev/null +++ b/apps/undefined-chat/src/chat-store/runtime-contract.test.ts @@ -0,0 +1,472 @@ +import { invoke } from "@tauri-apps/api/core"; +import { afterEach, beforeEach, describe, expect, test, vi } from "vitest"; +import { createTauriRuntimeClient } from "../runtime-client/tauri"; +import { event, historyItem, job, runtimeClientStub } from "../test-fixtures"; +import { createChatStore } from "./store"; + +vi.mock("@tauri-apps/api/core", () => ({ invoke: vi.fn() })); +vi.mock("@tauri-apps/api/event", () => ({ listen: vi.fn() })); + +function runtimeJob(overrides: Record = {}) { + return { + job_id: "job-1", + conversation_id: "default", + status: "running", + reply: "", + messages: [], + last_seq: 0, + current_tool_calls: [], + history_finalized: false, + ...overrides, + }; +} + +describe("Runtime job contract through the native client", () => { + beforeEach(() => { + vi.resetAllMocks(); + vi.useFakeTimers(); + }); + afterEach(() => { + vi.clearAllTimers(); + vi.useRealTimers(); + }); + + test("fallback preserves the timeline and completed calls without double counting snapshot text", async () => { + const nativeClient = createTauriRuntimeClient(); + const running = job(); + vi.mocked(invoke).mockResolvedValue({ + status: 200, + ok: true, + body: { + job: runtimeJob({ + reply: "你好\n\n世界", + messages: ["你好", "世界"], + last_seq: 4, + }), + after: 3, + last_seq: 4, + events: [ + { + seq: 4, + event: "message", + payload: { + job_id: "job-1", + conversation_id: "default", + content: "世界", + }, + }, + ], + }, + }); + const client = runtimeClientStub({ + getActiveJobs: vi.fn(async () => ({ job: running, jobs: [running] })), + fetchJobEventsJson: nativeClient.fetchJobEventsJson, + }); + const store = createChatStore({ client }); + await store.bootstrap(); + store.applyRuntimeEvents("job-1", [ + event({ seq: 1, event: "message", payload: { content: "你好" } }), + event({ + seq: 2, + event: "tool_start", + payload: { webchat_call_id: "search-1", name: "search" }, + }), + event({ + seq: 3, + event: "tool_end", + payload: { + webchat_call_id: "search-1", + ok: true, + result_preview: "结果", + }, + }), + ]); + store.handleRuntimeStatus({ + subscriptionId: "sub-1", + jobId: "job-1", + status: "error", + detail: "SSE disconnected", + }); + await vi.waitFor(() => { + expect(store.getSnapshot().activeJobsByConversation.default?.reply).toBe( + "你好\n\n世界", + ); + }); + const active = store.getSnapshot().activeJobsByConversation.default; + expect(active?.currentTimeline).toEqual([ + { type: "message", seq: 1, content: "你好" }, + { type: "call", seq: 2, callId: "search-1" }, + { type: "message", seq: 4, content: "世界" }, + ]); + expect(active?.currentToolCalls[0]).toMatchObject({ + id: "search-1", + status: "done", + resultPreview: "结果", + }); + expect(invoke).toHaveBeenCalledWith("fetch_job_events_json", { + input: { jobId: "job-1", afterSeq: 3, conversationId: "default" }, + }); + await vi.advanceTimersByTimeAsync(1000); + expect(store.getSnapshot().activeJobsByConversation.default?.reply).toBe( + "你好\n\n世界", + ); + expect( + store.getSnapshot().activeJobsByConversation.default?.currentTimeline, + ).toEqual(active?.currentTimeline); + }); + + test.each(["sse", "json"])( + "%s terminal delivery waits for history_finalized before releasing the job and reloading history", + async (transport) => { + const nativeClient = createTauriRuntimeClient(); + const running = job(); + const terminalEvent = { + seq: 1, + event: "done", + payload: { + job_id: "job-1", + conversation_id: "default", + status: "done", + }, + }; + vi.mocked(invoke) + .mockResolvedValueOnce({ + status: 200, + ok: true, + body: { + job: runtimeJob({ status: "done", last_seq: 1 }), + events: [terminalEvent], + }, + }) + .mockResolvedValueOnce({ + status: 200, + ok: true, + body: { + job: runtimeJob({ + status: "done", + last_seq: 1, + history_finalized: true, + }), + events: [], + }, + }); + const client = runtimeClientStub({ + getActiveJobs: vi.fn(async () => ({ job: running, jobs: [running] })), + fetchJobEventsJson: nativeClient.fetchJobEventsJson, + }); + const store = createChatStore({ client }); + await store.bootstrap(); + if (transport === "sse") { + store.applyRuntimeEvents("job-1", [terminalEvent]); + await vi.advanceTimersByTimeAsync(1000); + } else { + store.handleRuntimeStatus({ + subscriptionId: "sub-1", + jobId: "job-1", + status: "error", + detail: "closed", + }); + await vi.waitFor(() => expect(invoke).toHaveBeenCalledTimes(1)); + } + expect(client.getHistory).toHaveBeenCalledTimes(1); + expect(store.getSnapshot().jobConversationById["job-1"]).toBe("default"); + expect( + store.getSnapshot().activeJobsByConversation.default?.historyFinalized, + ).toBe(false); + store.updateDraft("default", "下一条"); + await store.sendSelectedMessage(); + expect(client.sendMessage).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(1000); + expect(client.getHistory).toHaveBeenCalledTimes(2); + expect(client.stopJobEventStream).toHaveBeenCalledWith("sub-1"); + expect( + store.getSnapshot().activeJobsByConversation.default, + ).toBeUndefined(); + expect(store.getSnapshot().jobConversationById["job-1"]).toBeUndefined(); + expect(store.getSnapshot().connectionState).toBe("connected"); + }, + ); + + test("an accepted message keeps its history and empty draft when native SSE startup rejects", async () => { + const nativeClient = createTauriRuntimeClient(); + vi.mocked(invoke).mockImplementation(async (command) => { + if (command === "start_job_event_stream") { + throw new Error("native subscription failed"); + } + return { + status: command === "send_message" ? 202 : 200, + ok: true, + body: + command === "send_message" + ? runtimeJob() + : { job: runtimeJob(), events: [] }, + }; + }); + const client = runtimeClientStub({ + sendMessage: nativeClient.sendMessage, + startJobEventStream: nativeClient.startJobEventStream, + fetchJobEventsJson: nativeClient.fetchJobEventsJson, + }); + const store = createChatStore({ client }); + await store.bootstrap(); + store.updateDraft("default", "已经发送"); + await store.sendSelectedMessage(); + await vi.waitFor(() => + expect(invoke).toHaveBeenCalledWith("fetch_job_events_json", { + input: { jobId: "job-1", afterSeq: 0, conversationId: "default" }, + }), + ); + expect(store.getSnapshot().draftsByConversation.default).toBe(""); + expect(store.getSnapshot().sendError).toBeNull(); + expect(store.getSnapshot().connectionState).toBe("json_fallback"); + expect(store.getSnapshot().historyByConversation.default?.items).toEqual([ + expect.objectContaining( + historyItem({ + content: "已经发送", + timestamp: expect.any(String), + messageId: expect.any(String), + }), + ), + ]); + expect(store.getSnapshot().activeJobsByConversation.default?.jobId).toBe( + "job-1", + ); + }); + + test.each(["response", "error"])( + "ignores a late old-job %s after finalization and a new send", + async (completion) => { + const nativeClient = createTauriRuntimeClient(); + let resolveOld!: (value: unknown) => void; + let rejectOld!: (reason: Error) => void; + const pending = new Promise((resolve, reject) => { + resolveOld = resolve; + rejectOld = reject; + }); + vi.mocked(invoke) + .mockReturnValueOnce(pending) + .mockResolvedValueOnce({ + status: 200, + ok: true, + body: { + job: runtimeJob({ + status: "done", + last_seq: 1, + history_finalized: true, + }), + events: [], + }, + }); + const running = job(); + const client = runtimeClientStub({ + getActiveJobs: vi.fn(async () => ({ job: running, jobs: [running] })), + fetchJobEventsJson: nativeClient.fetchJobEventsJson, + sendMessage: vi.fn(async () => job({ jobId: "job-new" })), + }); + const store = createChatStore({ client }); + await store.bootstrap(); + const broken = { + subscriptionId: "sub-1", + jobId: "job-1", + status: "error", + detail: "closed", + }; + store.handleRuntimeStatus(broken); + store.handleRuntimeStatus(broken); + await vi.waitFor(() => + expect( + store.getSnapshot().activeJobsByConversation.default, + ).toBeUndefined(), + ); + store.updateDraft("default", "新任务"); + await store.sendSelectedMessage(); + if (completion === "response") { + resolveOld({ + status: 200, + ok: true, + body: { job: runtimeJob(), events: [] }, + }); + } else { + rejectOld(new Error("old request failed")); + } + await vi.advanceTimersByTimeAsync(0); + expect(store.getSnapshot().activeJobsByConversation.default?.jobId).toBe( + "job-new", + ); + expect(store.getSnapshot().jobConversationById["job-1"]).toBeUndefined(); + expect(store.getSnapshot().connectionState).toBe("streaming"); + store.applyRuntimeEvents("job-1", [ + { seq: 2, event: "done", payload: { conversation_id: "default" } }, + ]); + expect(store.getSnapshot().activeJobsByConversation.default?.jobId).toBe( + "job-new", + ); + }, + ); + + test("an older concurrent response cannot replace the latest same-job snapshot", async () => { + const nativeClient = createTauriRuntimeClient(); + let resolveOld!: (value: unknown) => void; + const pending = new Promise((resolve) => { + resolveOld = resolve; + }); + vi.mocked(invoke) + .mockReturnValueOnce(pending) + .mockResolvedValueOnce({ + status: 200, + ok: true, + body: { + job: runtimeJob({ reply: "已有\n\n新增", last_seq: 2 }), + events: [{ seq: 2, event: "message", payload: { content: "新增" } }], + }, + }); + const running = job(); + const client = runtimeClientStub({ + getActiveJobs: vi.fn(async () => ({ job: running, jobs: [running] })), + fetchJobEventsJson: nativeClient.fetchJobEventsJson, + }); + const store = createChatStore({ client }); + await store.bootstrap(); + store.applyRuntimeEvents("job-1", [ + { seq: 1, event: "message", payload: { content: "已有" } }, + ]); + const broken = { + subscriptionId: "sub-1", + jobId: "job-1", + status: "error", + detail: "closed", + }; + store.handleRuntimeStatus(broken); + store.handleRuntimeStatus(broken); + await vi.waitFor(() => + expect(store.getSnapshot().eventCursorByJob["job-1"]).toBe(2), + ); + const latest = store.getSnapshot().activeJobsByConversation.default; + resolveOld({ + status: 200, + ok: true, + body: { job: runtimeJob({ reply: "已有", last_seq: 1 }), events: [] }, + }); + await vi.advanceTimersByTimeAsync(0); + expect(store.getSnapshot().activeJobsByConversation.default).toEqual( + latest, + ); + expect(store.getSnapshot().eventCursorByJob["job-1"]).toBe(2); + }); + + test("a snapshot overtaken by SSE keeps the consumed cursor and polls again", async () => { + const nativeClient = createTauriRuntimeClient(); + let resolveOld!: (value: unknown) => void; + const pending = new Promise((resolve) => { + resolveOld = resolve; + }); + vi.mocked(invoke) + .mockReturnValueOnce(pending) + .mockResolvedValueOnce({ + status: 200, + ok: true, + body: { + job: runtimeJob({ reply: "已有\n\n新增", last_seq: 2 }), + events: [], + }, + }); + const running = job(); + const client = runtimeClientStub({ + getActiveJobs: vi.fn(async () => ({ job: running, jobs: [running] })), + fetchJobEventsJson: nativeClient.fetchJobEventsJson, + }); + const store = createChatStore({ client }); + await store.bootstrap(); + store.applyRuntimeEvents("job-1", [ + { seq: 1, event: "message", payload: { content: "已有" } }, + ]); + store.handleRuntimeStatus({ + subscriptionId: "sub-1", + jobId: "job-1", + status: "error", + detail: "closed", + }); + store.applyRuntimeEvents("job-1", [ + { seq: 2, event: "message", payload: { content: "新增" } }, + ]); + const latest = store.getSnapshot().activeJobsByConversation.default; + resolveOld({ + status: 200, + ok: true, + body: { job: runtimeJob({ reply: "已有", last_seq: 1 }), events: [] }, + }); + await vi.advanceTimersByTimeAsync(0); + expect(store.getSnapshot().activeJobsByConversation.default).toEqual( + latest, + ); + expect(store.getSnapshot().eventCursorByJob["job-1"]).toBe(2); + await vi.advanceTimersByTimeAsync(1000); + expect(invoke).toHaveBeenLastCalledWith("fetch_job_events_json", { + input: { jobId: "job-1", afterSeq: 2, conversationId: "default" }, + }); + expect(store.getSnapshot().activeJobsByConversation.default?.reply).toBe( + "已有\n\n新增", + ); + expect( + store.getSnapshot().activeJobsByConversation.default?.currentTimeline, + ).toEqual(latest?.currentTimeline); + }); + + test("cancellation keeps the real event cursor so its finalized snapshot can finish", async () => { + const nativeClient = createTauriRuntimeClient(); + vi.mocked(invoke) + .mockResolvedValueOnce({ + status: 200, + ok: true, + body: runtimeJob({ status: "cancelled", last_seq: 4 }), + }) + .mockResolvedValueOnce({ + status: 200, + ok: true, + body: { + job: runtimeJob({ + status: "cancelled", + last_seq: 4, + history_finalized: true, + }), + events: [ + { + seq: 4, + event: "error", + payload: { + job_id: "job-1", + conversation_id: "default", + error: "cancelled", + }, + }, + ], + }, + }); + const running = job(); + const client = runtimeClientStub({ + getActiveJobs: vi.fn(async () => ({ job: running, jobs: [running] })), + cancelJob: nativeClient.cancelJob, + fetchJobEventsJson: nativeClient.fetchJobEventsJson, + }); + const store = createChatStore({ client }); + await store.bootstrap(); + store.applyRuntimeEvents("job-1", [ + { seq: 3, event: "message", payload: { content: "取消前回复" } }, + ]); + await store.cancelJob("job-1"); + expect(store.getSnapshot().eventCursorByJob["job-1"]).toBe(3); + expect(store.getSnapshot().activeJobsByConversation.default?.status).toBe( + "cancelled", + ); + expect(client.getHistory).toHaveBeenCalledTimes(1); + await vi.advanceTimersByTimeAsync(1000); + expect(invoke).toHaveBeenLastCalledWith("fetch_job_events_json", { + input: { jobId: "job-1", afterSeq: 3, conversationId: "default" }, + }); + expect( + store.getSnapshot().activeJobsByConversation.default, + ).toBeUndefined(); + expect(store.getSnapshot().eventCursorByJob["job-1"]).toBe(4); + expect(client.getHistory).toHaveBeenCalledTimes(2); + }); +}); diff --git a/apps/undefined-chat/src/chat-store/store.test.ts b/apps/undefined-chat/src/chat-store/store.test.ts index aeb4f08a..9e2f804e 100644 --- a/apps/undefined-chat/src/chat-store/store.test.ts +++ b/apps/undefined-chat/src/chat-store/store.test.ts @@ -383,6 +383,7 @@ describe("chat store", () => { jobId: "job-1", conversationId: "default", status: "done", + historyFinalized: true, lastSeq: 2, }); const client = runtimeClientStub({ @@ -455,6 +456,7 @@ describe("chat store", () => { jobId: "job-1", conversationId: "default", status: "cancelled", + historyFinalized: true, lastSeq: 4, }); const client = runtimeClientStub({ diff --git a/apps/undefined-chat/src/chat-store/store.ts b/apps/undefined-chat/src/chat-store/store.ts index 6202eebc..184cdee2 100644 --- a/apps/undefined-chat/src/chat-store/store.ts +++ b/apps/undefined-chat/src/chat-store/store.ts @@ -168,7 +168,9 @@ function withoutConversationJob( export function isJobRunning(job: ChatJob | null | undefined): boolean { return Boolean( - job && !["done", "error", "cancelled"].includes(job.status.toLowerCase()), + job && + (!["done", "error", "cancelled"].includes(job.status.toLowerCase()) || + !job.historyFinalized), ); } @@ -672,6 +674,8 @@ export function createChatStore({ const fallbackTimersByJob = new Map>(); /** 每个 job 当前的 JSON 轮询退避间隔(ms),随连续失败指数增长、成功后重置 */ const fallbackBackoffByJob = new Map(); + const latestFallbackRequestByJob = new Map(); + let fallbackRequestGeneration = 0; let unlistenRuntimeSse: (() => void) | null = null; /** bootstrap 重入保护:已在进行中的 bootstrap promise,避免并发重入导致 SSE 监听泄漏 */ let inFlightBootstrap: Promise | null = null; @@ -706,6 +710,7 @@ export function createChatStore({ clearTimeout(timer); } fallbackTimersByJob.clear(); + latestFallbackRequestByJob.clear(); } async function loadHistory(conversationId: string): Promise { @@ -737,13 +742,18 @@ export function createChatStore({ if (subscriptionsByJob.has(job.jobId)) { return; } - const subscription = await client.startJobEventStream({ - jobId: job.jobId, - afterSeq: state.eventCursorByJob[job.jobId] ?? job.lastSeq, - conversationId: job.conversationId, - }); - if (subscription.subscriptionId) { - subscriptionsByJob.set(job.jobId, subscription.subscriptionId); + try { + const subscription = await client.startJobEventStream({ + jobId: job.jobId, + afterSeq: state.eventCursorByJob[job.jobId] ?? job.lastSeq, + conversationId: job.conversationId, + }); + if (subscription.subscriptionId) { + subscriptionsByJob.set(job.jobId, subscription.subscriptionId); + } + } catch { + // job 已建立;订阅失败只切换传输方式,不能回滚成功发送的输入。 + fetchJobEventsFallback(job.jobId); } } @@ -754,6 +764,25 @@ export function createChatStore({ fallbackTimersByJob.delete(jobId); } fallbackBackoffByJob.delete(jobId); + latestFallbackRequestByJob.delete(jobId); + } + + function finishJob(jobId: string, conversationId: string): void { + const active = state.activeJobsByConversation[conversationId]; + if (active && active.jobId !== jobId) { + return; + } + stopFallbackPolling(jobId); + const subscriptionId = subscriptionsByJob.get(jobId); + if (subscriptionId) { + subscriptionsByJob.delete(jobId); + void client.stopJobEventStream(subscriptionId); + } + dispatch({ type: "job/remove", conversationId, jobId }); + if (Object.keys(state.activeJobsByConversation).length === 0) { + dispatch({ type: "connection/set", connectionState: "connected" }); + } + void loadHistory(conversationId); } /** 计算并推进某个 job 的下一次轮询退避间隔(指数增长,封顶 JSON_FALLBACK_MAX_MS) */ @@ -779,6 +808,18 @@ export function createChatStore({ } function fetchJobEventsFallback(jobId: string): void { + const conversationId = state.jobConversationById[jobId]; + if ( + !conversationId || + state.activeJobsByConversation[conversationId]?.jobId !== jobId + ) { + return; + } + const requestGeneration = ++fallbackRequestGeneration; + latestFallbackRequestByJob.set(jobId, requestGeneration); + const isCurrentRequest = (): boolean => + latestFallbackRequestByJob.get(jobId) === requestGeneration && + state.activeJobsByConversation[conversationId]?.jobId === jobId; const timer = fallbackTimersByJob.get(jobId); if (timer) { clearTimeout(timer); @@ -789,29 +830,51 @@ export function createChatStore({ .fetchJobEventsJson({ jobId, afterSeq: state.eventCursorByJob[jobId] ?? 0, - conversationId: state.jobConversationById[jobId], + conversationId, }) .then((response) => { + if (!isCurrentRequest()) { + return; + } + const active = state.activeJobsByConversation[conversationId]; + const consumedSeq = Math.max( + state.eventCursorByJob[jobId] ?? 0, + active?.lastSeq ?? 0, + ); + if (response.job.lastSeq < consumedSeq) { + // SSE 可能在请求途中推进;旧快照不能回退正文或阶段,继续拉取即可。 + scheduleFallbackPolling(jobId); + return; + } // 拉取成功:重置退避间隔 fallbackBackoffByJob.delete(jobId); - dispatch({ type: "job/upsert", job: response.job }); + // 先续接事件保留此前时间线,再应用权威正文快照,避免增量重复计数。 applyRuntimeEvents(jobId, response.events); - const hasTerminalEvent = response.events.some((item) => - TERMINAL_EVENTS.has(item.event), - ); + const replayedJob = + state.activeJobsByConversation[response.job.conversationId]; + dispatch({ + type: "job/upsert", + job: { + ...response.job, + currentTimeline: replayedJob?.currentTimeline ?? [], + // Runtime 运行快照只含未结束调用,完整调用树由事件回放保留。 + currentToolCalls: + replayedJob?.currentToolCalls ?? response.job.currentToolCalls, + }, + }); if (isJobRunning(response.job)) { scheduleFallbackPolling(jobId, JSON_FALLBACK_BASE_MS); - } else if (!hasTerminalEvent && response.job.conversationId) { - void loadHistory(response.job.conversationId); - dispatch({ type: "connection/set", connectionState: "connected" }); + } else if (response.job.conversationId) { + finishJob(jobId, response.job.conversationId); } }) .catch(() => { + if (!isCurrentRequest()) { + return; + } // 单次失败不直接放弃:仅当该 job 仍是活跃任务时按指数退避继续重试, // 否则停止轮询。连接状态保持 json_fallback(仍在尝试),而非直接 disconnected。 - const conversationId = state.jobConversationById[jobId]; const stillActive = - conversationId != null && state.activeJobsByConversation[conversationId]?.jobId === jobId; if (stillActive) { scheduleFallbackPolling(jobId, nextFallbackDelay(jobId)); @@ -891,7 +954,10 @@ export function createChatStore({ if (selected) { await loadHistory(selected); } - if (activeJobs.jobs.length > 0) { + if ( + activeJobs.jobs.length > 0 && + state.connectionState !== "json_fallback" + ) { dispatch({ type: "connection/set", connectionState: "streaming" }); } } catch (err) { @@ -1006,16 +1072,6 @@ export function createChatStore({ dispatch({ type: "draft/set", conversationId, draft }); } - function latestEventSeq(jobId: string): number { - const cursor = state.eventCursorByJob[jobId] ?? 0; - const events = state.eventsByJob[jobId] ?? []; - const lastEventSeq = events.reduce( - (maxSeq, item) => Math.max(maxSeq, item.seq), - 0, - ); - return Math.max(cursor, lastEventSeq); - } - async function sendSelectedMessage(): Promise { // in-flight 锁:从点击到 job 建立期间阻止重复发送, // 避免快速双击重复插入乐观消息 / 重复请求后端 @@ -1115,19 +1171,29 @@ export function createChatStore({ async function cancelJob(jobId: string): Promise { const conversationId = state.jobConversationById[jobId]; + if (!conversationId) { + return; + } try { const cancelled = await client.cancelJob(jobId); - dispatch({ type: "job/upsert", job: cancelled }); - applyRuntimeEvents(jobId, [ - { - seq: Math.max(latestEventSeq(jobId), cancelled.lastSeq) + 1, - event: "cancelled", - payload: { - job_id: jobId, - conversation_id: cancelled.conversationId || conversationId, - }, + const current = state.activeJobsByConversation[conversationId]; + if (current?.jobId !== jobId) { + return; + } + dispatch({ + type: "job/upsert", + job: { + ...cancelled, + currentTimeline: current.currentTimeline, + currentToolCalls: current.currentToolCalls, }, - ]); + }); + // 取消响应不是增量事件,不伪造序号;真正终态由 Runtime 游标续接。 + if (isJobRunning(cancelled)) { + scheduleFallbackPolling(jobId); + } else { + finishJob(jobId, conversationId); + } } catch (err) { dispatch({ type: "send/error", error: errorMessage(err) }); } @@ -1432,22 +1498,28 @@ export function createChatStore({ } } - // 终态事件:清理 job 并重载历史 - const doneEvent = events.find((item) => - ["done", "error", "cancelled"].includes(item.event), + // 终态事件先保留会话和展示状态,确认 Runtime 历史落盘后再释放发送锁。 + const doneEvent = events.find( + (item) => item.seq > prevCursor && TERMINAL_EVENTS.has(item.event), ); if (doneEvent && conversationId) { - stopFallbackPolling(jobId); - const subscriptionId = subscriptionsByJob.get(jobId); - if (subscriptionId) { - subscriptionsByJob.delete(jobId); - void client.stopJobEventStream(subscriptionId); + const currentJob = state.activeJobsByConversation[conversationId]; + if (currentJob && currentJob.jobId !== jobId) { + return; } - dispatch({ type: "job/remove", conversationId, jobId }); - if (Object.keys(state.activeJobsByConversation).length === 0) { - dispatch({ type: "connection/set", connectionState: "connected" }); + if (currentJob && !currentJob.historyFinalized) { + const status = + doneEvent.payload.error === "cancelled" + ? "cancelled" + : doneEvent.event; + dispatch({ + type: "job/upsert", + job: { ...currentJob, status }, + }); + scheduleFallbackPolling(jobId); + } else { + finishJob(jobId, conversationId); } - void loadHistory(conversationId); } } diff --git a/docs/undefined-chat.md b/docs/undefined-chat.md index 557b7b94..44612daf 100644 --- a/docs/undefined-chat.md +++ b/docs/undefined-chat.md @@ -57,7 +57,11 @@ Runtime 请求必须由 Tauri command 拼接到已配置 origin 下,避免 Rea 事件策略: - 默认使用 SSE 订阅 job events。 -- SSE error/closed 时使用 `job_id + seq` 调用 JSON events fallback 补齐遗漏事件。 +- SSE error/closed 或原生订阅建立失败时使用 `job_id + seq` 调用 JSON events fallback 补齐遗漏事件;消息已成功创建 job 后,订阅失败不会撤回用户消息或恢复已发送草稿。 +- JSON fallback 先续接增量事件,再应用 Runtime 的完整正文与阶段快照,保留已有消息/工具时间线和已结束调用,避免把快照正文再次累计。 +- 重叠 JSON 请求按 job 身份和请求代次校验;已结束、已被替换的 job 或落后于已消费事件游标的回包会被忽略,避免迟到响应覆盖下一轮任务或回退回复内容。 +- `done` / `error` / `cancelled` 事件到达后,客户端继续保留当前会话的 job 和发送锁,通过 JSON polling 等待 `history_finalized=true`,确认历史落盘后再刷新历史并清理订阅。 +- 历史分页的 `before` / `next_before` 是非负数值索引;React 与 Tauri command 使用同一数值契约,首屏不传 `before`。 - Android 回到前台时执行 `store.bootstrap()`,刷新配置、会话、当前历史页和 active jobs,并重新建立事件订阅。 - 当前没有公开的 store 方法在 resume 时逐 job 主动补齐;这部分仍依赖 SSE 断线 fallback。 From 8d6ca8dc73f41dae6bd89c36a111b013a7cd5760 Mon Sep 17 00:00:00 2001 From: Null <1708213363@qq.com> Date: Tue, 6 Oct 2026 11:35:32 +0800 Subject: [PATCH 008/121] fix(python-interpreter): clean directories created during cancellation --- src/Undefined/skills/shared.py | 12 +++- .../skills/tools/python_interpreter/README.md | 2 +- .../tools/python_interpreter/handler.py | 16 ++++- tests/test_python_interpreter_handler.py | 71 +++++++++++++++++++ 4 files changed, 97 insertions(+), 4 deletions(-) diff --git a/src/Undefined/skills/shared.py b/src/Undefined/skills/shared.py index 4c12d647..302dc96c 100644 --- a/src/Undefined/skills/shared.py +++ b/src/Undefined/skills/shared.py @@ -7,13 +7,23 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any +from collections.abc import Coroutine +from typing import TYPE_CHECKING, Any, TypeVar if TYPE_CHECKING: from jmcpy import Genre, SearchTarget, SortBy, SubGenre, TimeRange from Undefined.jm.searcher import JmListingMode +_ResultT = TypeVar("_ResultT") + + +async def run_cancellation_safe(operation: Coroutine[Any, Any, _ResultT]) -> _ResultT: + """复用公共 IO 的取消保护,等待临界区结束后再传播取消。""" + from Undefined.utils.io import run_cancellation_safe as run_io_cancellation_safe + + return await run_io_cancellation_safe(operation) + def private_access_error( runtime_config: Any, diff --git a/src/Undefined/skills/tools/python_interpreter/README.md b/src/Undefined/skills/tools/python_interpreter/README.md index 92ada344..11a9a8fd 100644 --- a/src/Undefined/skills/tools/python_interpreter/README.md +++ b/src/Undefined/skills/tools/python_interpreter/README.md @@ -27,7 +27,7 @@ Bot 本身在容器中且挂载宿主 Docker socket 时,Docker 的 bind mount 手工部署时,同时设置这两个变量为对应的绝对路径,并保证指向同一份共享目录。工具在 Bot 侧创建独立工作目录、写入脚本,向 Docker 传入对应宿主路径,再从 Bot 侧读取输出并清理。不会挂载整个宿主 `/tmp`。更换仓库位置后应重新生成部署配置,避免保留旧的宿主绝对路径。 -执行目录在完成后清理;需要回传文件时会保留 30 秒,让附件发送完成。 +执行目录在完成后清理;需要回传文件时会保留 30 秒,让附件发送完成。创建目录期间收到取消时,会先等待后台创建线程结束,再清理本次目录并传播取消,避免遗留临时工作目录。 目录结构: - `config.json`:工具定义 diff --git a/src/Undefined/skills/tools/python_interpreter/handler.py b/src/Undefined/skills/tools/python_interpreter/handler.py index 7fdb57e9..3f6c2f60 100644 --- a/src/Undefined/skills/tools/python_interpreter/handler.py +++ b/src/Undefined/skills/tools/python_interpreter/handler.py @@ -8,6 +8,8 @@ from pathlib import Path, PurePosixPath from typing import Any, Dict +from Undefined.skills.shared import run_cancellation_safe + logger = logging.getLogger(__name__) # Docker 执行配置 @@ -197,9 +199,17 @@ async def execute(args: Dict[str, Any], context: Dict[str, Any]) -> str: execution_dir: str | None = None defer_cleanup = False - try: + async def prepare_execution_dirs() -> tuple[str, str]: + nonlocal execution_dir local_tmpdir, docker_tmpdir = await asyncio.to_thread(_create_execution_dirs) + # 在线程完成后、取消传播前记录所有权,finally 才能清掉创建中的目录。 execution_dir = local_tmpdir + return local_tmpdir, docker_tmpdir + + try: + local_tmpdir, docker_tmpdir = await run_cancellation_safe( + prepare_execution_dirs() + ) for fpath in send_files: if _resolve_output_host_path(fpath, local_tmpdir) is None: return f"错误: 输出文件路径必须位于容器 /tmp 目录内: '{fpath}'" @@ -287,7 +297,9 @@ async def execute(args: Dict[str, Any], context: Dict[str, Any]) -> str: if defer_cleanup: _schedule_output_dir_cleanup(execution_dir) else: - await asyncio.to_thread(shutil.rmtree, execution_dir, True) + await run_cancellation_safe( + asyncio.to_thread(shutil.rmtree, execution_dir, True) + ) async def _send_output_files( diff --git a/tests/test_python_interpreter_handler.py b/tests/test_python_interpreter_handler.py index 6ba24d52..9edf1fe8 100644 --- a/tests/test_python_interpreter_handler.py +++ b/tests/test_python_interpreter_handler.py @@ -1,8 +1,10 @@ from __future__ import annotations import asyncio +import contextlib import os import tempfile +import threading from pathlib import Path from typing import Any from unittest.mock import AsyncMock, MagicMock @@ -252,3 +254,72 @@ def test_shared_directory_requires_both_paths( with pytest.raises(ValueError, match="必须同时配置"): python_handler._create_execution_dirs() assert not (tmp_path / "shared").exists() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("shared_directory", (False, True)) +async def test_cancel_during_directory_creation_waits_and_removes_directory( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + shared_directory: bool, +) -> None: + started = threading.Event() + release = threading.Event() + created_dirs: list[Path] = [] + real_mkdtemp = tempfile.mkdtemp + + def controlled_mkdtemp(prefix: str, dir: str | Path | None = None) -> str: + name = real_mkdtemp(prefix=prefix, dir=dir if dir is not None else tmp_path) + path = Path(name) + (path / "created.txt").write_text( + "created before cancellation", encoding="utf-8" + ) + created_dirs.append(path) + started.set() + if not release.wait(5): + raise RuntimeError("test did not release directory creation") + return name + + if shared_directory: + root = tmp_path / "shared" + monkeypatch.setenv(python_handler.SHARED_LOCAL_DIR_ENV, str(root)) + monkeypatch.setenv(python_handler.SHARED_HOST_DIR_ENV, str(root)) + else: + monkeypatch.delenv(python_handler.SHARED_LOCAL_DIR_ENV, raising=False) + monkeypatch.delenv(python_handler.SHARED_HOST_DIR_ENV, raising=False) + monkeypatch.setattr(tempfile, "mkdtemp", controlled_mkdtemp) + docker = AsyncMock( + side_effect=AssertionError("cancelled creation must not run Docker") + ) + monkeypatch.setattr(python_handler, "_run_docker_command", docker) + + task = asyncio.create_task( + python_handler.execute({"code": "print('cancelled')"}, {}) + ) + try: + assert await asyncio.to_thread(started.wait, 5), ( + "directory creation did not start" + ) + assert len(created_dirs) == 1 + assert (created_dirs[0] / "created.txt").is_file() + + task.cancel() + await asyncio.sleep(0) + assert not task.done(), "cancellation must wait for directory ownership" + assert created_dirs[0].is_dir() + docker.assert_not_awaited() + + release.set() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(task, timeout=5) + + assert task.cancelled() + assert not created_dirs[0].exists() + assert not list(tmp_path.rglob("pyinterp_*")) + docker.assert_not_awaited() + finally: + release.set() + if not task.done(): + task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await task From 33b83a19d7b4e1d0c1c9e9cb6ddd7117365b4830 Mon Sep 17 00:00:00 2001 From: Null <1708213363@qq.com> Date: Tue, 6 Oct 2026 11:38:51 +0800 Subject: [PATCH 009/121] fix(chat): ignore stale cancellation errors and stream statuses --- .../src/chat-store/runtime-contract.test.ts | 104 ++++++++++++++++++ apps/undefined-chat/src/chat-store/store.ts | 17 ++- docs/undefined-chat.md | 1 + 3 files changed, 116 insertions(+), 6 deletions(-) diff --git a/apps/undefined-chat/src/chat-store/runtime-contract.test.ts b/apps/undefined-chat/src/chat-store/runtime-contract.test.ts index 15d822ef..1ae9a9a4 100644 --- a/apps/undefined-chat/src/chat-store/runtime-contract.test.ts +++ b/apps/undefined-chat/src/chat-store/runtime-contract.test.ts @@ -469,4 +469,108 @@ describe("Runtime job contract through the native client", () => { expect(store.getSnapshot().eventCursorByJob["job-1"]).toBe(4); expect(client.getHistory).toHaveBeenCalledTimes(2); }); + + test("an old cancellation rejection cannot overwrite a new job's send error", async () => { + const nativeClient = createTauriRuntimeClient(); + let rejectCancel!: (reason: Error) => void; + const pendingCancel = new Promise((_resolve, reject) => { + rejectCancel = reject; + }); + vi.mocked(invoke) + .mockReturnValueOnce(pendingCancel) + .mockResolvedValueOnce({ + status: 200, + ok: true, + body: { + job: runtimeJob({ status: "done", history_finalized: true }), + events: [], + }, + }); + const running = job(); + const client = runtimeClientStub({ + getActiveJobs: vi.fn(async () => ({ job: running, jobs: [running] })), + cancelJob: nativeClient.cancelJob, + fetchJobEventsJson: nativeClient.fetchJobEventsJson, + sendMessage: vi.fn(async () => job({ jobId: "job-new" })), + }); + const store = createChatStore({ client }); + await store.bootstrap(); + const cancelling = store.cancelJob("job-1"); + store.handleRuntimeStatus({ + subscriptionId: "sub-1", + jobId: "job-1", + status: "error", + detail: "closed", + }); + await vi.waitFor(() => + expect( + store.getSnapshot().activeJobsByConversation.default, + ).toBeUndefined(), + ); + store.updateDraft("default", "新任务"); + await store.sendSelectedMessage(); + rejectCancel(new Error("old cancellation failed")); + await cancelling; + expect(store.getSnapshot().sendError).toBeNull(); + expect(store.getSnapshot().activeJobsByConversation.default?.jobId).toBe( + "job-new", + ); + }); + + test.each(["connected", "error", "closed"])( + "a late old SSE %s status cannot change a new job's connection state", + async (status) => { + const running = job(); + const client = runtimeClientStub({ + getActiveJobs: vi.fn(async () => ({ job: running, jobs: [running] })), + fetchJobEventsJson: vi.fn(async () => ({ + job: job({ status: "done", historyFinalized: true }), + after: 0, + lastSeq: 0, + events: [], + })), + sendMessage: vi.fn(async () => job({ jobId: "job-new" })), + }); + const store = createChatStore({ client }); + await store.bootstrap(); + store.handleRuntimeStatus({ + subscriptionId: "sub-1", + jobId: "job-1", + status: "error", + detail: "closed", + }); + await vi.waitFor(() => + expect( + store.getSnapshot().activeJobsByConversation.default, + ).toBeUndefined(), + ); + store.updateDraft("default", "新任务"); + await store.sendSelectedMessage(); + // 活跃新任务进入正常 fallback,随后旧通知不能改变这一状态或发起额外请求。 + const newStatus = { + subscriptionId: "sub-new", + jobId: "job-new", + detail: "closed", + }; + store.handleRuntimeStatus({ ...newStatus, status: "connected" }); + expect(store.getSnapshot().connectionState).toBe("streaming"); + vi.mocked(client.fetchJobEventsJson).mockImplementationOnce( + () => new Promise(() => {}), + ); + store.handleRuntimeStatus({ ...newStatus, status: "closed" }); + expect(store.getSnapshot().connectionState).toBe("json_fallback"); + const requests = vi.mocked(client.fetchJobEventsJson).mock.calls.length; + store.handleRuntimeStatus({ + subscriptionId: "sub-1", + jobId: "job-1", + status, + detail: "late", + }); + expect(store.getSnapshot().connectionState).toBe("json_fallback"); + expect(client.fetchJobEventsJson).toHaveBeenCalledTimes(requests); + expect(store.getSnapshot().activeJobsByConversation.default?.jobId).toBe( + "job-new", + ); + }, + ); }); diff --git a/apps/undefined-chat/src/chat-store/store.ts b/apps/undefined-chat/src/chat-store/store.ts index 184cdee2..b62ff717 100644 --- a/apps/undefined-chat/src/chat-store/store.ts +++ b/apps/undefined-chat/src/chat-store/store.ts @@ -1195,6 +1195,9 @@ export function createChatStore({ finishJob(jobId, conversationId); } } catch (err) { + if (state.activeJobsByConversation[conversationId]?.jobId !== jobId) { + return; + } dispatch({ type: "send/error", error: errorMessage(err) }); } } @@ -1528,6 +1531,13 @@ export function createChatStore({ } function handleRuntimeStatus(status: RuntimeSseStatus): void { + const conversationId = state.jobConversationById[status.jobId]; + if ( + !conversationId || + state.activeJobsByConversation[conversationId]?.jobId !== status.jobId + ) { + return; + } if (status.status === "connected") { dispatch({ type: "connection/set", connectionState: "streaming" }); return; @@ -1537,12 +1547,7 @@ export function createChatStore({ return; } if (status.status === "closed") { - const conversationId = state.jobConversationById[status.jobId]; - if (conversationId && state.activeJobsByConversation[conversationId]) { - fetchJobEventsFallback(status.jobId); - return; - } - dispatch({ type: "connection/set", connectionState: "connected" }); + fetchJobEventsFallback(status.jobId); } } diff --git a/docs/undefined-chat.md b/docs/undefined-chat.md index 44612daf..606165f6 100644 --- a/docs/undefined-chat.md +++ b/docs/undefined-chat.md @@ -60,6 +60,7 @@ Runtime 请求必须由 Tauri command 拼接到已配置 origin 下,避免 Rea - SSE error/closed 或原生订阅建立失败时使用 `job_id + seq` 调用 JSON events fallback 补齐遗漏事件;消息已成功创建 job 后,订阅失败不会撤回用户消息或恢复已发送草稿。 - JSON fallback 先续接增量事件,再应用 Runtime 的完整正文与阶段快照,保留已有消息/工具时间线和已结束调用,避免把快照正文再次累计。 - 重叠 JSON 请求按 job 身份和请求代次校验;已结束、已被替换的 job 或落后于已消费事件游标的回包会被忽略,避免迟到响应覆盖下一轮任务或回退回复内容。 +- 取消请求错误和 SSE 的连接状态通知同样只作用于对应的活跃 job,旧任务的迟到通知不会改写新任务的错误或连接状态。 - `done` / `error` / `cancelled` 事件到达后,客户端继续保留当前会话的 job 和发送锁,通过 JSON polling 等待 `history_finalized=true`,确认历史落盘后再刷新历史并清理订阅。 - 历史分页的 `before` / `next_before` 是非负数值索引;React 与 Tauri command 使用同一数值契约,首屏不传 `before`。 - Android 回到前台时执行 `store.bootstrap()`,刷新配置、会话、当前历史页和 active jobs,并重新建立事件订阅。 From 7d93d19b75683aa1c97c601193d3a5fc97ab3286 Mon Sep 17 00:00:00 2001 From: Null <1708213363@qq.com> Date: Tue, 6 Oct 2026 11:59:50 +0800 Subject: [PATCH 010/121] fix(douyin): stop fallback after uncertain delivery errors --- src/Undefined/douyin/sender.py | 4 ++ .../skills/tools/douyin_video/README.md | 2 + tests/test_douyin_sender.py | 40 +++++++++++++++++++ 3 files changed, 46 insertions(+) diff --git a/src/Undefined/douyin/sender.py b/src/Undefined/douyin/sender.py index 7a6a90d8..7d4a6810 100644 --- a/src/Undefined/douyin/sender.py +++ b/src/Undefined/douyin/sender.py @@ -237,6 +237,10 @@ async def send_douyin_video( return f"已发送抖音合并转发「{video_info.title}」" except Exception as exc: logger.exception("[Douyin] 处理视频失败: %s", video_id) + if getattr(exc, "delivery_uncertain", False) or getattr( + exc, "file_transfer_error", False + ): + raise video_info, error = await _recover_video_info_after_failure( video_id, video_info, diff --git a/src/Undefined/skills/tools/douyin_video/README.md b/src/Undefined/skills/tools/douyin_video/README.md index e3167d6f..4f666a81 100644 --- a/src/Undefined/skills/tools/douyin_video/README.md +++ b/src/Undefined/skills/tools/douyin_video/README.md @@ -10,5 +10,7 @@ `send` 模式会发送一条两节点合并转发:视频信息、视频文件或视频状态。`uid` 模式会下载视频并注册为当前会话附件 UID,供 `file_analysis_agent` 继续分析。`info` 模式只返回标题、作者、时长、简介、封面 URL 和分享链接,不下载视频、不发送消息、不注册附件。 +发送遇到投递结果不确定或本地文件准备失败时,会保留该错误并停止发送,不再发送降级信息卡片;已经下载的视频仍会清理。 + 配置依赖: - `config.toml` 中的 `[douyin]` 段控制代理、自动提取、时长限制、文件大小限制和清晰度探测顺序。 diff --git a/tests/test_douyin_sender.py b/tests/test_douyin_sender.py index 35539b45..72c04d6e 100644 --- a/tests/test_douyin_sender.py +++ b/tests/test_douyin_sender.py @@ -10,6 +10,8 @@ import Undefined.douyin.sender as douyin_sender from Undefined.attachments import AttachmentRegistry from Undefined.douyin.models import DouyinVideoInfo +from Undefined.onebot.client import OneBotDeliveryUncertainError +from Undefined.onebot.file_errors import FileTransferError def _video_info() -> DouyinVideoInfo: @@ -25,6 +27,44 @@ def _video_info() -> DouyinVideoInfo: ) +@pytest.mark.asyncio +@pytest.mark.parametrize( + "error", + [ + OneBotDeliveryUncertainError("send_forward_msg", "timeout"), + FileTransferError("stream", "prepare failed"), + ], +) +async def test_send_douyin_video_does_not_fallback_after_delivery_error( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + error: RuntimeError, +) -> None: + video_path = tmp_path / "douyin.mp4" + video_path.write_bytes(b"video") + send_mock = AsyncMock(side_effect=error) + sender: Any = SimpleNamespace(send_group_forward_message=send_mock) + monkeypatch.setattr( + douyin_sender, + "download_video", + AsyncMock(return_value=(video_path, _video_info(), "1080p", 5)), + ) + cleanup_mock = AsyncMock() + monkeypatch.setattr(douyin_sender, "cleanup_path", cleanup_mock) + recover_mock = AsyncMock() + monkeypatch.setattr(douyin_sender, "get_video_info", recover_mock) + + with pytest.raises(type(error)) as caught: + await douyin_sender.send_douyin_video( + _video_info().aweme_id, sender, "group", 123456 + ) + + assert caught.value is error + send_mock.assert_awaited_once() + recover_mock.assert_not_awaited() + cleanup_mock.assert_awaited_once_with(video_path) + + @pytest.mark.asyncio async def test_send_douyin_video_records_two_node_forward( monkeypatch: pytest.MonkeyPatch, From 58dabe7d73317c7e9c8cd216090f23e0a4c1cf23 Mon Sep 17 00:00:00 2001 From: Null <1708213363@qq.com> Date: Tue, 6 Oct 2026 12:00:01 +0800 Subject: [PATCH 011/121] fix(chat): create attachment downloads without overwriting files --- apps/undefined-chat/src-tauri/src/download.rs | 39 +++++++------ .../src-tauri/src/native_tests.rs | 55 ++++++++++++++++++- docs/undefined-chat.md | 2 +- 3 files changed, 78 insertions(+), 18 deletions(-) diff --git a/apps/undefined-chat/src-tauri/src/download.rs b/apps/undefined-chat/src-tauri/src/download.rs index f8611267..57d66478 100644 --- a/apps/undefined-chat/src-tauri/src/download.rs +++ b/apps/undefined-chat/src-tauri/src/download.rs @@ -10,6 +10,7 @@ use tauri_plugin_http::reqwest::Response; use tokio::io::AsyncWriteExt; const MAX_PREVIEW_BYTES: usize = 10 * 1024 * 1024; +pub(crate) const MAX_DOWNLOAD_NAME_CANDIDATES: usize = 1000; #[derive(Debug, Clone, Deserialize)] #[serde(rename_all = "camelCase")] @@ -153,12 +154,11 @@ pub async fn save_attachment( tokio::fs::create_dir_all(&download_dir) .await .map_err(|err| format!("download directory create failed: {err}"))?; - let mut path = download_dir.join(safe_file_name( + let path = download_dir.join(safe_file_name( input.file_name.as_deref(), &input.attachment_id, )); - path = unique_path(path).await; - let mut file = tokio::fs::File::create(&path) + let (path, mut file) = create_unique_file(path) .await .map_err(|err| format!("attachment output create failed: {err}"))?; let mut bytes_written = 0u64; @@ -224,11 +224,9 @@ pub async fn preview_attachment_bytes( }) } -async fn unique_path(path: PathBuf) -> PathBuf { - if tokio::fs::metadata(&path).await.is_err() { - return path; - } - +pub(crate) async fn create_unique_file( + path: PathBuf, +) -> std::io::Result<(PathBuf, tokio::fs::File)> { let parent = path .parent() .map(PathBuf::from) @@ -238,18 +236,27 @@ async fn unique_path(path: PathBuf) -> PathBuf { .and_then(|value| value.to_str()) .unwrap_or("attachment"); let extension = path.extension().and_then(|value| value.to_str()); - for index in 1..1000 { - let file_name = if let Some(extension) = extension { - format!("{stem} ({index}).{extension}") + for index in 0..MAX_DOWNLOAD_NAME_CANDIDATES { + let candidate = if index == 0 { + path.clone() } else { - format!("{stem} ({index})") + let file_name = if let Some(extension) = extension { + format!("{stem} ({index}).{extension}") + } else { + format!("{stem} ({index})") + }; + parent.join(file_name) }; - let candidate = parent.join(file_name); - if tokio::fs::metadata(&candidate).await.is_err() { - return candidate; + match tokio::fs::File::create_new(&candidate).await { + Ok(file) => return Ok((candidate, file)), + Err(err) if err.kind() == std::io::ErrorKind::AlreadyExists => continue, + Err(err) => return Err(err), } } - path + Err(std::io::Error::new( + std::io::ErrorKind::AlreadyExists, + "no unused attachment download file name is available", + )) } #[cfg(test)] diff --git a/apps/undefined-chat/src-tauri/src/native_tests.rs b/apps/undefined-chat/src-tauri/src/native_tests.rs index 78e50909..0039b9a8 100644 --- a/apps/undefined-chat/src-tauri/src/native_tests.rs +++ b/apps/undefined-chat/src-tauri/src/native_tests.rs @@ -20,7 +20,7 @@ use crate::state::{ AppRuntimeConfig, RuntimeConfigInput, RuntimeRequestPath, APP_CONFIG_FILE_NAME, }; use crate::{ - download::test_safe_file_name, + download::{create_unique_file, test_safe_file_name, MAX_DOWNLOAD_NAME_CANDIDATES}, upload::{ attachment_file_name, attachments_url, parse_file_path, requires_regular_file_check, upload_uses_streaming_body, UploadAttachmentInput, @@ -298,6 +298,59 @@ fn attachment_save_file_name_sanitizes_path_like_input() { ); } +#[test] +fn concurrent_attachment_saves_preserve_existing_and_each_other() { + let root = std::env::temp_dir().join(format!("undefined-download-{}", uuid::Uuid::new_v4())); + std::fs::create_dir_all(&root).unwrap(); + let path = root.join("report.txt"); + std::fs::write(&path, b"existing").unwrap(); + tauri::async_runtime::block_on(async { + use tokio::io::AsyncWriteExt; + let (first, second) = futures_util::future::join( + create_unique_file(path.clone()), + create_unique_file(path.clone()), + ) + .await; + let (first_path, mut first_file) = first.unwrap(); + let (second_path, mut second_file) = second.unwrap(); + assert_ne!(first_path, second_path); + assert_ne!(first_path, path); + assert_ne!(second_path, path); + first_file.write_all(b"first attachment").await.unwrap(); + second_file.write_all(b"second attachment").await.unwrap(); + first_file.flush().await.unwrap(); + second_file.flush().await.unwrap(); + assert_eq!(tokio::fs::read(&path).await.unwrap(), b"existing"); + assert_eq!( + tokio::fs::read(first_path).await.unwrap(), + b"first attachment" + ); + assert_eq!( + tokio::fs::read(second_path).await.unwrap(), + b"second attachment" + ); + }); + std::fs::remove_dir_all(root).unwrap(); +} + +#[test] +fn attachment_save_fails_when_all_file_names_are_occupied() { + let root = std::env::temp_dir().join(format!("undefined-download-{}", uuid::Uuid::new_v4())); + std::fs::create_dir_all(&root).unwrap(); + let path = root.join("report.txt"); + std::fs::write(&path, b"existing").unwrap(); + for index in 1..MAX_DOWNLOAD_NAME_CANDIDATES { + std::fs::write(root.join(format!("report ({index}).txt")), b"existing").unwrap(); + } + let error = tauri::async_runtime::block_on(create_unique_file(path.clone())).unwrap_err(); + assert_eq!(error.kind(), std::io::ErrorKind::AlreadyExists); + assert!(error + .to_string() + .contains("no unused attachment download file name")); + assert_eq!(std::fs::read(path).unwrap(), b"existing"); + std::fs::remove_dir_all(root).unwrap(); +} + #[test] fn upload_file_path_parser_keeps_android_content_uri_as_url() { let path = parse_file_path("content://media/external/images/media/42").unwrap(); diff --git a/docs/undefined-chat.md b/docs/undefined-chat.md index 606165f6..b05ebe0d 100644 --- a/docs/undefined-chat.md +++ b/docs/undefined-chat.md @@ -81,7 +81,7 @@ API Key 不应暴露给 React 状态树或日志。Tauri 负责保存、读取 附件上传由 Tauri 从文件句柄流式读取并转发到 Runtime,React 只持有文件选择结果和上传状态。桌面路径和 `file://` URL 会做 regular file 校验;Android `content://` URI 交给 `tauri-plugin-fs` 打开,不强制使用本地 `metadata().is_file()` 判定。 -下载和预览同样通过 Tauri 受控命令处理。图片附件通过 `AttachmentImage` 或附件预览按钮调用 `previewAttachment`,Rust 端带 auth 拉取附件字节,再转为 `Blob` URL 渲染;全屏预览走应用内 `ImageViewerModal`,关闭时释放临时 URL。 +下载和预览同样通过 Tauri 受控命令处理。下载以原子创建方式占用文件名,同名文件依次添加数字后缀,并发下载也不会覆盖已有文件;可用候选名耗尽时明确报错。图片附件通过 `AttachmentImage` 或附件预览按钮调用 `previewAttachment`,Rust 端带 auth 拉取附件字节,再转为 `Blob` URL 渲染;全屏预览走应用内 `ImageViewerModal`,关闭时释放临时 URL。 历史时间线未接入全局图片查看回调时使用本地 `ImagePreview`。可从图片通过 Enter / Space 打开,焦点进入关闭按钮,Tab / Shift+Tab 保持在预览内;Escape、点击遮罩或关闭按钮均可关闭,焦点恢复到触发图片。 From c55e3a2b24e6df68c3a358a55a2e4115302495c2 Mon Sep 17 00:00:00 2001 From: Null <1708213363@qq.com> Date: Tue, 6 Oct 2026 12:01:03 +0800 Subject: [PATCH 012/121] fix(automations): serialize snapshot persistence through cancellation --- docs/automations.md | 2 + src/Undefined/automations/storage.py | 22 +++-- tests/test_automation_storage_ordering.py | 98 +++++++++++++++++++++++ 3 files changed, 114 insertions(+), 8 deletions(-) create mode 100644 tests/test_automation_storage_ordering.py diff --git a/docs/automations.md b/docs/automations.md index 2f0fce8b..f153491a 100644 --- a/docs/automations.md +++ b/docs/automations.md @@ -91,6 +91,8 @@ 启动时若还没有 `data/automations.json` 但存在旧的 `scheduled_tasks.json`,会自动把旧任务转换为流程图格式写入新文件;旧文件保留不动,之后也不再双写。旧 `/api/v1/schedules` 接口与 `scheduler.*` 工具已下线,统一使用 `/api/v1/automations` 与 `automation.*`。 +任务编辑、删除与运行结果的保存共用串行写入锁,每次写入冻结完整快照并原子替换文件。并发保存不会让旧快照覆盖后续删除或新增;保存期间收到取消时,会先等磁盘写入完成再释放锁并传播取消。 + ## 配置与限制 `[automations]` 配置节控制总开关、最大节点数(30)、全局并发(16,支持热更新)、节点 / 整图超时(600s / 1200s)、循环上限(默认 25,无内置硬顶)等,完整字段见 [配置说明](configuration.md)。事件类工作流默认不设冷却时间。 diff --git a/src/Undefined/automations/storage.py b/src/Undefined/automations/storage.py index 91f64226..c830f0b6 100644 --- a/src/Undefined/automations/storage.py +++ b/src/Undefined/automations/storage.py @@ -2,6 +2,8 @@ from __future__ import annotations +import asyncio +from copy import deepcopy import logging from pathlib import Path from typing import Any @@ -25,6 +27,7 @@ def __init__( ) -> None: self.path = path or AUTOMATIONS_FILE_PATH self.legacy_path = legacy_path or LEGACY_TASKS_FILE_PATH + self._save_lock = asyncio.Lock() def load_tasks(self) -> dict[str, Any]: loaded_from_legacy = False @@ -59,14 +62,17 @@ def load_tasks(self) -> dict[str, Any]: async def save_all(self, tasks: dict[str, Any]) -> None: from Undefined.utils import io - data_to_save: dict[str, Any] = {} - for task_id, task_info in tasks.items(): - if isinstance(task_info, dict): - data_to_save[str(task_id)] = task_info - else: - logger.warning("[自动化] 跳过未知任务格式: %s", task_id) - await io.write_json(self.path, data_to_save, use_lock=True) - logger.info("[自动化] 已保存 %s 条到 %s", len(data_to_save), self.path) + async with self._save_lock: + data_to_save: dict[str, Any] = {} + for task_id, task_info in tasks.items(): + if isinstance(task_info, dict): + data_to_save[str(task_id)] = deepcopy(task_info) + else: + logger.warning("[自动化] 跳过未知任务格式: %s", task_id) + await io.run_cancellation_safe( + io.write_json(self.path, data_to_save, use_lock=True) + ) + logger.info("[自动化] 已保存 %s 条到 %s", len(data_to_save), self.path) def _write_json_sync(self, path: Path, data: dict[str, Any]) -> None: from Undefined.utils.io import write_json_sync diff --git a/tests/test_automation_storage_ordering.py b/tests/test_automation_storage_ordering.py new file mode 100644 index 00000000..2a2bdebc --- /dev/null +++ b/tests/test_automation_storage_ordering.py @@ -0,0 +1,98 @@ +"""Automation persistence stays ordered across concurrent mutations and cancellation.""" + +from __future__ import annotations + +import asyncio +import json +from pathlib import Path +from types import SimpleNamespace +from typing import Any + +import pytest + +from Undefined.automations.service import AutomationService +from Undefined.automations.storage import AutomationStorage +from Undefined.utils import io + + +@pytest.mark.asyncio +@pytest.mark.parametrize("cancel_first", [False, True]) +async def test_running_task_save_cannot_restore_concurrently_deleted_task( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, cancel_first: bool +) -> None: + storage = AutomationStorage(tmp_path / "automations.json", tmp_path / "legacy.json") + service = AutomationService( + SimpleNamespace(), + SimpleNamespace(), + SimpleNamespace(), + SimpleNamespace(), + storage, + ) + service.tasks.update({"a": {"task_id": "a"}, "b": {"task_id": "b"}}) + started = asyncio.Event() + release = asyncio.Event() + writes: list[dict[str, Any]] = [] + write_json = io.write_json + + async def gated_write(path: Path | str, data: Any, use_lock: bool = True) -> None: + writes.append(data) + if len(writes) == 1: + started.set() + await release.wait() + await write_json(path, data, use_lock=use_lock) + + monkeypatch.setattr(io, "write_json", gated_write) + first = asyncio.create_task(service._mark_run("a", status="ok")) + second: asyncio.Task[bool] | None = None + try: + await asyncio.wait_for(started.wait(), 1) + if cancel_first: + first.cancel() + second = asyncio.create_task(service.remove_task("b")) + await asyncio.sleep(0) + assert "b" not in service.tasks + assert len(writes) == 1 + release.set() + results = await asyncio.gather(first, second, return_exceptions=True) + if cancel_first: + assert isinstance(results[0], asyncio.CancelledError) + else: + assert results[0] is None + assert results[1] is True + saved = json.loads(storage.path.read_text("utf-8")) + assert set(saved) == {"a"} + assert saved["a"]["current_executions"] == 1 + assert set(storage.load_tasks()) == {"a"} + finally: + release.set() + await asyncio.gather( + first, *([second] if second is not None else []), return_exceptions=True + ) + service.shutdown() + + +@pytest.mark.asyncio +async def test_snapshot_does_not_change_while_disk_write_is_pending( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + storage = AutomationStorage(tmp_path / "automations.json", tmp_path / "legacy.json") + tasks: dict[str, Any] = {"a": {"nodes": [{"args": {"message": "before"}}]}} + started = asyncio.Event() + release = asyncio.Event() + write_json = io.write_json + + async def gated_write(path: Path | str, data: Any, use_lock: bool = True) -> None: + started.set() + await release.wait() + await write_json(path, data, use_lock=use_lock) + + monkeypatch.setattr(io, "write_json", gated_write) + saving = asyncio.create_task(storage.save_all(tasks)) + try: + await asyncio.wait_for(started.wait(), 1) + tasks["a"]["nodes"][0]["args"]["message"] = "after" + finally: + release.set() + await saving + saved = json.loads(storage.path.read_text("utf-8")) + assert saved["a"]["nodes"][0]["args"]["message"] == "before" From 2dfb9c3457af4dc5102fcc91942029a7afb95285 Mon Sep 17 00:00:00 2001 From: Null <1708213363@qq.com> Date: Tue, 6 Oct 2026 12:01:55 +0800 Subject: [PATCH 013/121] fix(arxiv): settle inflight waiters when owner is cancelled --- src/Undefined/arxiv/sender.py | 4 ++ .../skills/tools/arxiv_paper/README.md | 2 + tests/test_arxiv_sender.py | 52 +++++++++++++++++++ 3 files changed, 58 insertions(+) diff --git a/src/Undefined/arxiv/sender.py b/src/Undefined/arxiv/sender.py index cf3115d5..553c5169 100644 --- a/src/Undefined/arxiv/sender.py +++ b/src/Undefined/arxiv/sender.py @@ -380,6 +380,10 @@ async def send_arxiv_paper( summary_preview_chars=summary_preview_chars, context=context, ) + except asyncio.CancelledError: + if not future.done(): + future.cancel() + raise except Exception as exc: if not future.done(): future.set_exception(exc) diff --git a/src/Undefined/skills/tools/arxiv_paper/README.md b/src/Undefined/skills/tools/arxiv_paper/README.md index c514f5f1..d128ec36 100644 --- a/src/Undefined/skills/tools/arxiv_paper/README.md +++ b/src/Undefined/skills/tools/arxiv_paper/README.md @@ -15,6 +15,8 @@ 4. 尝试下载并上传 PDF 5. 下载超限或 PDF 失败时仅保留信息消息 +同一会话的同一论文并发发送会共享在途结果。主发送任务取消时,等待该结果的调用也会收到取消,不记录成功去重时间;单个等待调用取消不会取消主发送任务。 + `uid` 模式流程: 1. 下载 PDF 2. 注册为当前会话附件 UID diff --git a/tests/test_arxiv_sender.py b/tests/test_arxiv_sender.py index 312075b5..8934d072 100644 --- a/tests/test_arxiv_sender.py +++ b/tests/test_arxiv_sender.py @@ -43,6 +43,58 @@ def _sender() -> Any: ) +@pytest.mark.asyncio +@pytest.mark.parametrize("cancel_owner", [True, False]) +async def test_inflight_send_cancellation_settles_waiters( + monkeypatch: pytest.MonkeyPatch, + cancel_owner: bool, +) -> None: + started = asyncio.Event() + release = asyncio.Event() + + async def _fake_once(**_: object) -> str: + started.set() + await release.wait() + return "sent" + + monkeypatch.setattr(arxiv_sender, "_send_arxiv_paper_once", _fake_once) + options: dict[str, Any] = { + "paper_id": "2501.01234", + "sender": _sender(), + "target_type": "group", + "target_id": 123456, + "max_file_size": 100, + "author_preview_limit": 2, + "summary_preview_chars": 1000, + } + owner = asyncio.create_task(send_arxiv_paper(**options)) + await started.wait() + follower = asyncio.create_task(send_arxiv_paper(**options)) + await asyncio.sleep(0) + key = ("group", 123456, "2501.01234") + future = arxiv_sender._INFLIGHT_SENDS[key] + assert not follower.done() + + cancelled_task = owner if cancel_owner else follower + cancelled_task.cancel() + with pytest.raises(asyncio.CancelledError): + await cancelled_task + + if cancel_owner: + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(follower, timeout=1) + assert future.cancelled() + assert key not in arxiv_sender._RECENT_SENDS + else: + assert not future.done() + assert not owner.done() + release.set() + assert await asyncio.wait_for(owner, timeout=1) == "sent" + assert future.result() == "sent" + assert key in arxiv_sender._RECENT_SENDS + assert key not in arxiv_sender._INFLIGHT_SENDS + + @pytest.mark.asyncio async def test_send_arxiv_paper_sends_info_and_pdf( monkeypatch: pytest.MonkeyPatch, From 9288097c0c3c3e80f3e6ec71258bc5b4f6fda9c3 Mon Sep 17 00:00:00 2001 From: Null <1708213363@qq.com> Date: Tue, 6 Oct 2026 12:02:21 +0800 Subject: [PATCH 014/121] fix(webui): preserve file charset when compressing downloads --- docs/management-api.md | 2 +- src/Undefined/webui/app.py | 6 +++- tests/test_webui_gzip.py | 68 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 74 insertions(+), 2 deletions(-) create mode 100644 tests/test_webui_gzip.py diff --git a/docs/management-api.md b/docs/management-api.md index fba87f4a..a5902ad1 100644 --- a/docs/management-api.md +++ b/docs/management-api.md @@ -339,7 +339,7 @@ Content-Type: multipart/form-data } ``` -`runtime/chat/files` 仍保留为旧 WebUI 浏览器文件缓存兼容路径;新客户端和新 WebChat 功能应使用 `runtime/chat/attachments`。 +`runtime/chat/files` 仍保留为旧 WebUI 浏览器文件缓存兼容路径;新客户端和新 WebChat 功能应使用 `runtime/chat/attachments`。其 `runtime/chat/file?id=...` 下载响应支持 gzip 协商,可压缩的文本 / HTML 文件会保留 MIME、UTF-8 charset 和附件下载头。 WebUI 内嵌聊天渲染 `` / `` 图片时会把 Runtime 返回的 `/api/v1/chat/attachments/{uid}/preview` 规整为 Management 代理路径 `/api/runtime/chat/attachments/{uid}/preview`,浏览器不直接请求 Runtime API。普通 Markdown 图片(如 `![Image #1](https://...)`)仍由前端安全 renderer 输出可点击预览的 ``,不走附件代理。 diff --git a/src/Undefined/webui/app.py b/src/Undefined/webui/app.py index ad0b5379..f9dfdb8d 100644 --- a/src/Undefined/webui/app.py +++ b/src/Undefined/webui/app.py @@ -160,7 +160,11 @@ async def gzip_middleware( mime = _GZIP_MIME.get( file_path.suffix.lower(), "application/octet-stream" ) - new_resp = web.Response(body=compressed, content_type=mime) + new_resp = web.Response( + body=compressed, + status=response.status, + headers={"Content-Type": mime}, + ) # 保留外层中间件已设置的 header(如安全头) _skip = { "content-length", diff --git a/tests/test_webui_gzip.py b/tests/test_webui_gzip.py new file mode 100644 index 00000000..56cfab5a --- /dev/null +++ b/tests/test_webui_gzip.py @@ -0,0 +1,68 @@ +from __future__ import annotations + +import gzip +from pathlib import Path + +import pytest +from aiohttp import web +from aiohttp.test_utils import make_mocked_request + +from Undefined.webui.app import gzip_middleware + + +@pytest.mark.parametrize( + ("suffix", "content_type", "charset"), + [ + (".txt", "text/plain", "utf-8"), + (".html", "text/html", "utf-8"), + (".json", "application/json", None), + ], +) +async def test_gzip_file_download_preserves_content_and_headers( + tmp_path: Path, suffix: str, content_type: str, charset: str | None +) -> None: + original = ("聊天附件内容\n" * 100).encode("utf-8") + path = tmp_path / f"attachment{suffix}" + path.write_bytes(original) + + async def handler(request: web.Request) -> web.StreamResponse: + return web.FileResponse( + path, + status=202, + headers={ + "Content-Disposition": 'attachment; filename="report.txt"', + "Cache-Control": "no-store", + }, + ) + + request = make_mocked_request( + "GET", "/api/runtime/chat/file?id=abc", headers={"Accept-Encoding": "gzip"} + ) + response = await gzip_middleware(request, handler) + assert isinstance(response, web.Response) + assert isinstance(response.body, bytes) + assert gzip.decompress(response.body) == original + assert response.status == 202 + assert response.content_type == content_type + assert response.charset == charset + assert response.headers["Content-Encoding"] == "gzip" + assert ( + response.headers["Content-Disposition"] == 'attachment; filename="report.txt"' + ) + assert response.headers["Cache-Control"] == "no-store" + + +async def test_gzip_file_download_without_accept_encoding_is_unchanged( + tmp_path: Path, +) -> None: + path = tmp_path / "report.txt" + path.write_bytes(b"chat attachment\n" * 100) + original = web.FileResponse(path) + + async def handler(request: web.Request) -> web.StreamResponse: + return original + + request = make_mocked_request("GET", "/api/runtime/chat/file?id=abc") + response = await gzip_middleware(request, handler) + assert response is original + assert "Content-Encoding" not in response.headers From d4030f1ed4ae5299bc5dea78c0e43d661caaa85e Mon Sep 17 00:00:00 2001 From: Null <1708213363@qq.com> Date: Tue, 6 Oct 2026 12:03:56 +0800 Subject: [PATCH 015/121] fix(onebot): reconnect without waiting for message handlers --- ARCHITECTURE.md | 2 + src/Undefined/onebot/client.py | 37 +++++--- tests/test_onebot_connection_lifecycle.py | 107 ++++++++++++++++++++++ 3 files changed, 133 insertions(+), 13 deletions(-) create mode 100644 tests/test_onebot_connection_lifecycle.py diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 6a095b02..06833845 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -886,6 +886,8 @@ description: 从 PDF 文件中提取文本和表格,填写表单。当用户 微信接入由主进程独立管理,不依赖 OneBot 是否在线。`WeixinService` 先校验帐号与 peer,再把已绑定来源映射为逻辑 QQ 私聊交给 `MessageHandler`;未知来源在写历史或调用 AI 之前进入隔离存储。`DeliveryAddress` 将逻辑身份与物理通道分离:`qq:` 和 `wechat:` 可共享同一个用户历史、认知记忆与权限,但发送器、消息合并 scope 和 transport 元数据保持通道隔离。自动化同样持久化规范 `address`,旧 `target_type + target_id` 在加载时转换为 `group:` 或 `qq:` 地址。 +OneBot 消息处理任务由客户端跨连接跟踪。网络断线只结束接收循环、通知未完成的 API 请求连接失败,不等待媒体下载等处理任务即可重连;已有处理任务继续运行。主动断开、停止或取消连接任务时,会取消并等待这些处理任务收尾,避免后台任务遗留。 + ### "车站-列车" 队列模型 针对高并发消息处理,Undefined 实现了全新的 **ModelQueue** 调度机制: diff --git a/src/Undefined/onebot/client.py b/src/Undefined/onebot/client.py index 52993435..b05e6c48 100644 --- a/src/Undefined/onebot/client.py +++ b/src/Undefined/onebot/client.py @@ -126,6 +126,7 @@ def __init__( Callable[[dict[str, Any]], Coroutine[Any, Any, None]] | None ) = None self._running = False + self._tasks: set[asyncio.Task[None]] = set() self.file_transport = ( file_transport if file_transport is not None @@ -190,12 +191,22 @@ async def connect(self) -> None: async def disconnect(self) -> None: """断开连接""" - self._running = False - if self.ws: - logger.info("[WebSocket] 正在主动断开连接...") - await self.ws.close() - self.ws = None - logger.info("[WebSocket] 连接已断开") + self.stop() + try: + if self.ws: + logger.info("[WebSocket] 正在主动断开连接...") + await self.ws.close() + self.ws = None + logger.info("[WebSocket] 连接已断开") + finally: + await self._cancel_message_tasks() + + async def _cancel_message_tasks(self) -> None: + tasks = tuple(self._tasks) + for task in tasks: + task.cancel() + if tasks: + await asyncio.gather(*tasks, return_exceptions=True) async def _call_api( self, @@ -914,7 +925,7 @@ async def run(self) -> None: raise RuntimeError("WebSocket 未连接") self._running = True - self._tasks: set[asyncio.Task[None]] = set() + cancelled = False logger.info("[WebSocket] 消息接收循环已启动") try: @@ -941,6 +952,9 @@ async def run(self) -> None: break except Exception as e: logger.exception(f"[WebSocket] 接收消息时发生异常: {e}") + except asyncio.CancelledError: + cancelled = True + raise finally: self._running = False for future in self._pending_responses.values(): @@ -948,12 +962,9 @@ async def run(self) -> None: future.set_exception( ConnectionError("OneBot WebSocket 接收循环已停止") ) - # 等待所有后台任务完成 - if self._tasks: - logger.debug( - f"[WebSocket] 正在等待 {len(self._tasks)} 个异步任务完成..." - ) - await asyncio.gather(*self._tasks, return_exceptions=True) + # 消息处理跨连接跟踪;网络断线不等待长任务,主动停止才取消收尾。 + if cancelled or getattr(self, "_should_stop", False): + await self._cancel_message_tasks() logger.info("[WebSocket] 接收循环已停止") async def _dispatch_message(self, data: dict[str, Any]) -> None: diff --git a/tests/test_onebot_connection_lifecycle.py b/tests/test_onebot_connection_lifecycle.py new file mode 100644 index 00000000..15ec8473 --- /dev/null +++ b/tests/test_onebot_connection_lifecycle.py @@ -0,0 +1,107 @@ +"""OneBot reconnect does not wait for unrelated message work.""" + +from __future__ import annotations + +import asyncio +import json +from typing import Any, cast + +import pytest +from websockets.exceptions import ConnectionClosedOK + +from Undefined.onebot.client import OneBotClient + + +class _WebSocket: + def __init__(self, handler_started: asyncio.Event | None = None) -> None: + self.handler_started = handler_started + self.sent_message = False + self.closed = asyncio.Event() + + async def recv(self) -> str: + if self.handler_started is not None: + if not self.sent_message: + self.sent_message = True + return json.dumps({"post_type": "message", "message_type": "group"}) + await self.handler_started.wait() + else: + await self.closed.wait() + raise ConnectionClosedOK(None, None) + + async def close(self) -> None: + self.closed.set() + + +@pytest.mark.asyncio +async def test_reconnect_preserves_inflight_handler_without_waiting( + monkeypatch: pytest.MonkeyPatch, +) -> None: + client = OneBotClient("ws://example.invalid") + started = asyncio.Event() + release = asyncio.Event() + finished = asyncio.Event() + reconnected = asyncio.Event() + connections = 0 + + async def handler(data: dict[str, Any]) -> None: + started.set() + await release.wait() + finished.set() + + async def connect() -> None: + nonlocal connections + connections += 1 + client.ws = cast(Any, _WebSocket(started if connections == 1 else None)) + if connections == 2: + reconnected.set() + + client.set_message_handler(handler) + monkeypatch.setattr(client, "connect", connect) + pending: asyncio.Future[dict[str, Any]] = asyncio.get_running_loop().create_future() + client._pending_responses["waiting"] = pending + running = asyncio.create_task(client.run_with_reconnect(reconnect_interval=0)) + try: + await asyncio.wait_for(reconnected.wait(), 1) + assert connections == 2 + assert len(client._tasks) == 1 + assert not finished.is_set() + assert isinstance(pending.exception(), ConnectionError) + release.set() + await asyncio.wait_for(finished.wait(), 1) + await asyncio.sleep(0) + assert not client._tasks + finally: + release.set() + running.cancel() + await asyncio.gather(running, return_exceptions=True) + await client.disconnect() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("disconnect", [False, True]) +async def test_shutdown_cancels_and_collects_message_handlers(disconnect: bool) -> None: + client = OneBotClient("ws://example.invalid") + started = asyncio.Event() + cancelled = asyncio.Event() + + async def handler(data: dict[str, Any]) -> None: + started.set() + try: + await asyncio.Event().wait() + finally: + cancelled.set() + + client.set_message_handler(handler) + client.ws = cast(Any, _WebSocket()) + await client._dispatch_message({"post_type": "message"}) + running = asyncio.create_task(client.run()) + await asyncio.wait_for(started.wait(), 1) + if disconnect: + await asyncio.wait_for(client.disconnect(), 1) + await asyncio.wait_for(running, 1) + else: + running.cancel() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(running, 1) + assert cancelled.is_set() + assert not client._tasks From 9a3803fedf4fecae8016f13bc93cfc7ea16ac6d1 Mon Sep 17 00:00:00 2001 From: Null <1708213363@qq.com> Date: Tue, 6 Oct 2026 12:06:53 +0800 Subject: [PATCH 016/121] fix(ai): propagate cancellation to queued model calls --- docs/message-batching.md | 1 + src/Undefined/ai/client/queue.py | 17 +++ .../services/coordinator/background.py | 18 ++- tests/test_queued_llm_cancellation.py | 134 ++++++++++++++++++ 4 files changed, 169 insertions(+), 1 deletion(-) create mode 100644 tests/test_queued_llm_cancellation.py diff --git a/docs/message-batching.md b/docs/message-batching.md index 77557b0a..726ef228 100644 --- a/docs/message-batching.md +++ b/docs/message-batching.md @@ -149,6 +149,7 @@ allow_cancel_after_send = false - `flush_all()` 在关停时设置 shutdown 标记,循环遍历所有桶执行等价 T1 路径,并 `await` 所有未完成的 flush task;若收尾过程中又出现新桶,会继续清空直到没有 pending bucket。shutdown 之后的新消息直接发车,不再开缓冲桶。 - `MessageHandler.close()` 的顺序是:停止自动管线热重载 → `message_batcher.flush_all()` → `queue_manager.drain()` 等待已入队/在途回复自然收敛 → `queue_manager.stop()` → flush 历史落盘。 - coordinator 在 `execute_reply` 入口调用 `register_inflight(scope, sender_id, task, ctx)`,在 `finally` 调 `unregister_inflight(...)`;`asyncio.CancelledError` 被识别为 "投机抢占",仅记录信息日志且不重试。 +- 主调用取消或等待超时会同步取消其 LLM 子请求:尚未执行或正在等待重试的子请求会跳过模型调用,已执行的模型任务会收到取消。已经发到上游的请求无法撤回,但不会继续排队重试或产生无人消费的新调用。 ### 兼容回退 diff --git a/src/Undefined/ai/client/queue.py b/src/Undefined/ai/client/queue.py index fb97fcb1..7262e043 100644 --- a/src/Undefined/ai/client/queue.py +++ b/src/Undefined/ai/client/queue.py @@ -3,6 +3,7 @@ from __future__ import annotations import asyncio +from dataclasses import dataclass import logging from typing import Any from uuid import uuid4 @@ -29,6 +30,19 @@ logger = logging.getLogger(__name__) +@dataclass(slots=True) +class QueuedLLMCallState: + """Share caller cancellation with queued retries and the active model task.""" + + cancelled: bool = False + task: asyncio.Task[dict[str, Any]] | None = None + + def cancel(self) -> None: + self.cancelled = True + if self.task is not None and not self.task.done(): + self.task.cancel() + + class ClientQueueMixin(ClientSetupMixin): """统一队列 LLM 调用与会话摘要投递。""" @@ -107,6 +121,7 @@ async def submit_queued_llm_call( ) request_id = uuid4().hex event: asyncio.Event = asyncio.Event() + call_state = QueuedLLMCallState() # 挂起表:QueueManager 回调 set_llm_call_result 时唤醒等待方 self._pending_llm_calls[request_id] = (event, None) model_name = getattr(model_config, "model_name", "default") @@ -121,6 +136,7 @@ async def submit_queued_llm_call( "max_tokens": effective_max_tokens, "transport_state": transport_state, "skip_prefetch_tools": skip_prefetch_tools, + "_llm_call_state": call_state, } ctx = RequestContext.current() if ctx is not None: @@ -168,6 +184,7 @@ async def submit_queued_llm_call( ) raise finally: + call_state.cancel() entry = self._pending_llm_calls.pop(request_id, None) _, result = entry if entry is not None else (None, None) if isinstance(result, Exception): diff --git a/src/Undefined/services/coordinator/background.py b/src/Undefined/services/coordinator/background.py index 34e8b956..1a75fae3 100644 --- a/src/Undefined/services/coordinator/background.py +++ b/src/Undefined/services/coordinator/background.py @@ -3,10 +3,12 @@ from __future__ import annotations +import asyncio import logging from typing import TYPE_CHECKING, Any, cast from Undefined.ai.queue_budget import resolve_effective_retry_count +from Undefined.ai.client.queue import QueuedLLMCallState from Undefined.services.queue_manager import QUEUE_LANE_BACKGROUND from Undefined.utils.resources import read_text_resource @@ -149,12 +151,18 @@ async def _execute_queued_llm_call(self, request: dict[str, Any]) -> None: retry_count = int(request.get("_retry_count", 0) or 0) queue_lane = str(request.get("_queue_lane") or QUEUE_LANE_BACKGROUND) call_type = str(request.get("call_type", "background") or "background") + call_state = request.get("_llm_call_state") + if isinstance(call_state, QueuedLLMCallState) and call_state.cancelled: + logger.info( + "[queued_llm_cancelled] 跳过已取消的调用: request_id=%s", request_id + ) + return try: max_tokens_raw = request.get("max_tokens") if max_tokens_raw is None: max_tokens_raw = getattr(request["model_config"], "max_tokens", 4096) max_tokens = int(max_tokens_raw) if max_tokens_raw is not None else 4096 - result = await self.ai.request_model( + model_call = self.ai.request_model( model_config=request["model_config"], messages=request["messages"], tools=request.get("tools"), @@ -164,6 +172,14 @@ async def _execute_queued_llm_call(self, request: dict[str, Any]) -> None: transport_state=request.get("transport_state"), skip_prefetch_tools=bool(request.get("skip_prefetch_tools", False)), ) + if isinstance(call_state, QueuedLLMCallState): + call_state.task = asyncio.create_task(model_call) + try: + result = await call_state.task + finally: + call_state.task = None + else: + result = await model_call self.ai.set_llm_call_result(request_id, result) if retry_count > 0: logger.info( diff --git a/tests/test_queued_llm_cancellation.py b/tests/test_queued_llm_cancellation.py new file mode 100644 index 00000000..2ae7fa4b --- /dev/null +++ b/tests/test_queued_llm_cancellation.py @@ -0,0 +1,134 @@ +"""Caller cancellation propagates through queued LLM execution and retries.""" + +from __future__ import annotations + +import asyncio +from types import SimpleNamespace +from typing import Any +from unittest.mock import AsyncMock + +import pytest + +from Undefined.ai.client import AIClient +from Undefined.ai.client import queue as queue_module +from Undefined.services.coordinator.background import BackgroundMixin + + +class _CapturingQueue: + def __init__(self) -> None: + self.enqueued = asyncio.Event() + self.request: dict[str, Any] = {} + + def get_max_retries(self) -> int: + return 1 + + async def add_queued_llm_request( + self, request: dict[str, Any], **kwargs: Any + ) -> SimpleNamespace: + self.request = dict(request) + self.enqueued.set() + return SimpleNamespace(estimated_wait_seconds=0) + + +def _client_and_coordinator() -> tuple[Any, Any, _CapturingQueue]: + queue = _CapturingQueue() + client: Any = object.__new__(AIClient) + client._queue_manager = queue + client._pending_llm_calls = {} + client._get_runtime_config = lambda: SimpleNamespace(ai_request_max_retries=1) + client.request_model = AsyncMock(return_value={"choices": []}) + coordinator: Any = object.__new__(BackgroundMixin) + coordinator.ai = client + coordinator.config = SimpleNamespace(ai_request_max_retries=1) + coordinator.queue_manager = queue + return client, coordinator, queue + + +def _submit(client: Any) -> asyncio.Task[dict[str, Any]]: + return asyncio.create_task( + client.submit_queued_llm_call( + model_config=SimpleNamespace(model_name="chat", max_tokens=64), + messages=[{"role": "user", "content": "hello"}], + queue_lane="background", + ) + ) + + +def _zero_timeout(*args: Any, **kwargs: Any) -> float: + return 0.0 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("timeout", [False, True]) +async def test_cancelled_waiter_is_skipped_before_model_dispatch( + monkeypatch: pytest.MonkeyPatch, timeout: bool +) -> None: + client, coordinator, queue = _client_and_coordinator() + if timeout: + monkeypatch.setattr( + queue_module, "compute_queued_llm_timeout_seconds", _zero_timeout + ) + caller = _submit(client) + await asyncio.wait_for(queue.enqueued.wait(), 1) + if not timeout: + caller.cancel() + with pytest.raises(TimeoutError if timeout else asyncio.CancelledError): + await caller + await coordinator._execute_queued_llm_call(queue.request) + client.request_model.assert_not_awaited() + assert not client._pending_llm_calls + + +@pytest.mark.asyncio +async def test_caller_cancellation_stops_active_model_task() -> None: + client, coordinator, queue = _client_and_coordinator() + started = asyncio.Event() + cancelled = asyncio.Event() + + async def request_model(**kwargs: Any) -> dict[str, Any]: + started.set() + try: + await asyncio.Event().wait() + finally: + cancelled.set() + return {} + + client.request_model = AsyncMock(side_effect=request_model) + caller = _submit(client) + await asyncio.wait_for(queue.enqueued.wait(), 1) + dispatch = asyncio.create_task(coordinator._execute_queued_llm_call(queue.request)) + await asyncio.wait_for(started.wait(), 1) + caller.cancel() + with pytest.raises(asyncio.CancelledError): + await caller + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(dispatch, 1) + assert cancelled.is_set() + assert not client._pending_llm_calls + + +@pytest.mark.asyncio +async def test_cancelled_waiter_is_not_called_again_after_retryable_failure() -> None: + client, coordinator, queue = _client_and_coordinator() + client.request_model = AsyncMock(side_effect=RuntimeError("retryable")) + caller = _submit(client) + await asyncio.wait_for(queue.enqueued.wait(), 1) + with pytest.raises(RuntimeError, match="retryable"): + await coordinator._execute_queued_llm_call(queue.request) + assert not caller.done() + caller.cancel() + with pytest.raises(asyncio.CancelledError): + await caller + await coordinator._execute_queued_llm_call({**queue.request, "_retry_count": 1}) + client.request_model.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_uncancelled_queued_call_delivers_its_result() -> None: + client, coordinator, queue = _client_and_coordinator() + caller = _submit(client) + await asyncio.wait_for(queue.enqueued.wait(), 1) + await coordinator._execute_queued_llm_call(queue.request) + assert await asyncio.wait_for(caller, 1) == {"choices": []} + client.request_model.assert_awaited_once() + assert not client._pending_llm_calls From 67d61b4d6cf4e962ae3fe4b0c1e95dcfec877291 Mon Sep 17 00:00:00 2001 From: Null <1708213363@qq.com> Date: Tue, 6 Oct 2026 12:07:02 +0800 Subject: [PATCH 017/121] fix(config): preserve valid settings and retry failed hot updates --- docs/configuration.md | 6 +- docs/python-api.md | 4 +- src/Undefined/config/build_config.py | 11 +- src/Undefined/config/config_class.py | 9 +- src/Undefined/config/hot_reload.py | 14 +- src/Undefined/config/load_sections/domains.py | 11 +- src/Undefined/config/manager.py | 20 +- src/Undefined/config/parsers/helpers.py | 6 +- src/Undefined/config/webui_settings.py | 8 +- tests/test_config_from_mapping.py | 40 ++++ tests/test_config_hot_reload.py | 193 +++++++++++++++++- 11 files changed, 296 insertions(+), 26 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index a3939cdd..771f8c8d 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -69,6 +69,8 @@ Python 显式 mapping / builder.override > config.toml > 环境变量 > `from_mapping` / `builder` 仍会读取进程环境变量中**已注册**的兜底项(TOML / mapping 未提供的字段)。注册表见 [`env_registry.py`](../src/Undefined/config/env_registry.py) 与本文 [§8 环境变量兜底](#8-环境变量兜底迁移建议)。 +内存构建的所有字段(包括 `[webui]`)都来自传入的 mapping 与上述环境变量;不会读取 `config.toml`、`config.local.json` 或 `.env`。动态管理员文件只在 `Config.load()` 的文件加载入口合并,内存构建仍会把显式超级管理员纳入管理员列表。 + ### 2.3 `Config.from_mapping` 结构与 `config.toml` 一致,例如: @@ -1489,11 +1491,13 @@ api_key = "replace-with-your-key" ### 5.5 热更新失败的可见性 +TOML 语法错误、编码错误或读取失败会中止本次加载,热重载保留上次有效的完整配置,包括白名单、Runtime API 密钥和 WebUI 密码;已加载配置的文件被删除时也不会切换为默认值。修正文件并再次保存后会正常重载。`strict=False` 只放宽必填项校验,不允许把损坏的文件作为空配置使用。 + 热更新在共享的 `Config` 实例上逐字段就地生效,整个过程没有 `await`,同一事件循环内的读方不会在一次读取里看到“改了一半”的对象;跨 `await` 的多次读取仍可能分别落在变更前后,需要严格一致的快照时请在单次读取中取全所需字段。 应用阶段按步骤隔离:单步抛错不会中断后续步骤,失败步骤会以 `error` 级日志逐条打印(`热更新步骤失败`),并在末尾汇总一条“热更新未完全生效(运行时状态与 config.toml 不一致)”。异步步骤(技能热重载、自动化并发、配置监听器重启)使用被强引用跟踪的后台任务,任务异常同样以 `error` 级日志输出(`热更新后台任务失败`)。配置订阅者回调抛错时记录失败回调名单,其余回调继续执行。 -因此若看到上述日志,说明 `config.toml` 已改但对应运行时未生效,需要修复报错原因后重新保存配置;必要时重启进程。 +失败的同步回调、应用步骤和后台任务会保留相应变更。修复报错原因后重新保存相同配置也会重试,不必先把字段改成另一个值;已被后续配置覆盖的旧后台失败不会重放旧值。因此若看到上述应用失败日志,需要修复报错原因后重新保存配置;必要时重启进程。 - 具体功能是否“立刻体现”,取决于模块是“每次读取配置”还是“启动时缓存”。 - 对于行为不确定项,建议改完观察日志;必要时重启进程确认。 diff --git a/docs/python-api.md b/docs/python-api.md index 4b243f20..6d7ef330 100644 --- a/docs/python-api.md +++ b/docs/python-api.md @@ -107,10 +107,12 @@ OneBot 内部时间解析提供两种约定:`parse_message_time(message)` 在 Python 显式 mapping / override > config.toml > 环境变量 > 代码默认值 ``` -- `Config.from_mapping()` / `Config.builder()`:**不读取** `config.toml`,适合测试与无文件部署。 +- `Config.from_mapping()` / `Config.builder()`:**不读取** `config.toml`、`config.local.json` 或 `.env`,适合测试与无文件部署;`[webui]` 与其余字段一样使用传入 mapping,显式超级管理员仍自动加入管理员列表。 - `Config.load()`:从指定或 CWD 下的 `config.toml` 加载(CLI 路径)。 - 环境变量仅在 TOML / mapping **未提供**对应项时兜底;详见 [配置详解 — 环境变量兜底](configuration.md#8-环境变量兜底迁移建议)。 +`Config.load(strict=False)` 只放宽必填项校验,文件的 TOML 语法、编码或读取错误仍会抛出 `ValueError`。`ConfigManager.reload()` 加载失败时保留原配置;订阅者应用失败会记录日志并保留变更,下次重载相同内容也会再次通知。运行时热更新应用函数会在完成其余同步步骤后汇报失败,后台失败同样会进入重试。 + ### `Config.from_mapping` 从内存 dict 构建配置,结构与 `config.toml` 一致: diff --git a/src/Undefined/config/build_config.py b/src/Undefined/config/build_config.py index a0e3562b..df353b94 100644 --- a/src/Undefined/config/build_config.py +++ b/src/Undefined/config/build_config.py @@ -3,8 +3,7 @@ from __future__ import annotations -from pathlib import Path -from typing import TYPE_CHECKING, Any, Optional +from typing import TYPE_CHECKING, Any if TYPE_CHECKING: from .config_class import Config @@ -28,7 +27,7 @@ def build_config( data: dict[str, Any], *, strict: bool = True, - config_path: Optional[Path] = None, + include_local_admins: bool = False, ) -> "Config": """从已解析的 TOML mapping 构建 Config。""" from .config_class import Config @@ -42,7 +41,9 @@ def build_config( # 合并 config.toml 与本地 admins.json,超管始终纳入 admin 列表 superadmin_qq, admin_qqs = _merge_admins( - superadmin_qq=ctx["superadmin_qq"], admin_qqs=ctx["admin_qqs"] + superadmin_qq=ctx["superadmin_qq"], + admin_qqs=ctx["admin_qqs"], + include_local_admins=include_local_admins, ) ctx["superadmin_qq"] = superadmin_qq ctx["admin_qqs"] = admin_qqs @@ -52,6 +53,6 @@ def build_config( ctx.update(load_network(data)) ctx.update(load_integrations(data)) # domains 含 WebUI/API/认知/合并器等子域,放最后以便前面模型段已就绪 - ctx.update(load_domains(data, config_path=config_path)) + ctx.update(load_domains(data)) load_finalize(ctx, strict=strict) # strict 时校验必填项并打 debug 摘要 return Config(**ctx) diff --git a/src/Undefined/config/config_class.py b/src/Undefined/config/config_class.py index b23d0e7e..968c7d2c 100644 --- a/src/Undefined/config/config_class.py +++ b/src/Undefined/config/config_class.py @@ -394,15 +394,16 @@ def load(cls, config_path: Optional[Path] = None, strict: bool = True) -> "Confi from .build_config import build_config _load_env() # 先加载 .env,供 _get_value 环境变量回退 - data = load_toml_data(config_path, strict=strict) - return build_config(data, strict=strict, config_path=config_path) + # strict 只控制必填项校验,损坏的文件始终不能当作空配置应用。 + data = load_toml_data(config_path, strict=True) + return build_config(data, strict=strict, include_local_admins=True) @classmethod def from_mapping(cls, data: dict[str, Any], *, strict: bool = True) -> "Config": """从内存 mapping 构建配置(无 TOML 文件)。""" from .build_config import build_config - return build_config(data, strict=strict, config_path=None) + return build_config(data, strict=strict) @classmethod def builder(cls) -> "ConfigBuilder": @@ -742,4 +743,4 @@ def build(self, *, strict: bool = True) -> Config: # TOML-style nested overrides for simple top-level keys only for key, value in self._overrides.items(): data[key] = value - return build_config(data, strict=strict, config_path=None) + return build_config(data, strict=strict) diff --git a/src/Undefined/config/hot_reload.py b/src/Undefined/config/hot_reload.py index 1dc9d350..37eb2990 100644 --- a/src/Undefined/config/hot_reload.py +++ b/src/Undefined/config/hot_reload.py @@ -159,6 +159,7 @@ class HotReloadContext: def _spawn_hot_reload_task( coro: Coroutine[Any, Any, None], description: str, + on_failure: Callable[[], None] | None = None, ) -> None: """创建热更新后台任务并回收异常,避免静默失败。""" task = asyncio.create_task(coro, name=f"config-hot-reload:{description}") @@ -170,6 +171,8 @@ def _on_done(finished: asyncio.Task[None]) -> None: return exc = finished.exception() if exc is not None: + if on_failure is not None: + on_failure() logger.error( "[配置] 热更新后台任务失败: %s(该部分配置未生效)", description, @@ -198,6 +201,9 @@ def apply_config_updates( handler = context.message_handler + def _retry_failed_update() -> None: + context.config_manager.retry_failed_update(changes) + def _apply_security() -> None: context.security_service.apply_config(updated) @@ -244,6 +250,7 @@ def _apply_automations() -> None: _spawn_hot_reload_task( _apply_message_handler_automations_hot_reload(updated, handler), "automations", + on_failure=_retry_failed_update, ) def _apply_model_configs() -> None: @@ -261,10 +268,12 @@ def _apply_skills_reload() -> None: _spawn_hot_reload_task( _apply_skills_hot_reload(updated, context.ai_client), "skills", + on_failure=_retry_failed_update, ) _spawn_hot_reload_task( _apply_message_handler_skills_hot_reload(updated, handler), "message-handler-skills", + on_failure=_retry_failed_update, ) def _apply_config_watcher() -> None: @@ -275,6 +284,7 @@ def _apply_config_watcher() -> None: updated.skills_hot_reload_debounce, ), "config-watcher", + on_failure=_retry_failed_update, ) steps: list[tuple[str, Callable[[], None]]] = [] @@ -319,6 +329,7 @@ def _apply_config_watcher() -> None: "如怀疑异步部分未生效,请检索『热更新后台任务失败』日志", ", ".join(failed), ) + raise RuntimeError(f"热更新步骤失败: {', '.join(failed)}") def _log_restart_required(changed_keys: set[str]) -> None: @@ -410,6 +421,7 @@ def _registry_name(registry: Any) -> str: logger.info("[配置] 技能热重载已禁用") if failed: logger.error("[配置] 部分注册表热重载未停止: %s", ", ".join(failed)) + raise RuntimeError(f"技能热重载未停止: {', '.join(failed)}") return for registry in registries: @@ -426,7 +438,7 @@ def _registry_name(registry: Any) -> str: ) if failed: logger.error("[配置] 以下注册表热重载未生效: %s", ", ".join(failed)) - return + raise RuntimeError(f"技能热重载未生效: {', '.join(failed)}") logger.info( "[配置] 技能热重载已更新: interval=%.2fs debounce=%.2fs", updated.skills_hot_reload_interval, diff --git a/src/Undefined/config/load_sections/domains.py b/src/Undefined/config/load_sections/domains.py index 5d4028f1..d3a78898 100644 --- a/src/Undefined/config/load_sections/domains.py +++ b/src/Undefined/config/load_sections/domains.py @@ -5,8 +5,7 @@ # 配置分段加载:按 table 解析 TOML → ctx 字段 dict import logging -from pathlib import Path -from typing import Any, Optional +from typing import Any from ..domain_parsers import ( _parse_api_config, @@ -25,16 +24,14 @@ _parse_image_gen_config, _parse_image_gen_model_config, ) -from ..webui_settings import load_webui_settings +from ..webui_settings import parse_webui_settings logger = logging.getLogger(__name__) -def load_domains( - data: dict[str, Any], *, config_path: Optional[Path] = None -) -> dict[str, Any]: +def load_domains(data: dict[str, Any]) -> dict[str, Any]: # 子域配置:WebUI/API/认知/表情包/合并器/Naga/生图等,与 core/models 段解耦 - webui_settings = load_webui_settings(config_path) + webui_settings = parse_webui_settings(data) api_config = _parse_api_config(data) cognitive = _parse_cognitive_config(data) diff --git a/src/Undefined/config/manager.py b/src/Undefined/config/manager.py index 825d4b71..50f93851 100644 --- a/src/Undefined/config/manager.py +++ b/src/Undefined/config/manager.py @@ -24,6 +24,7 @@ def __init__(self, config_path: Path | None = None) -> None: self._watch_stop: asyncio.Event | None = None self._last_snapshot: dict[str, tuple[int, int]] = {} self._reload_lock = asyncio.Lock() + self._pending_changes: dict[str, tuple[Any, Any]] = {} def load(self, strict: bool = True) -> Config: if self._config is None: @@ -33,17 +34,33 @@ def load(self, strict: bool = True) -> Config: def replace(self, config: Config) -> None: """替换缓存的配置实例(库嵌入 ``set_config`` 注入时使用)。""" self._config = config + self._pending_changes.clear() def reload(self, strict: bool = False) -> dict[str, tuple[Any, Any]]: if self._config is None: self._config = Config.load(config_path=self.config_path, strict=strict) return {} + if not self.config_path.is_file(): + raise FileNotFoundError(f"配置文件不存在,保留当前配置: {self.config_path}") new_config = Config.load(config_path=self.config_path, strict=strict) - changes = self._config.update_from(new_config) + changes = dict(self._pending_changes) + changes.update(self._config.update_from(new_config)) + self._pending_changes.clear() if changes: self._notify(changes) return changes + def retry_failed_update(self, changes: dict[str, tuple[Any, Any]]) -> None: + """保留未应用的变更,下次保存时重试;忽略已被新配置取代的异步失败。""" + if self._config is None: + return + for key, change in changes.items(): + current: Any = self._config + for part in key.split("."): + current = getattr(current, part, None) + if current == change[1]: + self._pending_changes[key] = change + def subscribe(self, callback: ConfigChangeCallback) -> None: self._callbacks.append(callback) @@ -128,6 +145,7 @@ def _notify(self, changes: dict[str, tuple[Any, Any]]) -> None: try: callback(config, changes) except Exception: + self.retry_failed_update(changes) name = getattr(callback, "__qualname__", repr(callback)) failed.append(name) logger.error("[配置] 热更新回调执行失败: %s", name, exc_info=True) diff --git a/src/Undefined/config/parsers/helpers.py b/src/Undefined/config/parsers/helpers.py index c60baeea..fa47676e 100644 --- a/src/Undefined/config/parsers/helpers.py +++ b/src/Undefined/config/parsers/helpers.py @@ -22,9 +22,11 @@ # 合并多来源配置 -def _merge_admins(superadmin_qq: int, admin_qqs: list[int]) -> tuple[int, list[int]]: +def _merge_admins( + superadmin_qq: int, admin_qqs: list[int], *, include_local_admins: bool = True +) -> tuple[int, list[int]]: # admins.json 与 config.toml 的 admin_qq 合并,去重后超管必在列表中 - local_admins = load_local_admins() + local_admins = load_local_admins() if include_local_admins else [] all_admins = list(set(admin_qqs + local_admins)) if superadmin_qq and superadmin_qq not in all_admins: all_admins.append(superadmin_qq) diff --git a/src/Undefined/config/webui_settings.py b/src/Undefined/config/webui_settings.py index 47206f51..46f4303f 100644 --- a/src/Undefined/config/webui_settings.py +++ b/src/Undefined/config/webui_settings.py @@ -4,7 +4,7 @@ from dataclasses import dataclass from pathlib import Path -from typing import Optional +from typing import Any, Optional from .coercers import _coerce_bool, _coerce_int, _coerce_str, _normalize_str, _get_value @@ -36,7 +36,11 @@ def display_url(self) -> str: def load_webui_settings(config_path: Optional[Path] = None) -> WebUISettings: from .loader import load_toml_data # lazy to avoid circular - data = load_toml_data(config_path) + return parse_webui_settings(load_toml_data(config_path)) + + +def parse_webui_settings(data: dict[str, Any]) -> WebUISettings: + """从已解析的配置构建 WebUI 设置,不再读取文件。""" config_exists = bool(data) url_value = _get_value(data, ("webui", "url"), None) port_value = _get_value(data, ("webui", "port"), None) diff --git a/tests/test_config_from_mapping.py b/tests/test_config_from_mapping.py index 5d88efba..ad1a5d2d 100644 --- a/tests/test_config_from_mapping.py +++ b/tests/test_config_from_mapping.py @@ -1,6 +1,7 @@ from __future__ import annotations from pathlib import Path +from typing import Any import pytest @@ -35,6 +36,45 @@ def test_from_mapping_builds_without_toml() -> None: assert cfg.chat_model.model_name == "gpt-test" +@pytest.mark.parametrize("use_builder", [False, True]) +def test_memory_config_uses_webui_mapping_without_disk_reads( + monkeypatch: pytest.MonkeyPatch, use_builder: bool +) -> None: + def _unexpected_read(*args: Any, **kwargs: Any) -> Any: + raise AssertionError("内存配置不应读取 TOML 或本地管理员文件") + + monkeypatch.setattr("Undefined.config.loader.load_toml_data", _unexpected_read) + monkeypatch.setattr( + "Undefined.config.config_class.load_toml_data", _unexpected_read + ) + monkeypatch.setattr( + "Undefined.config.parsers.helpers.load_local_admins", _unexpected_read + ) + monkeypatch.setenv("WEBUI_PORT", "19876") + mapping = { + **_MINIMAL_MAPPING, + "core": {"superadmin_qq": 11, "admin_qq": [22]}, + "webui": { + "url": "::1", + "port": 12345, + "password": "mapping-password", + "autostart_bot": True, + "check_updates": False, + }, + } + config = ( + Config.builder().with_mapping(mapping).build(strict=False) + if use_builder + else Config.from_mapping(mapping, strict=False) + ) + assert config.webui_url == "::1" + assert config.webui_port == 12345 + assert config.webui_password == "mapping-password" + assert config.webui_autostart_bot is True + assert config.webui_check_updates is False + assert set(config.admin_qqs) == {11, 22} + + def test_tool_search_defaults() -> None: cfg = Config.from_mapping(_MINIMAL_MAPPING, strict=False) diff --git a/tests/test_config_hot_reload.py b/tests/test_config_hot_reload.py index 3e77b5a4..c046af41 100644 --- a/tests/test_config_hot_reload.py +++ b/tests/test_config_hot_reload.py @@ -7,6 +7,7 @@ import pytest +from Undefined.config import Config from Undefined.config.hot_reload import HotReloadContext, apply_config_updates @@ -705,7 +706,7 @@ def apply_config(self, config: Any) -> None: security_service=cast(Any, _BrokenSecurity()), ) - with caplog.at_level("ERROR"): + with caplog.at_level("ERROR"), pytest.raises(RuntimeError, match="security"): apply_config_updates( updated, {"security_model.model_name": ("old", "new")}, @@ -763,6 +764,194 @@ def _record(config: Any, changes: dict[str, Any]) -> None: assert "回调未完成" in caplog.text +@pytest.mark.parametrize("broken", ["[core\n", "\xff"]) +def test_invalid_reload_preserves_access_and_authentication( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, broken: str +) -> None: + from Undefined.config.manager import ConfigManager + + monkeypatch.chdir(tmp_path) + monkeypatch.setattr("Undefined.config.config_class._load_env", lambda: None) + path = tmp_path / "config.toml" + valid = '[access]\nmode="allowlist"\nallowed_group_ids=[111]\n[api]\nauth_key="private-key"\n' + path.write_text(valid, encoding="utf-8") + manager = ConfigManager(path) + config = manager.load(strict=False) + calls: list[dict[str, Any]] = [] + manager.subscribe(lambda _config, changes: calls.append(changes)) + + path.write_bytes(broken.encode("latin-1")) + with pytest.raises(ValueError): + manager.reload(strict=False) + assert manager.load(strict=False) is config + assert config.access_mode == "allowlist" + assert config.is_group_allowed(111) + assert not config.is_group_allowed(999) + assert config.api.auth_key == "private-key" + assert calls == [] + + path.write_text(valid.replace("[111]", "[222]"), encoding="utf-8") + assert manager.reload(strict=False) + assert config.is_group_allowed(222) + assert not config.is_group_allowed(111) + assert len(calls) == 1 + + +def test_failed_callback_retries_identical_saved_config( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + from Undefined.config.manager import ConfigManager + + monkeypatch.chdir(tmp_path) + monkeypatch.setattr("Undefined.config.config_class._load_env", lambda: None) + path = tmp_path / "config.toml" + path.write_text('[models.chat]\nmodel_name="old"\n', encoding="utf-8") + manager = ConfigManager(path) + config = manager.load(strict=False) + attempts = 0 + applied: list[str] = [] + + def _apply(updated: Config, changes: dict[str, Any]) -> None: + nonlocal attempts + attempts += 1 + assert "chat_model.model_name" in changes + if attempts == 1: + raise RuntimeError("transient failure") + applied.append(updated.chat_model.model_name) + + manager.subscribe(_apply) + path.write_text('[models.chat]\nmodel_name="new"\n', encoding="utf-8") + assert manager.reload(strict=False) + assert config.chat_model.model_name == "new" + assert applied == [] + assert manager.reload(strict=False) + assert applied == ["new"] + assert manager.reload(strict=False) == {} + assert attempts == 2 + + +def test_runtime_step_failure_retries_identical_saved_config( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + from Undefined.config.manager import ConfigManager + + class _FlakyQueue(_FakeQueueManager): + def update_max_retries(self, max_retries: int) -> None: + if not self.max_retries: + self.max_retries.append(-1) + raise RuntimeError("queue temporarily unavailable") + super().update_max_retries(max_retries) + + monkeypatch.chdir(tmp_path) + monkeypatch.setattr("Undefined.config.config_class._load_env", lambda: None) + path = tmp_path / "config.toml" + path.write_text("[core]\nai_request_max_retries=2\n", encoding="utf-8") + manager = ConfigManager(path) + manager.load(strict=False) + queue = _FlakyQueue() + context = HotReloadContext( + ai_client=cast(Any, _FakeAIClient()), + queue_manager=cast(Any, queue), + config_manager=manager, + security_service=cast(Any, _FakeSecurityService()), + ) + manager.subscribe( + lambda config, changes: apply_config_updates(config, changes, context) + ) + path.write_text("[core]\nai_request_max_retries=7\n", encoding="utf-8") + manager.reload(strict=False) + assert queue.max_retries == [-1] + manager.reload(strict=False) + assert queue.max_retries == [-1, 7] + assert manager.reload(strict=False) == {} + + +@pytest.mark.asyncio +async def test_runtime_background_failure_retries_identical_saved_config( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + from Undefined.config.manager import ConfigManager + + class _FlakyHandler(_FakeMessageHandler): + async def apply_automations_hot_reload_config( + self, *, max_concurrent: int + ) -> None: + if not self.automation_updates: + self.automation_updates.append(-1) + raise RuntimeError("automation temporarily unavailable") + await super().apply_automations_hot_reload_config( + max_concurrent=max_concurrent + ) + + monkeypatch.chdir(tmp_path) + monkeypatch.setattr("Undefined.config.config_class._load_env", lambda: None) + path = tmp_path / "config.toml" + path.write_text("[automations]\nmax_concurrent=2\n", encoding="utf-8") + manager = ConfigManager(path) + manager.load(strict=False) + handler = _FlakyHandler() + context = HotReloadContext( + ai_client=cast(Any, _FakeAIClient()), + queue_manager=cast(Any, _FakeQueueManager()), + config_manager=manager, + security_service=cast(Any, _FakeSecurityService()), + message_handler=cast(Any, handler), + ) + manager.subscribe( + lambda config, changes: apply_config_updates(config, changes, context) + ) + path.write_text("[automations]\nmax_concurrent=7\n", encoding="utf-8") + manager.reload(strict=False) + for _ in range(3): + await asyncio.sleep(0) + assert handler.automation_updates == [-1] + manager.reload(strict=False) + for _ in range(3): + await asyncio.sleep(0) + assert handler.automation_updates == [-1, 7] + assert manager.reload(strict=False) == {} + + +@pytest.mark.asyncio +async def test_background_failure_retries_without_replaying_stale_values( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + from Undefined.config.hot_reload import _spawn_hot_reload_task + from Undefined.config.manager import ConfigManager + + monkeypatch.chdir(tmp_path) + monkeypatch.setattr("Undefined.config.config_class._load_env", lambda: None) + path = tmp_path / "config.toml" + path.write_text('[models.chat]\nmodel_name="old"\n', encoding="utf-8") + manager = ConfigManager(path) + manager.load(strict=False) + attempts: list[str] = [] + release = asyncio.Event() + + async def _fail() -> None: + await release.wait() + raise RuntimeError("background failure") + + def _apply(updated: Config, changes: dict[str, Any]) -> None: + attempts.append(updated.chat_model.model_name) + if len(attempts) == 1: + _spawn_hot_reload_task( + _fail(), "retry-test", lambda: manager.retry_failed_update(changes) + ) + + manager.subscribe(_apply) + path.write_text('[models.chat]\nmodel_name="new"\n', encoding="utf-8") + manager.reload(strict=False) + release.set() + for _ in range(3): + await asyncio.sleep(0) + assert manager.reload(strict=False) + assert attempts == ["new", "new"] + + manager.retry_failed_update({"chat_model.model_name": ("old", "obsolete")}) + assert manager.reload(strict=False) == {} + + def test_weixin_config_updates_in_place_for_hot_reload(tmp_path: Path) -> None: """WeixinService 持有 config.weixin 对象,热更新必须原地改字段而非换身份。""" from dataclasses import replace as dc_replace @@ -805,7 +994,7 @@ def test_security_step_skipped_when_unrelated_keys_change() -> None: ) security_service = _FakeSecurityService() context = HotReloadContext( - ai_client=cast(Any, SimpleNamespace()), + ai_client=cast(Any, SimpleNamespace(apply_search_config=lambda _url: None)), queue_manager=cast(Any, _FakeQueueManager()), config_manager=cast(Any, SimpleNamespace()), security_service=cast(Any, security_service), From 55a238ab5015ec46e7562c6adc4c51784db3177d Mon Sep 17 00:00:00 2001 From: Null <1708213363@qq.com> Date: Tue, 6 Oct 2026 12:08:01 +0800 Subject: [PATCH 018/121] fix(skills): preserve inflight stats across hot reload --- src/Undefined/skills/README.md | 2 ++ src/Undefined/skills/registry.py | 10 ++++--- tests/test_skills_registry_stats.py | 45 +++++++++++++++++++++++++++++ 3 files changed, 53 insertions(+), 4 deletions(-) diff --git a/src/Undefined/skills/README.md b/src/Undefined/skills/README.md index ca78dd17..bd9cabae 100644 --- a/src/Undefined/skills/README.md +++ b/src/Undefined/skills/README.md @@ -4,6 +4,8 @@ 技能目录,包含基础工具(tools)、智能代理(agents)、工具集合(toolsets)和 Anthropic Skills(anthropic_skills)。它构成了 Undefined 机器人的所有核心能力基座。 +技能热重载删除或重命名工具时,已经开始的调用继续使用原处理器和统计对象完成,成功结果、异常与取消语义保持不变;后续调用使用重载后的技能列表。 + ## 目录结构 ``` diff --git a/src/Undefined/skills/registry.py b/src/Undefined/skills/registry.py index f1fbb3bc..60a2afc9 100644 --- a/src/Undefined/skills/registry.py +++ b/src/Undefined/skills/registry.py @@ -490,6 +490,8 @@ async def _execute( args = parse_tool_arguments(args, logger=logger, tool_name=name) async with self._items_lock: item = self._items.get(name) + if item is not None: + stats = self._stats.setdefault(name, SkillStats()) if not item: if logger.isEnabledFor(logging.INFO) and self.kind in { @@ -547,7 +549,7 @@ async def _execute( result = await self._execute_with_timeout(handler, args, context) duration = time.monotonic() - start_time - self._stats[name].record_success(duration) + stats.record_success(duration) self._log_event( "execute", name, status="success", duration_ms=int(duration * 1000) ) @@ -558,7 +560,7 @@ async def _execute( except RegistryExecutionTimeoutError: duration = time.monotonic() - start_time - self._stats[name].record_failure(duration, "timeout") + stats.record_failure(duration, "timeout") self._log_event( "execute", name, status="timeout", duration_ms=int(duration * 1000) ) @@ -569,7 +571,7 @@ async def _execute( except asyncio.CancelledError: duration = time.monotonic() - start_time - self._stats[name].record_failure(duration, "cancelled") + stats.record_failure(duration, "cancelled") self._log_event( "execute", name, status="cancelled", duration_ms=int(duration * 1000) ) @@ -580,7 +582,7 @@ async def _execute( except Exception as e: duration = time.monotonic() - start_time - self._stats[name].record_failure(duration, str(e)) + stats.record_failure(duration, str(e)) logger.exception(f"[执行异常] {name}") self._log_event( "execute", name, status="error", duration_ms=int(duration * 1000) diff --git a/tests/test_skills_registry_stats.py b/tests/test_skills_registry_stats.py index 0f446074..2b30a9de 100644 --- a/tests/test_skills_registry_stats.py +++ b/tests/test_skills_registry_stats.py @@ -4,6 +4,7 @@ import asyncio from typing import Any +from pathlib import Path import pytest @@ -165,3 +166,47 @@ async def wait_forever(args: dict[str, Any], context: dict[str, Any]) -> str: with pytest.raises(asyncio.CancelledError): await task + + +@pytest.mark.asyncio +@pytest.mark.parametrize("outcome", ["success", "failure", "cancelled"]) +async def test_inflight_execution_keeps_stats_after_skill_removed( + tmp_path: Path, + outcome: str, +) -> None: + started = asyncio.Event() + release = asyncio.Event() + + async def handler(args: dict[str, Any], context: dict[str, Any]) -> str: + _ = args, context + started.set() + await release.wait() + if outcome == "failure": + raise RuntimeError("handler failure") + return "completed" + + registry = BaseRegistry(base_dir=tmp_path, kind="tool", timeout_seconds=0) + registry.register_external_item("removed", _schema("removed"), handler) + stats = registry.get_stats()["removed"] + task = asyncio.create_task(registry.execute_strict("removed", {}, {})) + await started.wait() + await registry._reload_items() + assert "removed" not in registry.get_stats() + + if outcome == "cancelled": + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert stats.last_error == "cancelled" + else: + release.set() + if outcome == "failure": + with pytest.raises(RuntimeError, match="handler failure"): + await task + assert stats.last_error == "handler failure" + else: + assert await task == "completed" + assert stats.success == 1 + assert stats.count == 1 + assert stats.failure == (0 if outcome == "success" else 1) + assert await registry.execute("removed", {}, {}) == "未找到项目: removed" From 3ee4f02175f6e4f7869e6d3cb48f77f5a0226248 Mon Sep 17 00:00:00 2001 From: Null <1708213363@qq.com> Date: Tue, 6 Oct 2026 12:08:33 +0800 Subject: [PATCH 019/121] fix(deploy): abort the wizard on keyboard interrupt --- docs/deployment.md | 2 +- docs/docker-deploy.md | 2 +- src/Undefined/deploy/runner.py | 2 +- tests/test_deploy_runner.py | 50 +++++++++++++++++++++++++++++++++- 4 files changed, 52 insertions(+), 4 deletions(-) diff --git a/docs/deployment.md b/docs/deployment.md index 5c75d6b2..6b5d47c7 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -201,7 +201,7 @@ uv run deploy up 部署结束后,按终端显示的地址和密码进入 WebUI,填写模型 API 与 QQ 身份配置,到 NapCat 扫码登录后,再返回 WebUI 点击“启动机器人”。`container` 与 `host` 模式都会将 `[webui].autostart_bot` 设为 `false`、`[onebot].file_send_mode` 设为 `"stream"`。**环境准备、首次配置、远程访问和常见问题请直接看 [Docker 一键部署指南](docker-deploy.md)**。 -向导在最终确认前只读取配置,取消不会生成文件或初始化子模块;已有 `config.toml` 支持 UTF-8 BOM 编码。`container` 模式还会为 Python 解释器自动映射已共享的 `deploy/data/cache/python_interpreter/`,让宿主 Docker 从正确的宿主路径读取执行文件。手工挂载 Docker socket 的容器需按 [Python 解释器工具说明](../src/Undefined/skills/tools/python_interpreter/README.md#容器内使用宿主-docker)配置共享路径。 +向导在最终确认前只读取配置,选择“否”或按 Ctrl+C 取消都不会生成文件或初始化子模块;Ctrl+C 会立即结束命令,不会采用默认的“继续”。已有 `config.toml` 支持 UTF-8 BOM 编码。`container` 模式还会为 Python 解释器自动映射已共享的 `deploy/data/cache/python_interpreter/`,让宿主 Docker 从正确的宿主路径读取执行文件。手工挂载 Docker socket 的容器需按 [Python 解释器工具说明](../src/Undefined/skills/tools/python_interpreter/README.md#容器内使用宿主-docker)配置共享路径。 --- diff --git a/docs/docker-deploy.md b/docs/docker-deploy.md index 2f612f3a..c7498c20 100644 --- a/docs/docker-deploy.md +++ b/docs/docker-deploy.md @@ -56,7 +56,7 @@ uv run deploy up | 发布端口的绑定地址 | 保持 `127.0.0.1`;在服务器上部署时,访问方法见下文[远程访问](#在自己的电脑上访问服务器) | | 是否拉取 NagaAgent 子模块 | 不需要 NagaAgent 代码问答时选“否” | -向导随后会展示配置变更,最终确认前只读取配置和模板;选择取消不会创建 `config.toml` 或拉取 NagaAgent 子模块。确认后才生成配置、按需初始化子模块、拉取镜像并启动容器。首次运行需要下载依赖与镜像,请等待命令完成。 +向导随后会展示配置变更,最终确认前只读取配置和模板;选择“否”或按 Ctrl+C 取消不会创建 `config.toml` 或拉取 NagaAgent 子模块。Ctrl+C 会立即结束命令,不会采用默认的“继续”。确认后才生成配置、按需初始化子模块、拉取镜像并启动容器。首次运行需要下载依赖与镜像,请等待命令完成。 如需跳过向导,可用 `uv run deploy up --yes` 代替;首次执行采用默认配置,之后执行会沿用上次的部署选择。只想提前查看变更时,可用 `uv run deploy up --dry-run`,它不会写文件或启动容器。 diff --git a/src/Undefined/deploy/runner.py b/src/Undefined/deploy/runner.py index 4ca053e5..6acddee9 100644 --- a/src/Undefined/deploy/runner.py +++ b/src/Undefined/deploy/runner.py @@ -175,7 +175,7 @@ def confirm(self, question: str, default: bool = True) -> bool: def _ask(self, prompt: str) -> str: try: return input(f"{prompt}: ") - except (EOFError, KeyboardInterrupt): + except EOFError: print("\n(未输入,使用默认值)") return "" diff --git a/tests/test_deploy_runner.py b/tests/test_deploy_runner.py index 8b551e6a..bcc574a7 100644 --- a/tests/test_deploy_runner.py +++ b/tests/test_deploy_runner.py @@ -17,7 +17,8 @@ import pytest import yaml -from Undefined.deploy import catalog, docker_cli, generate, nagaagent, runner +from Undefined.deploy import catalog, cli, docker_cli, generate, nagaagent, runner +from Undefined.deploy.catalog import EXIT_ERROR from Undefined.deploy.docker_cli import CommandResult, DockerAvailability from Undefined.deploy.state import DeployLayout, read_state @@ -657,6 +658,53 @@ def unexpected_side_effect(*args: object, **kwargs: object) -> None: assert "已取消,未改动任何文件。" in capsys.readouterr().out +@pytest.mark.parametrize("existing_config", (False, True)) +def test_ctrl_c_at_final_confirmation_aborts_without_side_effects( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], + existing_config: bool, +) -> None: + repo = tmp_path / "repo" + repo.mkdir() + (repo / "config.toml.example").write_text(CONFIG_TOML, encoding="utf-8") + if existing_config: + (repo / "config.toml").write_text(CONFIG_TOML, encoding="utf-8") + before = {path.name: path.read_bytes() for path in repo.iterdir()} + monkeypatch.setattr(runner, "repo_root", lambda: repo) + monkeypatch.setattr(runner, "wizard_enabled", lambda options: True) + + def _interrupt_final_confirmation(prompt: str) -> str: + if "按以上计划继续" in prompt: + raise KeyboardInterrupt + return "" + + def _unexpected_side_effect(*args: object, **kwargs: object) -> None: + raise AssertionError("Ctrl+C 取消前不能写入子模块或调用 Docker") + + monkeypatch.setattr("builtins.input", _interrupt_final_confirmation) + monkeypatch.setattr(nagaagent, "ensure_submodule", _unexpected_side_effect) + monkeypatch.setattr(docker_cli, "check_availability", _unexpected_side_effect) + + assert ( + cli.main( + [ + "up", + "--mode", + "container", + "--with", + "", + "--with-nagaagent", + "--port-bind", + "127.0.0.1", + ] + ) + == EXIT_ERROR + ) + assert {path.name: path.read_bytes() for path in repo.iterdir()} == before + assert "已取消。" in capsys.readouterr().out + + def test_up_accepts_bom_config( fake_repo: Path, monkeypatch: pytest.MonkeyPatch ) -> None: From 9762adf483750a83f1caeadcfd126adc3c9ed86d Mon Sep 17 00:00:00 2001 From: Null <1708213363@qq.com> Date: Tue, 6 Oct 2026 12:09:51 +0800 Subject: [PATCH 020/121] fix(file-analysis): confine archive extraction and cache cleanup --- .../agents/file_analysis_agent/README.md | 2 + .../agents/file_analysis_agent/prompt.md | 1 + .../tools/cleanup_temp/config.json | 2 +- .../tools/cleanup_temp/handler.py | 30 ++- .../tools/extract_archive/config.json | 4 +- .../tools/extract_archive/handler.py | 25 +- src/Undefined/skills/shared.py | 27 +++ tests/test_file_analysis_archive_cleanup.py | 218 ++++++++++++++++++ tests/test_skills_import_boundary.py | 1 - 9 files changed, 296 insertions(+), 14 deletions(-) create mode 100644 tests/test_file_analysis_archive_cleanup.py diff --git a/src/Undefined/skills/agents/file_analysis_agent/README.md b/src/Undefined/skills/agents/file_analysis_agent/README.md index 92b94552..64542628 100644 --- a/src/Undefined/skills/agents/file_analysis_agent/README.md +++ b/src/Undefined/skills/agents/file_analysis_agent/README.md @@ -15,6 +15,8 @@ - 内部附件 UID(`pic_xxx` / `file_xxx`)由工具按当前会话作用域解析;多模态分析可直接传 UID,其他解析工具先用 `download_file` 转成本地路径 - `download_file` 的附件回源遇到目标策略拒绝、响应读取异常等失败时,会记录日志并返回“附件 UID 本地化失败”,不会继续尝试 URL 下载;`TimeoutError` 及其他 `OSError` 仍返回“附件文件读取失败”,任务取消正常向上传递 - PDF 文字提取走 `extract_pdf`;扫描版、图表、版式或指定页码范围视觉分析走 `describe_pdf_page` +- `extract_archive(list)` 可以列出本地压缩包;解压前先用 `download_file` 取得下载缓存内路径,解压目标限定为压缩包所在目录的子目录。TAR 显式使用 `data` 安全过滤器,拒绝越界成员、指向外部的链接及设备文件。缺少 `tarfile.data_filter` 的旧 Python 补丁版本仍可列出文件,但会拒绝 TAR 解压并提示更新补丁版本,不回退到无过滤解压。 +- `cleanup_temp.task_uuid` 只接受下载缓存的直接子目录名,拒绝绝对路径、路径跳转和符号链接;留空清理时跳过符号链接目录。 开发提示: - 解析类工具尽量使用异步 I/O 或 `asyncio.to_thread` diff --git a/src/Undefined/skills/agents/file_analysis_agent/prompt.md b/src/Undefined/skills/agents/file_analysis_agent/prompt.md index bae93718..3a015e43 100644 --- a/src/Undefined/skills/agents/file_analysis_agent/prompt.md +++ b/src/Undefined/skills/agents/file_analysis_agent/prompt.md @@ -24,6 +24,7 @@ - 对图片和多模态文件,重点报告客观可见信息,例如文字、UI、场景、人物、角色、应用/游戏名称和关键元素。 - `analyze_multimodal` 可能返回同文件历史分析记录;历史内容足够时直接基于它回答,只有确实需要新角度时才强制重新分析。 - 涉及临时下载或解压后,任务结束前清理临时目录。 +- 压缩包必须先通过 `download_file` 放入下载缓存;解压目标使用压缩包所在目录的子目录。清理时只传缓存的直接子目录名,不传完整路径。 回答要求: - 先给最有用的结论,再整理证据、摘录或结构化数据。 diff --git a/src/Undefined/skills/agents/file_analysis_agent/tools/cleanup_temp/config.json b/src/Undefined/skills/agents/file_analysis_agent/tools/cleanup_temp/config.json index 637730ef..bcae9e15 100644 --- a/src/Undefined/skills/agents/file_analysis_agent/tools/cleanup_temp/config.json +++ b/src/Undefined/skills/agents/file_analysis_agent/tools/cleanup_temp/config.json @@ -8,7 +8,7 @@ "properties": { "task_uuid": { "type": "string", - "description": "任务 UUID(缓存目录名),留空则清理所有下载缓存目录" + "description": "下载缓存的直接子目录名,不接受路径或符号链接;留空则清理所有非符号链接下载缓存子目录" } } } diff --git a/src/Undefined/skills/agents/file_analysis_agent/tools/cleanup_temp/handler.py b/src/Undefined/skills/agents/file_analysis_agent/tools/cleanup_temp/handler.py index c431119f..c662db60 100644 --- a/src/Undefined/skills/agents/file_analysis_agent/tools/cleanup_temp/handler.py +++ b/src/Undefined/skills/agents/file_analysis_agent/tools/cleanup_temp/handler.py @@ -1,6 +1,12 @@ from typing import Any, Dict import logging -import shutil +from pathlib import Path + +from Undefined.skills.shared import ( + delete_download_cache_directory, + download_cache_root, + resolve_download_cache_path, +) logger = logging.getLogger(__name__) @@ -9,29 +15,39 @@ async def execute(args: Dict[str, Any], context: Dict[str, Any]) -> str: """清理系统生成的临时文件和缓存""" task_uuid: str = args.get("task_uuid", "") - from Undefined.utils.paths import DOWNLOAD_CACHE_DIR - - cache_dir = DOWNLOAD_CACHE_DIR + cache_dir = download_cache_root(context) if not cache_dir.exists(): return "下载缓存目录不存在" try: if task_uuid: + if ( + task_uuid in {".", ".."} + or Path(task_uuid).name != task_uuid + or "\\" in task_uuid + ): + return "错误:task_uuid 必须是下载缓存的直接子目录名" target_dir = cache_dir / task_uuid + if target_dir.is_symlink(): + return "错误:不能清理符号链接目录" + target_dir = resolve_download_cache_path(target_dir, context) if target_dir.exists(): - shutil.rmtree(target_dir) + await delete_download_cache_directory(target_dir) return f"已清理下载缓存目录: {target_dir}" else: return f"下载缓存目录不存在: {target_dir}" else: count = 0 for item in cache_dir.iterdir(): - if item.is_dir(): - shutil.rmtree(item) + if item.is_dir() and not item.is_symlink(): + target_dir = resolve_download_cache_path(item, context) + await delete_download_cache_directory(target_dir) count += 1 return f"已清理 {count} 个下载缓存目录" + except ValueError as exc: + return f"错误:{exc}" except Exception as e: logger.exception(f"清理下载缓存目录失败: {e}") return "清理下载缓存目录失败,请稍后重试" diff --git a/src/Undefined/skills/agents/file_analysis_agent/tools/extract_archive/config.json b/src/Undefined/skills/agents/file_analysis_agent/tools/extract_archive/config.json index 5c24b92d..7032a6e9 100644 --- a/src/Undefined/skills/agents/file_analysis_agent/tools/extract_archive/config.json +++ b/src/Undefined/skills/agents/file_analysis_agent/tools/extract_archive/config.json @@ -8,7 +8,7 @@ "properties": { "file_path": { "type": "string", - "description": "本地压缩包路径" + "description": "本地压缩包路径;解压前须用 download_file 将其放入下载缓存" }, "action": { "type": "string", @@ -16,7 +16,7 @@ }, "extract_path": { "type": "string", - "description": "解压目标路径(可选),默认为临时目录" + "description": "解压目标路径(可选),必须是当前压缩包所在缓存目录的子目录;默认 extracted_<文件名>" } }, "required": ["file_path"] diff --git a/src/Undefined/skills/agents/file_analysis_agent/tools/extract_archive/handler.py b/src/Undefined/skills/agents/file_analysis_agent/tools/extract_archive/handler.py index 185eaee0..dff413bb 100644 --- a/src/Undefined/skills/agents/file_analysis_agent/tools/extract_archive/handler.py +++ b/src/Undefined/skills/agents/file_analysis_agent/tools/extract_archive/handler.py @@ -2,6 +2,8 @@ from typing import Any, Dict import logging +from Undefined.skills.shared import resolve_download_cache_path + logger = logging.getLogger(__name__) @@ -10,8 +12,23 @@ async def execute(args: Dict[str, Any], context: Dict[str, Any]) -> str: file_path: str = args.get("file_path", "") action: str = args.get("action", "list") extract_path: str | None = args.get("extract_path") - - path = Path(file_path) + if action not in {"list", "extract"}: + return "错误:action 只能是 list 或 extract" + try: + path = Path(file_path).resolve() + if action == "extract": + path = resolve_download_cache_path(path, context) + target = ( + Path(extract_path) + if extract_path + else path.parent / f"extracted_{path.stem}" + ) + target = resolve_download_cache_path(target, context) + if target == path.parent or not target.is_relative_to(path.parent): + raise ValueError("解压目标必须是当前压缩包所在目录的子目录") + extract_path = str(target) + except (OSError, ValueError) as exc: + return f"错误:{exc}" if not path.exists(): return f"错误:文件不存在 {file_path}" @@ -101,6 +118,8 @@ async def _extract_tar(path: Path, action: str, extract_path: str | None) -> str return "\n".join(info) else: + if not hasattr(tarfile, "data_filter"): + return "当前 Python 缺少安全 TAR 解压能力,请更新 Python 补丁版本;仍可使用 list 列出文件" if extract_path: target_dir = Path(extract_path) else: @@ -108,7 +127,7 @@ async def _extract_tar(path: Path, action: str, extract_path: str | None) -> str target_dir.mkdir(exist_ok=True) with tarfile.open(path, "r:*") as tar_ref: - tar_ref.extractall(target_dir) + tar_ref.extractall(target_dir, filter="data") info.append(f"\n已解压到:{target_dir}") info.append(f"解压文件数:{len(file_list)}") diff --git a/src/Undefined/skills/shared.py b/src/Undefined/skills/shared.py index 302dc96c..b4e07742 100644 --- a/src/Undefined/skills/shared.py +++ b/src/Undefined/skills/shared.py @@ -8,6 +8,7 @@ from __future__ import annotations from collections.abc import Coroutine +from pathlib import Path from typing import TYPE_CHECKING, Any, TypeVar if TYPE_CHECKING: @@ -25,6 +26,29 @@ async def run_cancellation_safe(operation: Coroutine[Any, Any, _ResultT]) -> _Re return await run_io_cancellation_safe(operation) +def download_cache_root(context: dict[str, Any]) -> Path: + """取得执行上下文的下载缓存根目录。""" + from Undefined.utils.paths import DOWNLOAD_CACHE_DIR + + return Path(context.get("download_cache_dir") or DOWNLOAD_CACHE_DIR).resolve() + + +def resolve_download_cache_path(path: Path, context: dict[str, Any]) -> Path: + """解析缓存内路径,拒绝缓存根本身和通过符号链接逃逸的路径。""" + root = download_cache_root(context) + resolved = path.resolve() + if resolved == root or not resolved.is_relative_to(root): + raise ValueError("路径必须位于下载缓存目录内,且不能是缓存根目录") + return resolved + + +async def delete_download_cache_directory(path: Path) -> None: + """通过公共异步 IO 清理已验证的下载缓存子目录。""" + from Undefined.utils.io import delete_tree + + await delete_tree(path) + + def private_access_error( runtime_config: Any, target_id: int, @@ -176,6 +200,8 @@ async def jm_send_book( __all__ = [ + "delete_download_cache_directory", + "download_cache_root", "jm_book_attachment_text", "jm_book_info_text", "jm_normalize_book_id", @@ -184,4 +210,5 @@ async def jm_send_book( "jm_send_book", "parse_positive_int", "private_access_error", + "resolve_download_cache_path", ] diff --git a/tests/test_file_analysis_archive_cleanup.py b/tests/test_file_analysis_archive_cleanup.py new file mode 100644 index 00000000..d9f1902d --- /dev/null +++ b/tests/test_file_analysis_archive_cleanup.py @@ -0,0 +1,218 @@ +from __future__ import annotations + +import io +from pathlib import Path +import tarfile +from typing import Any +from unittest.mock import Mock + +import pytest + +from Undefined.skills.agents.file_analysis_agent.tools.cleanup_temp import ( + handler as cleanup_handler, +) +from Undefined.skills.agents.file_analysis_agent.tools.extract_archive import ( + handler as archive_handler, +) + + +def _archive_context(tmp_path: Path) -> tuple[dict[str, Any], Path]: + root = tmp_path / "downloads" + task_dir = root / "task" + task_dir.mkdir(parents=True) + return {"download_cache_dir": root}, task_dir / "input.tar" + + +def _write_tar(path: Path, member: tarfile.TarInfo, content: bytes = b"") -> None: + with tarfile.open(path, "w") as archive: + if member.isfile(): + member.size = len(content) + archive.addfile(member, io.BytesIO(content)) + else: + archive.addfile(member) + + +@pytest.mark.asyncio +async def test_archive_extracts_regular_tar_inside_task_directory( + tmp_path: Path, +) -> None: + context, path = _archive_context(tmp_path) + _write_tar(path, tarfile.TarInfo("nested/readme.txt"), b"safe contents") + + result = await archive_handler.execute( + {"file_path": str(path), "action": "extract"}, context + ) + + assert "已解压到" in result + assert (path.parent / "extracted_input/nested/readme.txt").read_bytes() == ( + b"safe contents" + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("member_kind", ["traversal", "symlink", "hardlink", "device"]) +async def test_tar_rejects_unsafe_members( + tmp_path: Path, + member_kind: str, +) -> None: + context, path = _archive_context(tmp_path) + victim = tmp_path / "victim.txt" + victim.write_bytes(b"keep") + if member_kind == "traversal": + member = tarfile.TarInfo("../../../victim.txt") + else: + member = tarfile.TarInfo("unsafe") + member.type = { + "symlink": tarfile.SYMTYPE, + "hardlink": tarfile.LNKTYPE, + "device": tarfile.CHRTYPE, + }[member_kind] + member.linkname = "../../../victim.txt" + _write_tar(path, member, b"overwritten") + + result = await archive_handler.execute( + {"file_path": str(path), "action": "extract"}, context + ) + + assert "解析压缩包失败" in result + assert victim.read_bytes() == b"keep" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("target_kind", ["outside", "symlink", "other_task", "parent"]) +async def test_archive_rejects_extract_target_outside_current_task( + tmp_path: Path, + target_kind: str, +) -> None: + context, path = _archive_context(tmp_path) + _write_tar(path, tarfile.TarInfo("readme.txt"), b"data") + outside = tmp_path / "outside" + outside.mkdir() + if target_kind == "outside": + target = outside + elif target_kind == "symlink": + target = path.parent / "alias" + target.symlink_to(outside, target_is_directory=True) + elif target_kind == "other_task": + target = path.parent.parent / "other_task" + else: + target = path.parent + + result = await archive_handler.execute( + {"file_path": str(path), "action": "extract", "extract_path": str(target)}, + context, + ) + + assert result.startswith("错误:") + assert not (target / "readme.txt").exists() + assert not (outside / "readme.txt").exists() + + +@pytest.mark.asyncio +async def test_archive_rejects_source_outside_download_cache(tmp_path: Path) -> None: + context, _ = _archive_context(tmp_path) + source = tmp_path / "outside.tar" + _write_tar(source, tarfile.TarInfo("readme.txt"), b"data") + + result = await archive_handler.execute( + {"file_path": str(source), "action": "extract"}, context + ) + + assert "下载缓存目录内" in result + + +@pytest.mark.asyncio +async def test_archive_can_list_source_outside_download_cache(tmp_path: Path) -> None: + context, _ = _archive_context(tmp_path) + source = tmp_path / "outside.tar" + _write_tar(source, tarfile.TarInfo("readme.txt"), b"data") + + result = await archive_handler.execute({"file_path": str(source)}, context) + + assert "readme.txt" in result + assert not (tmp_path / "extracted_outside").exists() + + +@pytest.mark.asyncio +async def test_tar_without_safe_filter_refuses_extraction( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + context, path = _archive_context(tmp_path) + _write_tar(path, tarfile.TarInfo("readme.txt"), b"data") + monkeypatch.delattr(tarfile, "data_filter") + extract_mock = Mock() + monkeypatch.setattr(tarfile.TarFile, "extractall", extract_mock) + + result = await archive_handler.execute( + {"file_path": str(path), "action": "extract"}, context + ) + listed = await archive_handler.execute({"file_path": str(path)}, context) + + assert "缺少安全 TAR 解压能力" in result + assert "readme.txt" in listed + extract_mock.assert_not_called() + assert not (path.parent / "extracted_input").exists() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("task_uuid", ["../outside", "..", ".", "nested/task", "a\\b"]) +async def test_cleanup_rejects_path_arguments( + tmp_path: Path, + task_uuid: str, +) -> None: + context, _ = _archive_context(tmp_path) + outside = tmp_path / "outside" + outside.mkdir() + marker = outside / "keep.txt" + marker.write_text("keep") + + result = await cleanup_handler.execute({"task_uuid": task_uuid}, context) + + assert "错误:" in result + assert marker.read_text() == "keep" + assert Path(context["download_cache_dir"]).is_dir() + + +@pytest.mark.asyncio +async def test_cleanup_rejects_absolute_path_and_symbolic_link(tmp_path: Path) -> None: + context, _ = _archive_context(tmp_path) + outside = tmp_path / "outside" + outside.mkdir() + marker = outside / "keep.txt" + marker.write_text("keep") + alias = Path(context["download_cache_dir"]) / "alias" + alias.symlink_to(outside, target_is_directory=True) + + for task_uuid in (str(outside), "alias"): + result = await cleanup_handler.execute({"task_uuid": task_uuid}, context) + assert "错误:" in result + assert marker.read_text() == "keep" + assert alias.is_symlink() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("clear_all", [True, False]) +async def test_cleanup_removes_only_cache_directories( + tmp_path: Path, + clear_all: bool, +) -> None: + context, path = _archive_context(tmp_path) + path.write_bytes(b"data") + root = Path(context["download_cache_dir"]) + outside = tmp_path / "outside" + outside.mkdir() + marker = outside / "keep.txt" + marker.write_text("keep") + alias = root / "alias" + alias.symlink_to(outside, target_is_directory=True) + + result = await cleanup_handler.execute( + {} if clear_all else {"task_uuid": "task"}, context + ) + + assert "已清理" in result + assert not path.parent.exists() + assert root.is_dir() + assert alias.is_symlink() + assert marker.read_text() == "keep" diff --git a/tests/test_skills_import_boundary.py b/tests/test_skills_import_boundary.py index 7941024a..70a8188c 100644 --- a/tests/test_skills_import_boundary.py +++ b/tests/test_skills_import_boundary.py @@ -36,7 +36,6 @@ "agents/entertainment_agent/tools/minecraft_skin/handler.py::Undefined.utils.paths", "agents/entertainment_agent/tools/wenchang_dijun/handler.py::Undefined.attachments", "agents/file_analysis_agent/handler.py::Undefined.attachments", - "agents/file_analysis_agent/tools/cleanup_temp/handler.py::Undefined.utils.paths", "agents/file_analysis_agent/tools/describe_pdf_page/handler.py::Undefined.utils", "agents/file_analysis_agent/tools/describe_pdf_page/handler.py::Undefined.utils.paths", "agents/info_agent/tools/arxiv_search/handler.py::Undefined.arxiv.client", From af4b31ebbf879a9fa11d570406cb4db879a85e67 Mon Sep 17 00:00:00 2001 From: Null <1708213363@qq.com> Date: Tue, 6 Oct 2026 12:10:37 +0800 Subject: [PATCH 021/121] fix(ai): share task progress across tool call rounds --- src/Undefined/ai/client/ask_loop.py | 1 + .../skills/tools/task_progress/README.md | 2 + .../skills/tools/task_progress/handler.py | 5 +- tests/test_tool_search_ask_loop.py | 78 +++++++++++++++++++ 4 files changed, 83 insertions(+), 3 deletions(-) diff --git a/src/Undefined/ai/client/ask_loop.py b/src/Undefined/ai/client/ask_loop.py index 3ef5f40d..9ffb5213 100644 --- a/src/Undefined/ai/client/ask_loop.py +++ b/src/Undefined/ai/client/ask_loop.py @@ -316,6 +316,7 @@ async def emit_webchat_stage(stage: str, detail: Any | None = None) -> None: # 注入常用资源(用于工具执行) tool_context.setdefault("ai_client", self) tool_context.setdefault("runtime_config", self._get_runtime_config()) + tool_context.setdefault("_task_progress", []) if not isinstance( tool_context.get(MUSIC_TRACK_STORE_CONTEXT_KEY), MusicTrackReferenceStore, diff --git a/src/Undefined/skills/tools/task_progress/README.md b/src/Undefined/skills/tools/task_progress/README.md index 4b6c3ae1..4782de33 100644 --- a/src/Undefined/skills/tools/task_progress/README.md +++ b/src/Undefined/skills/tools/task_progress/README.md @@ -2,6 +2,8 @@ 任务进度追踪。处理需要多步骤、多 Agent 协作的复杂请求时,先用 plan 动作规划步骤,再在每步完成后用 update 动作标记进度。每次调用都会返回当前完整进度。 +同一次主 AI 请求的各轮工具调用共享计划;`plan` 替换计划内容,后续 `update` 能继续更新。不同请求默认各自保存计划,不跨请求持久化。 + | 参数 | 类型 | 必填 | 说明 | |------|------|------|------| | `action` | `string(plan/update)` | 是 | plan: 创建任务计划(会替换已有计划);update: 更新指定步骤的状态 | diff --git a/src/Undefined/skills/tools/task_progress/handler.py b/src/Undefined/skills/tools/task_progress/handler.py index 9327c70b..b75fcc75 100644 --- a/src/Undefined/skills/tools/task_progress/handler.py +++ b/src/Undefined/skills/tools/task_progress/handler.py @@ -61,7 +61,7 @@ async def execute(args: Dict[str, Any], context: Dict[str, Any]) -> str: if not isinstance(tasks_input, list) or len(tasks_input) == 0: return "tasks 必须是非空数组" - task_store: list[dict[str, Any]] = context.get("_task_progress", []) + task_store: list[dict[str, Any]] = context.setdefault("_task_progress", []) if action == "plan": new_tasks: list[dict[str, Any]] = [] @@ -79,8 +79,7 @@ async def execute(args: Dict[str, Any], context: Dict[str, Any]) -> str: new_tasks.append({"id": tid, "description": desc, "status": status}) new_tasks.sort(key=lambda t: t.get("id", 0)) - task_store = new_tasks - context["_task_progress"] = task_store + task_store[:] = new_tasks logger.info(f"[task_progress] 创建计划,共 {len(task_store)} 个步骤") return f"计划已创建\n{_format_task_list(task_store)}" diff --git a/tests/test_tool_search_ask_loop.py b/tests/test_tool_search_ask_loop.py index b06bfb0b..d3f23b65 100644 --- a/tests/test_tool_search_ask_loop.py +++ b/tests/test_tool_search_ask_loop.py @@ -16,6 +16,9 @@ MUSIC_TRACK_STORE_CONTEXT_KEY, MusicTrackReferenceStore, ) +from Undefined.skills.tools.task_progress.handler import ( + execute as execute_task_progress, +) from Undefined.utils.easter_egg_calls import ( format_batched_easter_egg_message, main_call_key, @@ -158,6 +161,81 @@ async def _submit_queued_llm_call(**kwargs: Any) -> dict[str, Any]: return client +@pytest.mark.asyncio +async def test_task_progress_state_survives_rounds_and_replanning() -> None: + stores: list[list[dict[str, Any]]] = [] + results: list[str] = [] + + async def execute_tool( + name: str, args: dict[str, Any], context: dict[str, Any] + ) -> str: + if name == "end": + context["conversation_ended"] = True + return "ended" + assert name == "task_progress" + stores.append(context["_task_progress"]) + result = await execute_task_progress(args, context) + results.append(result) + return result + + client = _build_client( + execute_tool=execute_tool, + tool_search_enabled=False, + schemas=[_tool("task_progress"), _tool("end")], + llm_responses=[ + _llm_tool_calls( + _tool_call( + "plan_first", + "task_progress", + '{"action":"plan","tasks":[{"id":1,"description":"first"}]}', + ) + ), + _llm_tool_calls( + _tool_call( + "update_first", + "task_progress", + '{"action":"update","tasks":[{"id":1,"status":"done"}]}', + ) + ), + _llm_tool_calls( + _tool_call( + "plan_again", + "task_progress", + '{"action":"plan","tasks":[{"id":2,"description":"second"}]}', + ) + ), + _llm_tool_calls( + _tool_call( + "update_second", + "task_progress", + '{"action":"update","tasks":[{"id":2,"status":"done"}]}', + ) + ), + _llm_tool_calls(_tool_call("end_first", "end")), + _llm_tool_calls( + _tool_call( + "update_new_request", + "task_progress", + '{"action":"update","tasks":[{"id":2,"status":"done"}]}', + ) + ), + _llm_tool_calls(_tool_call("end_second", "end")), + ], + ) + + assert await AIClient.ask(client, "plan and complete tasks") == "" + assert await AIClient.ask(client, "new request") == "" + + assert results[0].startswith("计划已创建") + assert "进度: 1/1" in results[1] + assert results[2].startswith("计划已创建") + assert "进度: 1/1" in results[3] + assert all(store is stores[0] for store in stores[:4]) + assert stores[0] == [{"id": 2, "description": "second", "status": "done"}] + assert stores[4] is not stores[0] + assert results[4] == "还没有任务计划,请先用 plan 动作创建" + + @pytest.mark.asyncio async def test_music_track_reference_store_survives_tool_search_rounds() -> None: stores: list[object] = [] From c0006d74ca5390152771b0e50d642d8212392b5e Mon Sep 17 00:00:00 2001 From: Null <1708213363@qq.com> Date: Tue, 6 Oct 2026 12:11:31 +0800 Subject: [PATCH 022/121] fix(webui): wait for authoritative history before releasing chat jobs --- docs/management-api.md | 2 +- docs/webui-guide.md | 4 +- src/Undefined/webui/static/js/runtime.js | 96 ++++++++++++----- tests/frontend/runtime_chat_harness.js | 119 ++++++++++++++++++++++ tests/test_webui_runtime_chat_behavior.py | 27 +++++ 5 files changed, 217 insertions(+), 31 deletions(-) diff --git a/docs/management-api.md b/docs/management-api.md index a5902ad1..b89375ac 100644 --- a/docs/management-api.md +++ b/docs/management-api.md @@ -355,7 +355,7 @@ WebUI 内嵌聊天渲染 `` / `` 图片 - 校验: - `conversation_id` 不一致时返回 `404`,避免跨会话误续接。 - 响应: - - 默认 JSON:持久事件、当前顶层 `stage` 快照、当前 `agent_stage` 快照、`current_tool_calls` 和耗时字段。 + - 默认 JSON:持久事件、当前顶层 `stage` 快照、当前 `agent_stage` 快照、`current_tool_calls`、耗时字段和 `job.history_finalized`。`done` / `error` / `cancelled` 终态可能先于历史收尾发布;WebUI 保留发送锁并继续轮询,直到 `history_finalized=true` 后刷新权威历史,刷新成功才释放任务;刷新失败时保留发送锁并通过现有轮询重试。 - SSE:事件帧和 keep-alive 由 Runtime 透传。 ```json diff --git a/docs/webui-guide.md b/docs/webui-guide.md index 32bfc73d..e5c02379 100644 --- a/docs/webui-guide.md +++ b/docs/webui-guide.md @@ -160,8 +160,8 @@ WebUI 内置的对话界面,直接与 Bot 的 AI 进行交互: - WebUI 场景下,代码优先直接作为聊天回复输出,不要默认转成文件附件;只有用户明确要求文件交付、代码过长不适合聊天展示,或确需附件工作流时才使用文件。所有代码块都应显式标注语言,例如 ` ```python `、` ```javascript `、` ```html `、` ```bash `,不确定语言时用 ` ```text `。 - AI 消息支持引用:可引用整条 AI 消息、选中的某段文字,或 HTML 预览中点选的元素。引用会显示在输入框右侧的待发送引用条中,可在发送前移除;实际发送时不会新增接口或附件,而是自动在用户消息前拼接为 Markdown `>` 引用块,例如 `> 引用 AI:`,随后和文本、图片、文件一起进入同一条 WebChat job 消息。发送后的引用块会像代码块一样折叠显示,展开后内部固定高度并可滚动。发送失败时待引用内容会保留,发送成功后清空。 - 默认加载最新 50 条消息并滚动到底部;向上滚动到顶部会按后端返回的 `next_before` 游标懒加载更早历史,并保持原视口偏移,避免一次性恢复大量工具块造成卡顿。“自动滚动到底部”开关默认开启并保存在浏览器本地,关闭后 AI 回复、工具 / Agent 状态和 AI 阶段刷新不会打断当前位置;首次加载和主动发送新消息仍会定位到底部。系统开启减少动态效果时,聊天滚动会改为即时跳转。 -- 对话由 WebChat job 执行。刷新页面、关闭页面、网络短暂中断或换另一个客户端重新访问 WebUI 时,后端任务继续运行,前端会从后端查询会话列表、当前运行 job,并用 `conversation_id + job_id + seq` 每 0.5 秒轮询增量事件、当前阶段快照和运行中工具 / Agent 快照自动续接;前端不会把关键任务状态只存放在浏览器本地,历史、会话、运行中 job 和事件游标都以 Runtime API 返回值为准。如果刷新后首次查询运行中 job 失败,WebUI 会退避重试并在网络恢复时再次尝试;如果后端已完成 job 并落盘,前端会刷新历史并解除发送锁。SSE 仅作为兼容方式保留,WebUI 不依赖长连接。 -- 运行中的 WebChat job 可从正在生成的 AI 气泡中取消。取消会调用 Runtime 的 job cancel 接口并解除当前会话的发送锁;如果取消发生在 AI 尚未产生任何回复内容前,前端会移除空 AI 气泡,并在最后一条用户消息上显示“重试”。点击重试会带 `reuse_previous_user_message=true` 用同一条纯文本重新发起 WebChat job,前端不再追加一条相同用户气泡,后端也会校验并复用最后一条可见用户历史,避免把同一条输入重复写入会话。 +- 对话由 WebChat job 执行。刷新页面、关闭页面、网络短暂中断或换另一个客户端重新访问 WebUI 时,后端任务继续运行,前端会从后端查询会话列表、当前运行 job,并用 `conversation_id + job_id + seq` 每 0.5 秒轮询增量事件、当前阶段快照和运行中工具 / Agent 快照自动续接;前端不会把关键任务状态只存放在浏览器本地,历史、会话、运行中 job 和事件游标都以 Runtime API 返回值为准。如果刷新后首次查询运行中 job 失败,WebUI 会退避重试并在网络恢复时再次尝试;`done` / `error` / `cancelled` 终态先到时仍保留 job 和发送锁,并继续轮询等待 `history_finalized=true`;确认后端收尾完成后刷新权威历史,刷新成功才解除发送锁;历史慢响应或刷新失败期间继续保留锁,并通过原轮询重试恢复。SSE 仅作为兼容方式保留,WebUI 不依赖长连接。 +- 运行中的 WebChat job 可从正在生成的 AI 气泡中取消。取消会调用 Runtime 的 job cancel 接口,等待历史收尾完成后解除当前会话的发送锁;如果取消发生在 AI 尚未产生任何回复内容前,前端会移除空 AI 气泡,并在最后一条用户消息上显示“重试”。点击重试会带 `reuse_previous_user_message=true` 用同一条纯文本重新发起 WebChat job,前端不再追加一条相同用户气泡,后端也会校验并复用最后一条可见用户历史,避免把同一条输入重复写入会话。 - 运行中的 AI 气泡会在 `AI` 标签后实时显示当前阶段和后端计算的总已用时,例如构建上下文、查长期记忆、查认知记忆、等待模型、等待工具、发送消息;任务完成后该位置显示本轮回复总用时。工具 / Agent 摘要行会在名称旁显示调用耗时,运行中先显示后端快照时间,轮询间隙用本地时间临时递增,下一次查询后自动校准;结束后固定显示后端结束事件的总耗时。轮询刷新只更新已有状态和计时节点,结构未变化时不会重建工具 / Agent 块,避免运行中闪烁。状态条区分运行中、成功和失败;整轮回复总耗时会写入 WebChat 历史 metadata。 - WebChat job 事件会更新同一个 AI 气泡;AI 正文和主对话工具 / Agent 调用按事件时序显示,例如工具调用、正文回复、结束工具会依次出现在同一气泡内。 - 工具 / Agent 调用块展开后会分区展示由后端脱敏截断后的输入和输出预览;Agent 内部工具、子 Agent 和发送出的正文会按后端提供的 timeline 嵌套显示,并保留各自状态、输出和耗时。Agent 内部阶段只作为对应 Agent 摘要行的当前状态展示,不单独占用一行。并发工具结束事件按实际完成时间发布,结构化预览会渲染为带颜色的键值字段。输出内容继续支持 Markdown、安全 HTML、图片和文件卡片渲染,并保留适合移动端的单行工具摘要。 diff --git a/src/Undefined/webui/static/js/runtime.js b/src/Undefined/webui/static/js/runtime.js index cc490150..a6102a27 100644 --- a/src/Undefined/webui/static/js/runtime.js +++ b/src/Undefined/webui/static/js/runtime.js @@ -28,6 +28,7 @@ chatHistoryCursor: null, chatHistoryHasMore: false, chatHistoryLoading: false, + chatHistoryFinalization: null, chatTopLoadSuppressedUntil: 0, chatAutoScroll: true, // 视图是否仍「钉」在底部。用户主动往上翻时置 false:此时不能再自动滚到底 @@ -5119,7 +5120,7 @@ async function loadChatHistory( force = false, - { resumeActiveJob = true } = {}, + { resumeActiveJob = true, expectedJobId = "" } = {}, ) { if (!currentChatConversationId()) { await loadChatConversations(); @@ -5132,6 +5133,7 @@ return; } runtimeState.chatHistoryLoading = true; + const conversationId = currentChatConversationId(); const res = await api( chatUrl("/api/runtime/chat/history", { limit: 50 }), ); @@ -5140,6 +5142,13 @@ runtimeState.chatHistoryLoading = false; throw new Error(buildRequestError(res, data)); } + if ( + conversationId !== currentChatConversationId() || + (expectedJobId && runtimeState.activeJobId !== expectedJobId) + ) { + runtimeState.chatHistoryLoading = false; + return; + } clearChatMessages(); const items = data && Array.isArray(data.items) ? data.items : []; @@ -5208,6 +5217,48 @@ } } + async function finishChatJob(payload = {}) { + // 终态事件先于历史落盘发布:保留 job 和轮询,直到 Runtime 确认收尾完成。 + if (payload.history_finalized === false) { + runtimeState.chatBusy = true; + syncChatBusyControls(); + return; + } + const jobId = runtimeState.activeJobId; + if (payload.history_finalized === true) { + let finalization = runtimeState.chatHistoryFinalization; + if (!finalization || finalization.jobId !== jobId) { + finalization = { + jobId, + promise: loadChatHistory(true, { + resumeActiveJob: false, + expectedJobId: jobId, + }), + }; + runtimeState.chatHistoryFinalization = finalization; + } + try { + await finalization.promise; + } catch (_error) { + // 历史刷新失败仍保留发送锁,现有 job 轮询会再次尝试收尾。 + return; + } finally { + if (runtimeState.chatHistoryFinalization === finalization) { + runtimeState.chatHistoryFinalization = null; + } + } + if (runtimeState.activeJobId !== jobId) return; + } + stopChatPolling(); + runtimeState.activeJobId = null; + runtimeState.activeJobConversationId = ""; + runtimeState.chatBusy = false; + setButtonLoading(get("btnRuntimeChatSend"), false); + syncChatBusyControls(); + syncChatMessageActions(); + loadChatConversations({ selectFirst: false }).catch(() => {}); + } + function applyChatEvent(event, payload, seq = 0) { if (seq) runtimeState.lastEventSeq = Math.max( @@ -5241,14 +5292,7 @@ (event === "done" || event === "error") && (!eventJobId || eventJobId === runtimeState.activeJobId) ) { - stopChatPolling(); - runtimeState.activeJobId = null; - runtimeState.activeJobConversationId = ""; - runtimeState.chatBusy = false; - setButtonLoading(get("btnRuntimeChatSend"), false); - syncChatBusyControls(); - syncChatMessageActions(); - loadChatConversations({ selectFirst: false }).catch(() => {}); + finishChatJob(payload || {}); } return; } @@ -5296,7 +5340,6 @@ return; } if (event === "done") { - stopChatPolling(); if ( payload && payload.reply && @@ -5317,27 +5360,15 @@ } } finalizeActiveChatMessage(payload || {}); - runtimeState.activeJobId = null; - runtimeState.activeJobConversationId = ""; - runtimeState.chatBusy = false; runtimeState.chatHistoryLoaded = true; - setButtonLoading(get("btnRuntimeChatSend"), false); - syncChatBusyControls(); - syncChatMessageActions(); - loadChatConversations({ selectFirst: false }).catch(() => {}); + finishChatJob(payload || {}); return; } if (event === "error") { - stopChatPolling(); const item = findActiveChatMessage(eventJobId); finalizeActiveChatMessage(); removeEmptyChatMessage(item); - runtimeState.activeJobId = null; - runtimeState.activeJobConversationId = ""; - runtimeState.chatBusy = false; - setButtonLoading(get("btnRuntimeChatSend"), false); - syncChatBusyControls(); - syncChatMessageActions(); + finishChatJob(payload || {}); const message = String( payload && (payload.error || payload.message) ? payload.error || payload.message @@ -5357,22 +5388,28 @@ function applyChatEventsPayload(data, jobId) { const events = data && Array.isArray(data.events) ? data.events : []; + const job = data && data.job ? data.job : null; events .filter((entry) => entry && typeof entry === "object") .sort((a, b) => Number(a.seq || 0) - Number(b.seq || 0)) .forEach((entry) => { applyChatEvent( String(entry.event || ""), - entry.payload || {}, + ["done", "error"].includes(String(entry.event || "")) && job + ? { + ...(entry.payload || {}), + history_finalized: job.history_finalized, + } + : entry.payload || {}, Number(entry.seq || 0), ); }); - const job = data && data.job ? data.job : null; applyChatJobSnapshot(job, jobId); if ( job && runtimeState.activeJobId === jobId && - ["done", "error", "cancelled"].includes(String(job.status || "")) + ["done", "error", "cancelled"].includes(String(job.status || "")) && + job.history_finalized !== false ) { applyChatEvent( job.status === "done" ? "done" : "error", @@ -5383,6 +5420,7 @@ job_id: job.job_id || jobId, conversation_id: job.conversation_id || "", duration_ms: job.duration_ms, + history_finalized: job.history_finalized, }, Number(job.last_seq || runtimeState.lastEventSeq), ); @@ -5391,6 +5429,8 @@ function applyChatJobSnapshot(job, jobId) { if (!job || runtimeState.activeJobId !== jobId) return; + const status = String(job.status || ""); + if (["done", "error", "cancelled"].includes(status)) return; const jobConversationId = String(job.conversation_id || "").trim(); if ( jobConversationId && @@ -5400,7 +5440,6 @@ } const item = ensureStreamingMessage(jobId); if (!item) return; - const status = String(job.status || ""); if (!["done", "error", "cancelled"].includes(status)) { const stage = String(job.current_stage || "").trim(); if (stage) { @@ -5616,6 +5655,7 @@ runtimeState.activeJobConversationId || currentChatConversationId(), duration_ms: data && data.duration_ms, + history_finalized: data && data.history_finalized, }, Number(data && data.last_seq ? data.last_seq : 0), ); diff --git a/tests/frontend/runtime_chat_harness.js b/tests/frontend/runtime_chat_harness.js index 04685d80..4d930e27 100644 --- a/tests/frontend/runtime_chat_harness.js +++ b/tests/frontend/runtime_chat_harness.js @@ -2756,6 +2756,125 @@ SCENARIOS.workflow_tool_args_json_round_trip = async (env) => { }; }; +async function jobWaitsForHistoryFinalization(env, status, controlledHistory = false) { + const { window, setRoutes } = env; + let jobPosts = 0; + let historyFinalized = false; + let historyReads = 0; + let firstEventBatch = true; + let historyAttempts = 0; + let resolveHistory = null; + let rejectHistory = null; + const snapshot = () => ({ + job_id: "job-finalizing", + conversation_id: "conv-1", + status, + last_seq: 2, + history_finalized: historyFinalized, + reply: "answer", + error: status === "done" ? "" : status, + }); + setRoutes([ + { + match: "/chat/conversations", + reply: () => ({ body: { + conversations: [{ id: "conv-1", title: "t" }], + active_job: jobPosts === 1 && !historyFinalized ? snapshot() : null, + } }), + }, + { + match: "/chat/history", + reply: () => { + historyReads += 1; + return { body: { + items: historyFinalized ? [{ role: "bot", content: "persisted answer" }] : [], + has_more: false, + next_before: null, + } }; + }, + }, + { match: "/chat/jobs/active", reply: { body: { job: null } } }, + { + match: "/jobs/job-finalizing/events", + reply: () => { + const events = firstEventBatch ? [ + { seq: 1, event: "message", payload: { job_id: "job-finalizing", conversation_id: "conv-1", content: "answer" } }, + { seq: 2, event: status === "done" ? "done" : "error", payload: { job_id: "job-finalizing", conversation_id: "conv-1", reply: "answer", error: status } }, + ] : []; + firstEventBatch = false; + return { body: { events, job: snapshot() } }; + }, + }, + { + match: "/jobs/job-next/events", + reply: { body: { events: [], job: { job_id: "job-next", conversation_id: "conv-1", status: "running" } } }, + }, + { + match: "/chat/jobs", + reply: () => { + jobPosts += 1; + return { body: { job_id: jobPosts === 1 ? "job-finalizing" : "job-next", conversation_id: "conv-1" } }; + }, + }, + ]); + if (controlledHistory) { + const fetch = window.fetch; + window.fetch = (url, options) => { + if (historyFinalized && String(url).includes("/chat/history")) { + historyAttempts += 1; + return new Promise((resolve, reject) => { + resolveHistory = () => fetch(url, options).then(resolve, reject); + rejectHistory = reject; + }); + } + return fetch(url, options); + }; + } + window.RuntimeController.init(); + await window.RuntimeController.loadChatHistory(true); + const input = window.document.getElementById("runtimeChatInput"); + const button = window.document.getElementById("btnRuntimeChatSend"); + input.value = "first"; + button.click(); + await tick(); + input.value = "second"; + button.click(); + await tick(); + const beforeFinalization = { disabled: button.disabled, jobPosts, historyReads }; + historyFinalized = true; + await settle(650); + let slowHistory = null; + if (controlledHistory) { + button.click(); + await tick(); + const whilePending = { disabled: button.disabled, jobPosts, historyAttempts }; + rejectHistory(new Error("temporary history refresh failure")); + await tick(); + await settle(650); + button.click(); + await tick(); + const afterFailure = { disabled: button.disabled, jobPosts, historyAttempts }; + resolveHistory(); + await tick(); + slowHistory = { whilePending, afterFailure }; + } + const afterFinalization = { disabled: button.disabled, historyReads, botTexts: botNodes(window).map((node) => node.contentText) }; + button.click(); + await tick(); + return { + beforeFinalization, + afterFinalization, + jobPosts, + eventRequests: env.requests.filter((url) => url.includes("/jobs/job-finalizing/events")).length, + slowHistory, + }; +} +SCENARIOS.history_finalization_slow_history = (env) => jobWaitsForHistoryFinalization(env, "done", true); + +for (const status of ["done", "error", "cancelled"]) { + SCENARIOS[`history_finalization_${status}`] = (env) => jobWaitsForHistoryFinalization(env, status); +} + async function main() { const args = parseArgs(process.argv.slice(2)); if (!args.scenario) throw new Error("缺少 --scenario"); diff --git a/tests/test_webui_runtime_chat_behavior.py b/tests/test_webui_runtime_chat_behavior.py index 8d9b67fa..c1777bad 100644 --- a/tests/test_webui_runtime_chat_behavior.py +++ b/tests/test_webui_runtime_chat_behavior.py @@ -117,6 +117,33 @@ def test_final_message_reuses_streaming_node() -> None: assert [node["contentTexts"] for node in _user_nodes(result)] == [["hi"]] +@pytest.mark.parametrize("status", ["done", "error", "cancelled"]) +def test_terminal_job_keeps_polling_until_history_is_finalized(status: str) -> None: + result = run_scenario(f"history_finalization_{status}") + assert result["beforeFinalization"] == { + "disabled": True, + "jobPosts": 1, + "historyReads": 1, + } + assert result["afterFinalization"]["disabled"] is False + assert result["afterFinalization"]["historyReads"] == 2 + assert result["afterFinalization"]["botTexts"] == ["persisted answer"] + assert result["jobPosts"] == 2 + assert result["eventRequests"] >= 2 + + +def test_history_refresh_keeps_send_locked_until_success_and_retries_failure() -> None: + result = run_scenario("history_finalization_slow_history") + assert result["slowHistory"] == { + "whilePending": {"disabled": True, "jobPosts": 1, "historyAttempts": 1}, + "afterFailure": {"disabled": True, "jobPosts": 1, "historyAttempts": 2}, + } + assert result["afterFinalization"]["disabled"] is False + assert result["afterFinalization"]["historyReads"] == 2 + assert result["afterFinalization"]["botTexts"] == ["persisted answer"] + assert result["jobPosts"] == 2 + + # --------------------------------------------------------------------------- # # 工具生命周期 # --------------------------------------------------------------------------- # From 21dfb0baf164df2418e97639a14979a5e9909dac Mon Sep 17 00:00:00 2001 From: Null <1708213363@qq.com> Date: Tue, 6 Oct 2026 12:12:27 +0800 Subject: [PATCH 023/121] fix(knowledge): preserve source chunks until replacement succeeds --- docs/knowledge.md | 2 + src/Undefined/knowledge/manager.py | 10 +++- src/Undefined/knowledge/store.py | 24 +++++++- tests/test_knowledge.py | 89 +++++++++++++++++++++++++++++- 4 files changed, 118 insertions(+), 7 deletions(-) diff --git a/docs/knowledge.md b/docs/knowledge.md index e212986d..b27be0fc 100644 --- a/docs/knowledge.md +++ b/docs/knowledge.md @@ -188,6 +188,8 @@ chunk_size=4, chunk_overlap=1 → step=3 `.manifest.json` 记录每个文件的 SHA256 hash。扫描时只对新增或内容变更的文件重新嵌入,未变更的文件跳过。 +变更文件先完成读取与嵌入,再写入新块,成功后才删除该来源的旧块并更新 hash。读取、嵌入或新块写入失败时保留旧索引和旧 hash,下次扫描可重试;主动删除文件或把内容清空则会移除其旧索引。索引写入与清理串行执行,取消时先等待已发起的磁盘操作完成。 + ### 站台/发车队列 嵌入与重排请求都通过内置队列串行发送,按各自 `queue_interval_seconds` 控制发车间隔;embedding/rerank 默认值均为 `0.0`,非正数都表示请求到达立即发车。多行文本按 `embed_batch_size` 分批,每批一次 API 调用。 diff --git a/src/Undefined/knowledge/manager.py b/src/Undefined/knowledge/manager.py index a7a73760..e81f0f8b 100644 --- a/src/Undefined/knowledge/manager.py +++ b/src/Undefined/knowledge/manager.py @@ -287,15 +287,18 @@ async def embed_knowledge_base(self, kb_name: str) -> int: continue if store is None: store = self._get_store(kb_name) - await store.delete_by_source(relative_source) try: content = await asyncio.to_thread(text_file.read_text, "utf-8") except (OSError, UnicodeDecodeError): - if manifest.pop(relative_source, None) is not None: - manifest_changed = True + logger.warning( + "[知识库] 读取失败,保留旧索引: kb=%s source=%s", + kb_name, + relative_source, + ) continue lines = chunk_lines(content, self._chunk_size, self._chunk_overlap) if not lines: + await store.delete_by_source(relative_source) manifest[relative_source] = fhash manifest_changed = True continue @@ -306,6 +309,7 @@ async def embed_knowledge_base(self, kb_name: str) -> int: lines, embeddings, metadata_base={"source": relative_source, "kb": kb_name}, + replace_source=True, ) manifest[relative_source] = fhash manifest_changed = True diff --git a/src/Undefined/knowledge/store.py b/src/Undefined/knowledge/store.py index 766a37c8..f433ce61 100644 --- a/src/Undefined/knowledge/store.py +++ b/src/Undefined/knowledge/store.py @@ -10,6 +10,8 @@ import chromadb +from Undefined.utils import io + logger = logging.getLogger(__name__) @@ -20,6 +22,7 @@ def __init__( self, name: str, chroma_dir: Path, *, create_if_missing: bool = True ) -> None: self.name = name + self._write_lock = asyncio.Lock() self._client = chromadb.PersistentClient(path=str(chroma_dir)) if create_if_missing: self._collection = self._client.get_or_create_collection( @@ -42,6 +45,8 @@ async def add_chunks( chunks: list[str], embeddings: list[list[float]], metadata_base: dict[str, Any] | None = None, + *, + replace_source: bool = False, ) -> int: if not chunks: return 0 @@ -52,14 +57,26 @@ async def add_chunks( ] def _upsert() -> None: + previous_ids = ( + self._collection.get(where={"source": source}, include=[])["ids"] + if replace_source and source + else [] + ) self._collection.upsert( ids=ids, embeddings=embeddings, # type: ignore[arg-type] documents=chunks, metadatas=metadatas, # type: ignore[arg-type] ) - - await asyncio.to_thread(_upsert) + current_ids = set(ids) + stale_ids = [ + chunk_id for chunk_id in previous_ids if chunk_id not in current_ids + ] + if stale_ids: + self._collection.delete(ids=stale_ids) + + async with self._write_lock: + await io.run_cancellation_safe(asyncio.to_thread(_upsert)) return len(chunks) async def delete_by_source(self, source: str) -> None: @@ -69,7 +86,8 @@ async def delete_by_source(self, source: str) -> None: def _delete() -> None: self._collection.delete(where={"source": source}) - await asyncio.to_thread(_delete) + async with self._write_lock: + await io.run_cancellation_safe(asyncio.to_thread(_delete)) async def query( self, query_embedding: list[float], top_k: int = 5 diff --git a/tests/test_knowledge.py b/tests/test_knowledge.py index ba03ec54..137a6b73 100644 --- a/tests/test_knowledge.py +++ b/tests/test_knowledge.py @@ -8,6 +8,8 @@ from typing import Any from unittest.mock import AsyncMock, MagicMock, patch +import pytest + from chromadb.errors import NotFoundError from Undefined.knowledge.chunker import chunk_lines from Undefined.knowledge.embedder import Embedder @@ -444,10 +446,95 @@ async def test_embed_replaces_changed_file_chunks(tmp_path: Path) -> None: added = await manager.embed_knowledge_base("kb1") assert added == 1 - store.delete_by_source.assert_awaited_once_with("texts/doc.txt") + store.delete_by_source.assert_not_awaited() + assert store.add_chunks.await_args.kwargs["replace_source"] is True assert "texts/doc.txt" in json.loads(manifest_path.read_text("utf-8")) +@pytest.mark.parametrize("failure", ["read", "embed", "upsert"]) +async def test_failed_reindex_keeps_real_old_chunks_and_retries( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, failure: str +) -> None: + manager, embedder = _make_manager(tmp_path) + kb_dir = _make_kb(tmp_path, "kb1", {"doc.txt": "new contents"}) + source_path = kb_dir / "texts/doc.txt" + manifest_path = kb_dir / ".manifest.json" + manifest_path.write_text('{"texts/doc.txt":"outdated"}', "utf-8") + store = manager._get_store("kb1") + await store.add_chunks( + ["old contents"], [[0.1, 0.2]], {"source": "texts/doc.txt", "kb": "kb1"} + ) + original_read = Path.read_text + + def unreadable(path: Path, *args: Any, **kwargs: Any) -> str: + if path == source_path: + raise UnicodeDecodeError("utf-8", b"\xff", 0, 1, "invalid") + return original_read(path, *args, **kwargs) + + def failed_upsert(**kwargs: Any) -> None: + raise RuntimeError("upsert failed") + + with monkeypatch.context() as failure_patch: + if failure == "read": + failure_patch.setattr(Path, "read_text", unreadable) + elif failure == "embed": + failure_patch.setattr( + embedder, + "embed", + AsyncMock(side_effect=TimeoutError("embedding failed")), + ) + else: + failure_patch.setattr(store._collection, "upsert", failed_upsert) + if failure == "read": + assert await manager.embed_knowledge_base("kb1") == 0 + else: + with pytest.raises((TimeoutError, RuntimeError), match="failed"): + await manager.embed_knowledge_base("kb1") + assert store._collection.get(where={"source": "texts/doc.txt"})[ + "documents" + ] == ["old contents"] + assert json.loads(manifest_path.read_text("utf-8")) == { + "texts/doc.txt": "outdated" + } + + assert await manager.embed_knowledge_base("kb1") == 1 + assert store._collection.get(where={"source": "texts/doc.txt"})["documents"] == [ + "new contents" + ] + assert json.loads(manifest_path.read_text("utf-8"))["texts/doc.txt"] != "outdated" + + +async def test_source_replacement_removes_stale_chunks_only_after_success( + tmp_path: Path, +) -> None: + store = KnowledgeStore("knowledge_test", tmp_path / "chroma") + await store.add_chunks( + ["keep", "remove"], [[0.1, 0.2], [0.2, 0.3]], {"source": "a.txt"} + ) + await store.add_chunks(["remove"], [[0.2, 0.3]], {"source": "b.txt"}) + + assert ( + await store.add_chunks( + ["keep"], [[0.3, 0.4]], {"source": "a.txt"}, replace_source=True + ) + == 1 + ) + assert store._collection.get(where={"source": "a.txt"})["documents"] == ["keep"] + assert store._collection.get(where={"source": "b.txt"})["documents"] == ["remove"] + + +async def test_empty_file_update_removes_real_previous_chunks(tmp_path: Path) -> None: + manager, embedder = _make_manager(tmp_path) + kb_dir = _make_kb(tmp_path, "kb1", {"doc.txt": " \n"}) + store = manager._get_store("kb1") + await store.add_chunks(["old"], [[0.1, 0.2]], {"source": "texts/doc.txt"}) + + assert await manager.embed_knowledge_base("kb1") == 0 + assert not store._collection.get(where={"source": "texts/doc.txt"})["ids"] + assert "texts/doc.txt" in json.loads((kb_dir / ".manifest.json").read_text("utf-8")) + embedder.embed.assert_not_awaited() + + async def test_semantic_search_missing_kb_does_not_create_store(tmp_path: Path) -> None: manager, embedder = _make_manager(tmp_path) results = await manager.semantic_search("missing", "hello") From 913e075cf600dd95d89727f26468338e8d56b2ab Mon Sep 17 00:00:00 2001 From: Null <1708213363@qq.com> Date: Tue, 6 Oct 2026 12:13:22 +0800 Subject: [PATCH 024/121] fix(webui): allow remote chat images under image CSP --- docs/management-api.md | 2 +- docs/webui-guide.md | 2 +- src/Undefined/webui/app.py | 2 +- tests/test_webui_csp_images.py | 80 ++++++++++++++++++++++++++++++ tests/test_webui_management_api.py | 5 +- 5 files changed, 87 insertions(+), 4 deletions(-) create mode 100644 tests/test_webui_csp_images.py diff --git a/docs/management-api.md b/docs/management-api.md index b89375ac..4468566a 100644 --- a/docs/management-api.md +++ b/docs/management-api.md @@ -341,7 +341,7 @@ Content-Type: multipart/form-data `runtime/chat/files` 仍保留为旧 WebUI 浏览器文件缓存兼容路径;新客户端和新 WebChat 功能应使用 `runtime/chat/attachments`。其 `runtime/chat/file?id=...` 下载响应支持 gzip 协商,可压缩的文本 / HTML 文件会保留 MIME、UTF-8 charset 和附件下载头。 -WebUI 内嵌聊天渲染 `` / `` 图片时会把 Runtime 返回的 `/api/v1/chat/attachments/{uid}/preview` 规整为 Management 代理路径 `/api/runtime/chat/attachments/{uid}/preview`,浏览器不直接请求 Runtime API。普通 Markdown 图片(如 `![Image #1](https://...)`)仍由前端安全 renderer 输出可点击预览的 ``,不走附件代理。 +WebUI 内嵌聊天渲染 `` / `` 图片时会把 Runtime 返回的 `/api/v1/chat/attachments/{uid}/preview` 规整为 Management 代理路径 `/api/runtime/chat/attachments/{uid}/preview`,浏览器不直接请求 Runtime API。普通 Markdown 图片(如 `![Image #1](https://...)`)仍由前端安全 renderer 输出可点击预览的 ``,浏览器直接加载 HTTP/HTTPS 图片,不走附件代理。WebUI CSP 的 `img-src` 允许 HTTP/HTTPS 来源;脚本与 API 请求继续受原有 `script-src` / `connect-src` 限制。 ### `runtime/chat/jobs/{job_id}/events` diff --git a/docs/webui-guide.md b/docs/webui-guide.md index e5c02379..d9a54fb4 100644 --- a/docs/webui-guide.md +++ b/docs/webui-guide.md @@ -156,7 +156,7 @@ WebUI 内置的对话界面,直接与 Bot 的 AI 进行交互: - 旧版 WebChat 历史会在首次打开时自动迁移到默认会话。迁移完成后会写入标记文件,之后不会重复迁移;即使删除该默认会话,也不会再从旧历史文件恢复。未选择会话的旧接口调用会按需创建一个空的默认会话以保持向后兼容。 - 会话标题先使用第一条用户消息的前若干字符作为临时标题;当会话已有首问和首答后,后端会使用 chat model 根据首问 + 首答生成正式标题。手动重命名的标题不会被自动生成覆盖;临时或生成失败的标题会在后续能处理时继续尝试。 - 支持文本、图片和文件消息。图片或文件可通过 `+` 选择,也可直接粘贴到输入框;粘贴只会加入待发送附件条,不会立即发送,点击发送或按 Enter 时才随同当前文本进入同一条 WebChat 消息。无待发送附件时输入框会占满可用宽度;添加附件后右侧预览轨道随数量平滑展开,图片显示缩略图,附件较多时输入框保持最小可用宽度并压缩预览卡片,避免输入区跳动。移动端会把引用和附件预览轨道放到输入框上方,保证正文输入和发送按钮不被挤压。 -- AI 回复支持 Markdown 渲染,Markdown 内的常见 HTML 片段会经过 WebUI 白名单净化后自动渲染;完整 HTML 文档或独立块级 HTML 片段会先净化再直渲染,避免被 Markdown 缩进规则误判成代码块。脚本、事件属性、危险协议、危险样式以及 `head` / `style` 等文档元信息会被剥离。聊天中的图片可点击放大查看,支持点击遮罩、关闭按钮或按 `Esc` 退出。代码块使用本地随包的 highlight.js 做多语言语法高亮,不依赖外部 CDN;显式语言优先,未知语言会自动检测,库不可用时回退为安全转义文本。较长代码块默认以固定高度折叠,代码区内部可滚动,可用常驻工具栏展开 / 折叠,复制和运行按钮始终可见。可运行的 HTML 代码块会额外显示“运行”,在前端沙箱 iframe 中本地预览完整 HTML/CSS/JS,不调用后端执行;预览环境允许 inline script 以支持完整 HTML 交互,但不允许 `unsafe-eval`,安全边界依赖 WebUI CSP 和 iframe sandbox。预览小窗可拖动位置并调整大小,窗口尺寸变化时会自动保持在可见区域。HTML 预览面板可关闭,也可进入选择模式:第一次点击 iframe 内元素只预览并锁定引用范围,第二次点击确认后才把对应 HTML 片段加入待引用区。 +- AI 回复支持 Markdown 渲染,Markdown 内的常见 HTML 片段会经过 WebUI 白名单净化后自动渲染;完整 HTML 文档或独立块级 HTML 片段会先净化再直渲染,避免被 Markdown 缩进规则误判成代码块。脚本、事件属性、危险协议、危险样式以及 `head` / `style` 等文档元信息会被剥离。聊天中的图片可点击放大查看,支持点击遮罩、关闭按钮或按 `Esc` 退出;普通 Markdown 远端图片由浏览器直接加载 HTTP/HTTPS 来源,附件图片仍通过 Management 代理预览。代码块使用本地随包的 highlight.js 做多语言语法高亮,不依赖外部 CDN;显式语言优先,未知语言会自动检测,库不可用时回退为安全转义文本。较长代码块默认以固定高度折叠,代码区内部可滚动,可用常驻工具栏展开 / 折叠,复制和运行按钮始终可见。可运行的 HTML 代码块会额外显示“运行”,在前端沙箱 iframe 中本地预览完整 HTML/CSS/JS,不调用后端执行;预览环境允许 inline script 以支持完整 HTML 交互,但不允许 `unsafe-eval`,安全边界依赖 WebUI CSP 和 iframe sandbox。预览小窗可拖动位置并调整大小,窗口尺寸变化时会自动保持在可见区域。HTML 预览面板可关闭,也可进入选择模式:第一次点击 iframe 内元素只预览并锁定引用范围,第二次点击确认后才把对应 HTML 片段加入待引用区。 - WebUI 场景下,代码优先直接作为聊天回复输出,不要默认转成文件附件;只有用户明确要求文件交付、代码过长不适合聊天展示,或确需附件工作流时才使用文件。所有代码块都应显式标注语言,例如 ` ```python `、` ```javascript `、` ```html `、` ```bash `,不确定语言时用 ` ```text `。 - AI 消息支持引用:可引用整条 AI 消息、选中的某段文字,或 HTML 预览中点选的元素。引用会显示在输入框右侧的待发送引用条中,可在发送前移除;实际发送时不会新增接口或附件,而是自动在用户消息前拼接为 Markdown `>` 引用块,例如 `> 引用 AI:`,随后和文本、图片、文件一起进入同一条 WebChat job 消息。发送后的引用块会像代码块一样折叠显示,展开后内部固定高度并可滚动。发送失败时待引用内容会保留,发送成功后清空。 - 默认加载最新 50 条消息并滚动到底部;向上滚动到顶部会按后端返回的 `next_before` 游标懒加载更早历史,并保持原视口偏移,避免一次性恢复大量工具块造成卡顿。“自动滚动到底部”开关默认开启并保存在浏览器本地,关闭后 AI 回复、工具 / Agent 状态和 AI 阶段刷新不会打断当前位置;首次加载和主动发送新消息仍会定位到底部。系统开启减少动态效果时,聊天滚动会改为即时跳转。 diff --git a/src/Undefined/webui/app.py b/src/Undefined/webui/app.py index f9dfdb8d..857f896b 100644 --- a/src/Undefined/webui/app.py +++ b/src/Undefined/webui/app.py @@ -34,7 +34,7 @@ "script-src 'self' 'nonce-{nonce}'; " "style-src 'self' 'unsafe-inline' https://fonts.googleapis.com; " "font-src 'self' https://fonts.gstatic.com data:; " - "img-src 'self' data: blob:; " + "img-src 'self' data: blob: http: https:; " "connect-src 'self'; " "base-uri 'self'; " "frame-ancestors 'none'" diff --git a/tests/test_webui_csp_images.py b/tests/test_webui_csp_images.py new file mode 100644 index 00000000..ea8bf1bf --- /dev/null +++ b/tests/test_webui_csp_images.py @@ -0,0 +1,80 @@ +from __future__ import annotations + +import base64 +from html import escape + +import pytest +from aiohttp import web +from aiohttp.test_utils import make_mocked_request +from playwright.async_api import Error, Route, async_playwright + +from test_webui_runtime_chat_behavior import run_scenario +from Undefined.render import _is_missing_playwright_browser +from Undefined.webui.app import security_headers_middleware + + +@pytest.mark.parametrize("scheme", ["http", "https"]) +async def test_rendered_remote_image_loads_under_actual_webui_csp(scheme: str) -> None: + rendered = run_scenario("markdown_sanitizes_unsafe_content") + source = next( + image["src"] + for image in rendered["images"] + if image["src"].startswith("https:") + ) + image_url = source.replace("https:", f"{scheme}:", 1) + page_url = "http://management.example.test/index" + script_url = "https://script.example.test/blocked.js" + fetch_url = "https://api.example.test/blocked" + html = ( + f'' + f'' + ) + + async def handler(request: web.Request) -> web.StreamResponse: + return web.Response(text=html, content_type="text/html") + + response = await security_headers_middleware( + make_mocked_request("GET", "/"), handler + ) + csp = response.headers["Content-Security-Policy"] + intercepted: list[str] = [] + pixel = base64.b64decode( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+jZrQAAAAASUVORK5CYII=" + ) + + async def fulfill(route: Route) -> None: + intercepted.append(route.request.url) + if route.request.url == page_url: + await route.fulfill( + status=200, + content_type="text/html", + headers={"Content-Security-Policy": csp}, + body=html, + ) + else: + await route.fulfill(status=200, content_type="image/png", body=pixel) + + async with async_playwright() as playwright: + try: + browser = await playwright.chromium.launch(headless=True) + except Error as error: + if _is_missing_playwright_browser(error): + pytest.skip( + "Run `uv run playwright install chromium` for CSP browser tests" + ) + raise + try: + page = await browser.new_page() + await page.route("**/*", fulfill) + await page.goto(page_url, wait_until="load") + assert await page.locator("#remote").evaluate( + "image => image.complete && image.naturalWidth === 1" + ) + assert await page.evaluate( + "url => fetch(url).then(() => false).catch(() => true)", fetch_url + ) + assert image_url in intercepted + assert script_url not in intercepted + assert fetch_url not in intercepted + finally: + await browser.close() diff --git a/tests/test_webui_management_api.py b/tests/test_webui_management_api.py index b4d330bd..743e84ca 100644 --- a/tests/test_webui_management_api.py +++ b/tests/test_webui_management_api.py @@ -1214,7 +1214,10 @@ async def _handler(_request: web.Request) -> web.StreamResponse: request = make_mocked_request("GET", "/") response = await webui_app.security_headers_middleware(request, _handler) - assert "img-src 'self' data: blob:;" in response.headers["Content-Security-Policy"] + assert ( + "img-src 'self' data: blob: http: https:;" + in response.headers["Content-Security-Policy"] + ) async def test_non_static_responses_have_no_explicit_cache_control() -> None: From a202c20e205915f9877d32752d02d30983b3afdb Mon Sep 17 00:00:00 2001 From: Null <1708213363@qq.com> Date: Tue, 6 Oct 2026 12:15:42 +0800 Subject: [PATCH 025/121] fix(automations): finish pending saves before propagating cancellation --- docs/automations.md | 2 +- src/Undefined/automations/storage.py | 23 ++++++++++++----------- tests/test_automation_storage_ordering.py | 17 ++++++++++++----- 3 files changed, 25 insertions(+), 17 deletions(-) diff --git a/docs/automations.md b/docs/automations.md index f153491a..722ab6de 100644 --- a/docs/automations.md +++ b/docs/automations.md @@ -91,7 +91,7 @@ 启动时若还没有 `data/automations.json` 但存在旧的 `scheduled_tasks.json`,会自动把旧任务转换为流程图格式写入新文件;旧文件保留不动,之后也不再双写。旧 `/api/v1/schedules` 接口与 `scheduler.*` 工具已下线,统一使用 `/api/v1/automations` 与 `automation.*`。 -任务编辑、删除与运行结果的保存共用串行写入锁,每次写入冻结完整快照并原子替换文件。并发保存不会让旧快照覆盖后续删除或新增;保存期间收到取消时,会先等磁盘写入完成再释放锁并传播取消。 +任务编辑、删除与运行结果的保存共用串行写入锁,每次写入冻结完整快照并原子替换文件。并发保存不会让旧快照覆盖后续删除或新增;等待保存锁或写入期间收到取消时,已发生的内存变更仍会完成持久化,再释放锁并传播取消。 ## 配置与限制 diff --git a/src/Undefined/automations/storage.py b/src/Undefined/automations/storage.py index c830f0b6..62f9402c 100644 --- a/src/Undefined/automations/storage.py +++ b/src/Undefined/automations/storage.py @@ -62,17 +62,18 @@ def load_tasks(self) -> dict[str, Any]: async def save_all(self, tasks: dict[str, Any]) -> None: from Undefined.utils import io - async with self._save_lock: - data_to_save: dict[str, Any] = {} - for task_id, task_info in tasks.items(): - if isinstance(task_info, dict): - data_to_save[str(task_id)] = deepcopy(task_info) - else: - logger.warning("[自动化] 跳过未知任务格式: %s", task_id) - await io.run_cancellation_safe( - io.write_json(self.path, data_to_save, use_lock=True) - ) - logger.info("[自动化] 已保存 %s 条到 %s", len(data_to_save), self.path) + async def save_snapshot() -> None: + async with self._save_lock: + data_to_save: dict[str, Any] = {} + for task_id, task_info in tasks.items(): + if isinstance(task_info, dict): + data_to_save[str(task_id)] = deepcopy(task_info) + else: + logger.warning("[自动化] 跳过未知任务格式: %s", task_id) + await io.write_json(self.path, data_to_save, use_lock=True) + logger.info("[自动化] 已保存 %s 条到 %s", len(data_to_save), self.path) + + await io.run_cancellation_safe(save_snapshot()) def _write_json_sync(self, path: Path, data: dict[str, Any]) -> None: from Undefined.utils.io import write_json_sync diff --git a/tests/test_automation_storage_ordering.py b/tests/test_automation_storage_ordering.py index 2a2bdebc..eeae1c45 100644 --- a/tests/test_automation_storage_ordering.py +++ b/tests/test_automation_storage_ordering.py @@ -16,9 +16,9 @@ @pytest.mark.asyncio -@pytest.mark.parametrize("cancel_first", [False, True]) +@pytest.mark.parametrize("cancel_when", ["none", "writing", "waiting"]) async def test_running_task_save_cannot_restore_concurrently_deleted_task( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch, cancel_first: bool + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, cancel_when: str ) -> None: storage = AutomationStorage(tmp_path / "automations.json", tmp_path / "legacy.json") service = AutomationService( @@ -46,19 +46,26 @@ async def gated_write(path: Path | str, data: Any, use_lock: bool = True) -> Non second: asyncio.Task[bool] | None = None try: await asyncio.wait_for(started.wait(), 1) - if cancel_first: + if cancel_when == "writing": first.cancel() second = asyncio.create_task(service.remove_task("b")) await asyncio.sleep(0) assert "b" not in service.tasks assert len(writes) == 1 + if cancel_when == "waiting": + second.cancel() + await asyncio.sleep(0) + assert not second.done() release.set() results = await asyncio.gather(first, second, return_exceptions=True) - if cancel_first: + if cancel_when == "writing": assert isinstance(results[0], asyncio.CancelledError) else: assert results[0] is None - assert results[1] is True + if cancel_when == "waiting": + assert isinstance(results[1], asyncio.CancelledError) + else: + assert results[1] is True saved = json.loads(storage.path.read_text("utf-8")) assert set(saved) == {"a"} assert saved["a"]["current_executions"] == 1 From c5f6cd4cccbd4359bcd22e41d4b34716c24b4a63 Mon Sep 17 00:00:00 2001 From: Null <1708213363@qq.com> Date: Tue, 6 Oct 2026 12:15:45 +0800 Subject: [PATCH 026/121] fix(scripts): keep reembedding dry runs read-only --- docs/cognitive-memory.md | 2 +- scripts/README.md | 6 +- scripts/reembed_cognitive.py | 106 ++++++++++++++-------- tests/test_reembed_cognitive_script.py | 118 ++++++++++++++++++++++++- 4 files changed, 192 insertions(+), 40 deletions(-) diff --git a/docs/cognitive-memory.md b/docs/cognitive-memory.md index c1e87df1..d37b460e 100644 --- a/docs/cognitive-memory.md +++ b/docs/cognitive-memory.md @@ -447,7 +447,7 @@ data/cognitive/ 更换嵌入模型(维度变化或模型升级)后,需要对向量库进行全量重嵌入。详见 [`scripts/reembed_cognitive.py`](../scripts/reembed_cognitive.py)。 -向量维度发生变化时脚本会先读全量记录、再删除并重建 collection 后写回(ChromaDB 定维后无法原地改维,直接 upsert 异维向量会失败);建议先 `--dry-run` 确认维度变化与记录数。 +向量维度发生变化时脚本先把新向量写入临时 collection,全部完成并校验后才删除旧库并换名(ChromaDB 定维后无法原地改维,直接 upsert 异维向量会失败)。建议先 `--dry-run` 确认维度变化与记录数:预览在系统临时目录的完整副本上读取,源 Chroma 文件和 collection 不会被修改,也不会清理或换名遗留的临时 collection;仍会调用嵌入 API,并可能记录 token 用量。请预留副本所需的临时磁盘空间,并按以下步骤停止写入后运行。 ```bash # 1. 先在 config.toml 中更新 [models.embedding] 为新模型配置 diff --git a/scripts/README.md b/scripts/README.md index a8968496..98b514e0 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -65,7 +65,9 @@ python scripts/sync_config_template.py --stdout **原理**:ChromaDB 存储了完整的原文本(`documents`),脚本读取所有记录,用新模型重新计算向量后 upsert 覆写,metadata 保持不变。 -**维度变化**:collection 在首次写入时定维,异维 upsert 会直接报 `InvalidArgumentError`。脚本会先比较新旧向量维度,检测到变化时先读全量记录、再删除并重建 collection,最后按新维度写回;`--dry-run` 只提示会重建,不做任何写入。 +**维度变化**:collection 在首次写入时定维,异维 upsert 会直接报 `InvalidArgumentError`。脚本把新向量写入临时 collection,全部写完并校验记录数后才删除旧库、把临时库换名为正式库。实际运行会清理未完成的临时库,或在正式库缺失时恢复临时库。 + +`--dry-run` 先复制向量库到系统临时目录,仅在副本上打开 Chroma 客户端,源向量库的数据和文件均不修改。它不会创建缺失的 collection、删除残留临时库或把临时库换名;若只有临时库,会读取其中的记录预览恢复后的重嵌入。预览仍会调用嵌入 API 计算向量,并可能记录 token 用量;需要足够临时磁盘空间保存完整副本。运行前停止机器人及其他写入该库的进程,保证复制和正式迁移读取一致的数据。 **前置条件**:先在 `config.toml` 中将 `[models.embedding]` 更新为新模型配置。 @@ -82,7 +84,7 @@ uv run python scripts/reembed_cognitive.py --profiles-only # 自定义批大小(默认 32,降低可减小 API 压力) uv run python scripts/reembed_cognitive.py --batch-size 16 -# 模拟运行(不实际写入,用于验证配置和统计数量) +# 模拟运行(不修改源 ChromaDB,用于验证配置和统计数量) uv run python scripts/reembed_cognitive.py --dry-run # 详细日志 diff --git a/scripts/reembed_cognitive.py b/scripts/reembed_cognitive.py index 74f362fe..da368b88 100755 --- a/scripts/reembed_cognitive.py +++ b/scripts/reembed_cognitive.py @@ -26,7 +26,7 @@ # 自定义批大小(默认 32) uv run python scripts/reembed_cognitive.py --batch-size 16 - # 模拟运行(不实际写入) + # 模拟运行(在临时副本读取,不修改源 ChromaDB) uv run python scripts/reembed_cognitive.py --dry-run """ @@ -37,7 +37,10 @@ import logging import sys import time +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager from pathlib import Path +from tempfile import TemporaryDirectory from typing import Any import chromadb @@ -53,6 +56,11 @@ from Undefined.config.models import EmbeddingModelConfig # noqa: E402 from Undefined.knowledge.embedder import Embedder # noqa: E402 from Undefined.token_usage_storage import TokenUsageStorage # noqa: E402 +from Undefined.utils.io import ( # noqa: E402 + copy_file_atomic, + ensure_dir, + iter_rglob_files, +) logger = logging.getLogger("reembed_cognitive") @@ -61,6 +69,7 @@ # 维度迁移时临时 collection 的后缀 _STAGING_SUFFIX = "__rebuild_tmp" +_COPY_CHUNK_SIZE = 1024 * 1024 def _staging_name(collection_name: str) -> str: @@ -90,7 +99,7 @@ def _parse_args() -> argparse.Namespace: parser.add_argument( "--dry-run", action="store_true", - help="模拟运行:计算向量但不写入 ChromaDB", + help="模拟运行:从临时副本读取并计算向量,不修改源 ChromaDB", ) parser.add_argument( "--db-path", @@ -380,6 +389,43 @@ def _build_embedder(config: Config) -> Embedder: return embedder +@asynccontextmanager +async def _open_chroma(db_path: Path, *, dry_run: bool) -> AsyncIterator[Any]: + """预览只打开临时副本,隔离客户端初始化、迁移或索引维护的写入。""" + if not dry_run: + yield chromadb.PersistentClient(path=str(db_path)) + return + with TemporaryDirectory(prefix="undefined-reembed-preview-") as directory: + preview_path = Path(directory) / "chroma" + await ensure_dir(preview_path) + async for source in iter_rglob_files(db_path): + await copy_file_atomic( + source, + preview_path / source.relative_to(db_path), + chunk_size=_COPY_CHUNK_SIZE, + ) + logger.info("dry-run:在临时副本读取,源 ChromaDB 不会被修改") + yield chromadb.PersistentClient(path=str(preview_path)) + + +def _load_collection(client: Any, name: str, *, dry_run: bool) -> Any | None: + if not dry_run: + _recover_stale_staging(client, name) + return client.get_or_create_collection(name, metadata={"hnsw:space": "cosine"}) + + names = {collection.name for collection in client.list_collections()} + staging_name = _staging_name(name) + if name in names: + if staging_name in names: + logger.info("dry-run:保留遗留临时 collection %s,不执行清理", staging_name) + return client.get_collection(name) + if staging_name in names: + logger.info("dry-run:读取 %s 预览恢复后的记录,不执行换名", staging_name) + return client.get_collection(staging_name) + logger.info("dry-run:collection %s 不存在,跳过且不创建", name) + return None + + async def _main(args: argparse.Namespace) -> None: config = Config.load(strict=False) @@ -396,41 +442,29 @@ async def _main(args: argparse.Namespace) -> None: logger.error("ChromaDB 目录不存在: %s", db_path) sys.exit(1) - client = chromadb.PersistentClient(path=str(db_path)) - embedder = _build_embedder(config) - total_processed = 0 - - try: - if not args.profiles_only: - _recover_stale_staging(client, "cognitive_events") - events_col = client.get_or_create_collection( - "cognitive_events", metadata={"hnsw:space": "cosine"} - ) - total_processed += await _reembed_collection( - events_col, - "cognitive_events", - embedder, - args.batch_size, - args.dry_run, - client, - ) - - if not args.events_only: - _recover_stale_staging(client, "cognitive_profiles") - profiles_col = client.get_or_create_collection( - "cognitive_profiles", metadata={"hnsw:space": "cosine"} - ) - total_processed += await _reembed_collection( - profiles_col, - "cognitive_profiles", - embedder, - args.batch_size, - args.dry_run, - client, - ) - finally: - await embedder.stop() + async with _open_chroma(Path(db_path), dry_run=args.dry_run) as client: + embedder = _build_embedder(config) + try: + collection_names = [] + if not args.profiles_only: + collection_names.append("cognitive_events") + if not args.events_only: + collection_names.append("cognitive_profiles") + for name in collection_names: + collection = _load_collection(client, name, dry_run=args.dry_run) + if collection is None: + continue + total_processed += await _reembed_collection( + collection, + name, + embedder, + args.batch_size, + args.dry_run, + client, + ) + finally: + await embedder.stop() logger.info("全部完成,共处理 %d 条记录。", total_processed) diff --git a/tests/test_reembed_cognitive_script.py b/tests/test_reembed_cognitive_script.py index 38f27e31..53442533 100644 --- a/tests/test_reembed_cognitive_script.py +++ b/tests/test_reembed_cognitive_script.py @@ -2,7 +2,7 @@ import importlib.util from pathlib import Path -from types import ModuleType +from types import ModuleType, SimpleNamespace from typing import Any, cast import chromadb @@ -29,11 +29,15 @@ class _FixedDimEmbedder: def __init__(self, dimension: int) -> None: self._dimension = dimension self.calls: list[list[str]] = [] + self.stopped = False async def embed(self, texts: list[str]) -> list[list[float]]: self.calls.append(list(texts)) return [[0.1] * self._dimension for _ in texts] + async def stop(self) -> None: + self.stopped = True + class _DriftingDimEmbedder: """首批返回 dim_a、之后返回 dim_b 的假嵌入器,模拟模型输出漂移。""" @@ -137,6 +141,118 @@ async def test_reembed_dry_run_reports_dimension_change_without_writing( assert len(unchanged_sample["embeddings"][0]) == 3 +@pytest.mark.asyncio +@pytest.mark.parametrize("name", ["cognitive_events", "cognitive_profiles"]) +@pytest.mark.parametrize("scenario", ["formal_and_staging", "only_staging", "missing"]) +async def test_main_dry_run_preserves_collections_and_source_files( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, name: str, scenario: str +) -> None: + module = _load_script_module() + db_path = tmp_path / "db" + persistent_client = chromadb.PersistentClient + source_client = persistent_client(path=str(db_path)) + staging_name = f"{name}__rebuild_tmp" + if scenario == "formal_and_staging": + _seed_collection(source_client, name, dimension=3, count=2) + if scenario != "missing": + _seed_collection(source_client, staging_name, dimension=3, count=2) + names_before = {collection.name for collection in source_client.list_collections()} + files_before = { + path.relative_to(db_path): path.read_bytes() + for path in db_path.rglob("*") + if path.is_file() + } + opened_paths: list[Path] = [] + + class _ReadOnlyClient: + def __init__(self, client: Any) -> None: + self._client = client + + def list_collections(self) -> list[Any]: + return cast(list[Any], self._client.list_collections()) + + def get_collection(self, collection_name: str) -> Any: + return self._client.get_collection(collection_name) + + def get_or_create_collection(self, *args: Any, **kwargs: Any) -> Any: + raise AssertionError("dry-run 不能创建 collection") + + def delete_collection(self, *args: Any, **kwargs: Any) -> None: + raise AssertionError("dry-run 不能删除 collection") + + def _open_preview(*, path: str) -> _ReadOnlyClient: + preview_path = Path(path) + assert preview_path != db_path + opened_paths.append(preview_path) + return _ReadOnlyClient(persistent_client(path=path)) + + def _unexpected_recovery(*args: Any, **kwargs: Any) -> None: + raise AssertionError("dry-run 不能清理或换名 staging") + + config = SimpleNamespace( + cognitive=SimpleNamespace(vector_store_path=str(db_path)), + resolve_embedding_model=lambda _feature: SimpleNamespace( + model_name="fake-embedding", dimensions=5 + ), + ) + embedder = _FixedDimEmbedder(5) + monkeypatch.setattr(module, "Config", SimpleNamespace(load=lambda **kwargs: config)) + monkeypatch.setattr(module, "_build_embedder", lambda _config: embedder) + monkeypatch.setattr(module, "_recover_stale_staging", _unexpected_recovery) + monkeypatch.setattr(module.chromadb, "PersistentClient", _open_preview) + args = SimpleNamespace( + db_path=None, + events_only=name == "cognitive_events", + profiles_only=name == "cognitive_profiles", + batch_size=2, + dry_run=True, + ) + await module._main(args) + + assert { + path.relative_to(db_path): path.read_bytes() + for path in db_path.rglob("*") + if path.is_file() + } == files_before + assert { + collection.name for collection in source_client.list_collections() + } == names_before + assert embedder.stopped + assert len(embedder.calls) == (0 if scenario == "missing" else 1) + assert len(opened_paths) == 1 + assert not opened_paths[0].exists() + + +@pytest.mark.asyncio +async def test_main_dry_run_does_not_initialize_an_empty_source_directory( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + module = _load_script_module() + db_path = tmp_path / "empty-db" + db_path.mkdir() + config = SimpleNamespace( + cognitive=SimpleNamespace(vector_store_path=str(db_path)), + resolve_embedding_model=lambda _feature: SimpleNamespace( + model_name="fake-embedding", dimensions=5 + ), + ) + embedder = _FixedDimEmbedder(5) + monkeypatch.setattr(module, "Config", SimpleNamespace(load=lambda **kwargs: config)) + monkeypatch.setattr(module, "_build_embedder", lambda _config: embedder) + await module._main( + SimpleNamespace( + db_path=None, + events_only=False, + profiles_only=False, + batch_size=2, + dry_run=True, + ) + ) + assert list(db_path.iterdir()) == [] + assert embedder.calls == [] + assert embedder.stopped + + @pytest.mark.asyncio async def test_reembed_migration_leaves_no_staging_collection( tmp_path: Path, From 2f4a73e8750c3f2e1b57332aa013a2dd2939da50 Mon Sep 17 00:00:00 2001 From: Null <1708213363@qq.com> Date: Tue, 6 Oct 2026 12:17:44 +0800 Subject: [PATCH 027/121] fix(onebot): collect message tasks when reconnect is cancelled --- ARCHITECTURE.md | 2 +- src/Undefined/onebot/client.py | 41 +++++++++-------- tests/test_onebot_connection_lifecycle.py | 54 +++++++++++++++++++++++ 3 files changed, 78 insertions(+), 19 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 06833845..0cd3a19d 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -886,7 +886,7 @@ description: 从 PDF 文件中提取文本和表格,填写表单。当用户 微信接入由主进程独立管理,不依赖 OneBot 是否在线。`WeixinService` 先校验帐号与 peer,再把已绑定来源映射为逻辑 QQ 私聊交给 `MessageHandler`;未知来源在写历史或调用 AI 之前进入隔离存储。`DeliveryAddress` 将逻辑身份与物理通道分离:`qq:` 和 `wechat:` 可共享同一个用户历史、认知记忆与权限,但发送器、消息合并 scope 和 transport 元数据保持通道隔离。自动化同样持久化规范 `address`,旧 `target_type + target_id` 在加载时转换为 `group:` 或 `qq:` 地址。 -OneBot 消息处理任务由客户端跨连接跟踪。网络断线只结束接收循环、通知未完成的 API 请求连接失败,不等待媒体下载等处理任务即可重连;已有处理任务继续运行。主动断开、停止或取消连接任务时,会取消并等待这些处理任务收尾,避免后台任务遗留。 +OneBot 消息处理任务由客户端跨连接跟踪。网络断线只结束接收循环、通知未完成的 API 请求连接失败,不等待媒体下载等处理任务即可重连;已有处理任务继续运行。主动断开、停止或取消连接任务时,会取消并等待这些处理任务收尾,连接建立或重连等待阶段收到取消也会执行同样的清理,避免后台任务遗留。 ### "车站-列车" 队列模型 diff --git a/src/Undefined/onebot/client.py b/src/Undefined/onebot/client.py index b05e6c48..3639b9e7 100644 --- a/src/Undefined/onebot/client.py +++ b/src/Undefined/onebot/client.py @@ -1067,24 +1067,29 @@ async def run_with_reconnect(self, reconnect_interval: float = 5.0) -> None: self._should_stop = False reconnect_count = 0 - while not self._should_stop: - try: - if reconnect_count > 0: - logger.info(f"[WebSocket] 正在尝试第 {reconnect_count} 次重连...") - await self.connect() - reconnect_count = 0 # 连接成功重置计数 - await self.run() - except websockets.ConnectionClosed as e: - logger.warning(f"[WebSocket] 连接已断开: {e}") - except Exception as e: - logger.error(f"[WebSocket] 发生错误: {e}") - - if self._should_stop: - break - - reconnect_count += 1 - logger.info(f"{reconnect_interval} 秒后尝试重连...") - await asyncio.sleep(reconnect_interval) + try: + while not self._should_stop: + try: + if reconnect_count > 0: + logger.info( + f"[WebSocket] 正在尝试第 {reconnect_count} 次重连..." + ) + await self.connect() + reconnect_count = 0 # 连接成功重置计数 + await self.run() + except websockets.ConnectionClosed as e: + logger.warning(f"[WebSocket] 连接已断开: {e}") + except Exception as e: + logger.error(f"[WebSocket] 发生错误: {e}") + + if self._should_stop: + break + + reconnect_count += 1 + logger.info(f"{reconnect_interval} 秒后尝试重连...") + await asyncio.sleep(reconnect_interval) + finally: + await self._cancel_message_tasks() def stop(self) -> None: """停止运行""" diff --git a/tests/test_onebot_connection_lifecycle.py b/tests/test_onebot_connection_lifecycle.py index 15ec8473..584bb0af 100644 --- a/tests/test_onebot_connection_lifecycle.py +++ b/tests/test_onebot_connection_lifecycle.py @@ -105,3 +105,57 @@ async def handler(data: dict[str, Any]) -> None: await asyncio.wait_for(running, 1) assert cancelled.is_set() assert not client._tasks + + +@pytest.mark.asyncio +@pytest.mark.parametrize("phase", ["sleep", "connect"]) +async def test_reconnect_gap_cancellation_collects_previous_handlers( + monkeypatch: pytest.MonkeyPatch, phase: str +) -> None: + client = OneBotClient("ws://example.invalid") + started = asyncio.Event() + cancelled = asyncio.Event() + gap = asyncio.Event() + connections = 0 + original_run = client.run + + async def handler(data: dict[str, Any]) -> None: + started.set() + try: + await asyncio.Event().wait() + finally: + cancelled.set() + + async def connect() -> None: + nonlocal connections + connections += 1 + if connections == 1: + client.ws = cast(Any, _WebSocket(started)) + else: + gap.set() + await asyncio.Event().wait() + + async def run() -> None: + await original_run() + if phase == "sleep": + gap.set() + + client.set_message_handler(handler) + monkeypatch.setattr(client, "connect", connect) + monkeypatch.setattr(client, "run", run) + running = asyncio.create_task( + client.run_with_reconnect(reconnect_interval=60 if phase == "sleep" else 0) + ) + try: + await asyncio.wait_for(gap.wait(), 1) + assert client._tasks + assert not cancelled.is_set() + running.cancel() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(running, 1) + assert cancelled.is_set() + assert not client._tasks + finally: + running.cancel() + await asyncio.gather(running, return_exceptions=True) + await client.disconnect() From 83036d2c049d4b527ff8df405f24df8f1e3f04ea Mon Sep 17 00:00:00 2001 From: Null <1708213363@qq.com> Date: Tue, 6 Oct 2026 12:19:49 +0800 Subject: [PATCH 028/121] fix(webui): apply validated settings without rereading config --- docs/configuration.md | 16 +++- docs/python-api.md | 2 + src/Undefined/config/webui_settings.py | 21 +++- src/Undefined/webui/app.py | 15 ++- src/Undefined/webui/routes/_auth.py | 9 +- tests/test_webui_config_hot_reload.py | 128 +++++++++++++++++++++++++ tests/test_webui_settings.py | 43 ++++++++- 7 files changed, 221 insertions(+), 13 deletions(-) create mode 100644 tests/test_webui_config_hot_reload.py diff --git a/docs/configuration.md b/docs/configuration.md index 771f8c8d..146d5e6c 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1207,9 +1207,12 @@ api_key = "replace-with-your-key" | `check_updates` | `true` | 打开 WebUI 时是否在后台检查 GitHub 正式 Release | 关键行为: -- 默认密码 `changeme` 禁止登录,必须先修改。 + +- 默认密码 `changeme` 禁止登录,必须先修改;显式写入 `password = "changeme"` 也保持默认密码模式。 - 未配置或为空时,会回退默认密码并标记为”默认密码模式”。 -- `webui.url/port/password/autostart_bot` 修改需重启 WebUI 进程(机器人主进程中也属于重启生效类)。 +- `webui.url` / `webui.port` 修改需重启 WebUI 进程,以重新绑定监听地址和端口。 +- `webui.password` 在配置成功加载后立即刷新 Management 的登录密码,无需重启 WebUI。 +- `webui.autostart_bot` 控制下一次 WebUI 启动时是否自动启动机器人;修改不会立即启动或停止当前机器人进程。 - `check_updates` 支持热更新;关闭后只停止页面打开时的自动检查,概览页仍可手动检查。 - `autostart_bot=true` 时,运行 `uv run Undefined-webui` 会自动拉起 bot 进程,无需手动点击启动按钮;与 WebUI 更新重启后的自动恢复机制(`pending_bot_autostart` marker)互不冲突。 - `uv run deploy up` 在 `container` 与 `host` 模式下都会写入 `autostart_bot = false`;部署后先完成配置与 QQ 登录,再在 WebUI 点击“启动机器人”。 @@ -1439,6 +1442,7 @@ api_key = "replace-with-your-key" ### 5.2 明确“需重启”的字段 以下变更会被记录为“需重启生效”: + - `logging.level` - `logging.file_path` - `logging.max_size_mb` @@ -1448,8 +1452,6 @@ api_key = "replace-with-your-key" - `onebot.token` - `webui.url` - `webui.port` -- `webui.password` -- `webui.autostart_bot` - `api.*`(`enabled/host/port/auth_key/openapi_enabled`) - `memes.blob_dir` - `memes.preview_dir` @@ -1459,7 +1461,11 @@ api_key = "replace-with-your-key" - `naga.*`(`enabled/api_url/api_key/use_proxy/moderation_enabled/mode/allowed_group_ids/blocked_group_ids/allowed_private_ids/blocked_private_ids`) - `models.embedding` / `models.embedding.features.*` / `models.rerank`(嵌入与重排运行时在启动时构造,热更新只提示需重启) +Management 的 WebUI 配置按 [§4.23](#423-webui) 的时机生效:密码和检查更新开关支持热更新,自动启动开关控制下次启动。机器人运行时日志可能仍把 `webui.password` / `webui.autostart_bot` 归为重启提示,这不表示 Management 必须重启才能使用新密码。 + ### 5.3 明确“会执行热应用”的字段 + +- `webui.password` / `webui.check_updates`(Management 的登录密码与检查更新开关立即刷新) - `onebot.file_send_mode` / `onebot.file_send_host`(新投递读取快照;进行中投递及旧 URL 生命周期不变) - 模型发车间隔 / 模型名 / 模型池变更(队列间隔刷新) - `models.grok.model_name` / `models.grok.queue_interval_seconds`(队列间隔刷新) @@ -1493,6 +1499,8 @@ api_key = "replace-with-your-key" TOML 语法错误、编码错误或读取失败会中止本次加载,热重载保留上次有效的完整配置,包括白名单、Runtime API 密钥和 WebUI 密码;已加载配置的文件被删除时也不会切换为默认值。修正文件并再次保存后会正常重载。`strict=False` 只放宽必填项校验,不允许把损坏的文件作为空配置使用。 +运行中的 WebUI 直接从本次已验证的 `Config` 快照刷新设置,不再次读取文件;即使文件随后被其他进程写坏或删除,也不会把密码重置为默认值。首次启动的独立 WebUI 设置加载仍允许用默认设置显示初始化页面,但默认密码始终禁止登录。 + 热更新在共享的 `Config` 实例上逐字段就地生效,整个过程没有 `await`,同一事件循环内的读方不会在一次读取里看到“改了一半”的对象;跨 `await` 的多次读取仍可能分别落在变更前后,需要严格一致的快照时请在单次读取中取全所需字段。 应用阶段按步骤隔离:单步抛错不会中断后续步骤,失败步骤会以 `error` 级日志逐条打印(`热更新步骤失败`),并在末尾汇总一条“热更新未完全生效(运行时状态与 config.toml 不一致)”。异步步骤(技能热重载、自动化并发、配置监听器重启)使用被强引用跟踪的后台任务,任务异常同样以 `error` 级日志输出(`热更新后台任务失败`)。配置订阅者回调抛错时记录失败回调名单,其余回调继续执行。 diff --git a/docs/python-api.md b/docs/python-api.md index 6d7ef330..e42b380f 100644 --- a/docs/python-api.md +++ b/docs/python-api.md @@ -113,6 +113,8 @@ Python 显式 mapping / override > config.toml > 环境变量 > 代码默 `Config.load(strict=False)` 只放宽必填项校验,文件的 TOML 语法、编码或读取错误仍会抛出 `ValueError`。`ConfigManager.reload()` 加载失败时保留原配置;订阅者应用失败会记录日志并保留变更,下次重载相同内容也会再次通知。运行时热更新应用函数会在完成其余同步步骤后汇报失败,后台失败同样会进入重试。 +`WebUISettings.from_config(config, config_exists=...)` 从已验证的 Config 构建设置,不读文件;运行中的 WebUI 使用该入口刷新同一次重载的快照。`config_exists` 表示这份快照是否来自现存配置;显式或隐式的 `changeme` 都会标记为默认密码模式,禁止登录。独立 `load_webui_settings()` 保留首次启动的宽松解析和初始化默认值,不应拿它替代已验证的运行时快照。 + ### `Config.from_mapping` 从内存 dict 构建配置,结构与 `config.toml` 一致: diff --git a/src/Undefined/config/webui_settings.py b/src/Undefined/config/webui_settings.py index 46f4303f..1372627b 100644 --- a/src/Undefined/config/webui_settings.py +++ b/src/Undefined/config/webui_settings.py @@ -4,10 +4,13 @@ from dataclasses import dataclass from pathlib import Path -from typing import Any, Optional +from typing import TYPE_CHECKING, Any, Optional from .coercers import _coerce_bool, _coerce_int, _coerce_str, _normalize_str, _get_value +if TYPE_CHECKING: + from .config_class import Config + DEFAULT_WEBUI_URL = "127.0.0.1" DEFAULT_WEBUI_PORT = 8787 DEFAULT_WEBUI_PASSWORD = "changeme" @@ -25,6 +28,20 @@ class WebUISettings: using_default_password: bool config_exists: bool + @classmethod + def from_config(cls, config: Config, *, config_exists: bool) -> WebUISettings: + """从已验证的 Config 构建运行时设置,不重新读取配置文件。""" + password = config.webui_password or DEFAULT_WEBUI_PASSWORD + return cls( + url=config.webui_url, + port=config.webui_port, + password=password, + autostart_bot=config.webui_autostart_bot, + check_updates=config.webui_check_updates, + using_default_password=password == DEFAULT_WEBUI_PASSWORD, + config_exists=config_exists, + ) + @property def display_url(self) -> str: """用于日志和展示的格式化 URL。""" @@ -72,6 +89,6 @@ def parse_webui_settings(data: dict[str, Any]) -> WebUISettings: password=password_normalized, autostart_bot=autostart_bot, check_updates=check_updates, - using_default_password=False, + using_default_password=password_normalized == DEFAULT_WEBUI_PASSWORD, config_exists=config_exists, ) diff --git a/src/Undefined/webui/app.py b/src/Undefined/webui/app.py index 857f896b..9833ba3f 100644 --- a/src/Undefined/webui/app.py +++ b/src/Undefined/webui/app.py @@ -8,7 +8,13 @@ from typing import Any, Awaitable, Callable from aiohttp import web -from Undefined.config import load_webui_settings, get_config_manager, get_config +from Undefined.config import ( + Config, + WebUISettings, + load_webui_settings, + get_config_manager, + get_config, +) from Undefined.utils.cors import is_allowed_cors_origin, normalize_origin from Undefined.utils import io as async_io from Undefined.utils.self_update import resolve_repo_root @@ -302,7 +308,7 @@ def create_app(*, redirect_to_config_once: bool = False) -> web.Application: # 一次性客户端重定向提示(由 index 处理) app[REDIRECT_TO_CONFIG_ONCE_APP_KEY] = redirect_to_config_once - def _on_config_change(config: Any, changes: dict[str, Any]) -> None: + def _on_config_change(config: Config, changes: dict[str, Any]) -> None: webui_keys = { "webui_url", "webui_port", @@ -314,7 +320,10 @@ def _on_config_change(config: Any, changes: dict[str, Any]) -> None: changes ): logger.info("[WebUI] 检测到 WebUI 配置变更,正在热重载设置...") - app[SETTINGS_APP_KEY] = load_webui_settings() + # 本次快照已成功读取;后续外部写入不能覆盖已验证的设置。 + app[SETTINGS_APP_KEY] = WebUISettings.from_config( + config, config_exists=True + ) config_manager.subscribe(_on_config_change) diff --git a/src/Undefined/webui/routes/_auth.py b/src/Undefined/webui/routes/_auth.py index bcab3ecc..ed4c0776 100644 --- a/src/Undefined/webui/routes/_auth.py +++ b/src/Undefined/webui/routes/_auth.py @@ -4,7 +4,7 @@ from aiohttp.web_response import Response from Undefined.config.loader import CONFIG_PATH, DEFAULT_WEBUI_PASSWORD -from Undefined.config import get_config_manager, load_webui_settings +from Undefined.config import WebUISettings, get_config_manager from ._shared import ( auth_capabilities, routes, @@ -260,8 +260,11 @@ async def password_handler(request: web.Request) -> Response: CONFIG_PATH.write_text( render_toml(patched, comments=load_comment_map()), encoding="utf-8" ) - get_config_manager().reload() - request.app[SETTINGS_APP_KEY] = load_webui_settings() + manager = get_config_manager() + manager.reload() + request.app[SETTINGS_APP_KEY] = WebUISettings.from_config( + manager.load(strict=False), config_exists=True + ) get_session_store(request).clear() resp = web.json_response({"success": True, "message": "Password updated"}) diff --git a/tests/test_webui_config_hot_reload.py b/tests/test_webui_config_hot_reload.py new file mode 100644 index 00000000..b7881621 --- /dev/null +++ b/tests/test_webui_config_hot_reload.py @@ -0,0 +1,128 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Any, cast + +import pytest +from aiohttp import web +from aiohttp.test_utils import make_mocked_request + +from Undefined.config import Config +from Undefined.config.manager import ConfigManager +from Undefined.webui import app as webui_app +from Undefined.webui.routes import _auth +from Undefined.webui.routes._shared import SESSION_STORE_APP_KEY, SETTINGS_APP_KEY + + +def _manager_and_app( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> tuple[Path, ConfigManager, web.Application]: + monkeypatch.chdir(tmp_path) + monkeypatch.setattr("Undefined.config.config_class._load_env", lambda: None) + path = tmp_path / "config.toml" + path.write_text('[webui]\npassword="private-old"\n', encoding="utf-8") + manager = ConfigManager(path) + manager.load(strict=False) + monkeypatch.setattr(webui_app, "get_config_manager", lambda: manager) + return path, manager, webui_app.create_app() + + +def test_management_invalid_reload_preserves_cached_settings( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + path, manager, app = _manager_and_app(tmp_path, monkeypatch) + previous = app[SETTINGS_APP_KEY] + path.write_text("[webui\n", encoding="utf-8") + with pytest.raises(ValueError): + manager.reload(strict=False) + assert app[SETTINGS_APP_KEY] is previous + assert previous.password == "private-old" + assert not previous.using_default_password + + +@pytest.mark.parametrize("external_write", ["malformed", "deleted"]) +def test_management_reload_uses_validated_snapshot_after_external_write( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, external_write: str +) -> None: + monkeypatch.chdir(tmp_path) + monkeypatch.setattr("Undefined.config.config_class._load_env", lambda: None) + path = tmp_path / "config.toml" + path.write_text('[webui]\npassword="private-old"\n', encoding="utf-8") + manager = ConfigManager(path) + config = manager.load(strict=False) + monkeypatch.setattr(webui_app, "get_config_manager", lambda: manager) + + def _write_after_config_loaded(_config: Config, _changes: dict[str, Any]) -> None: + if external_write == "malformed": + path.write_text("[webui\n", encoding="utf-8") + else: + path.unlink() + + # 模拟另一进程在已验证读取后、WebUI 订阅回调前编辑文件。 + manager.subscribe(_write_after_config_loaded) + app = webui_app.create_app() + valid = '[webui]\npassword="private-new"\nport=18787\ncheck_updates=false\n' + path.write_text(valid, encoding="utf-8") + assert manager.reload(strict=False) + settings = app[SETTINGS_APP_KEY] + assert settings.password == config.webui_password == "private-new" + assert settings.port == config.webui_port == 18787 + assert not settings.check_updates + assert not settings.using_default_password + assert settings.config_exists + + path.write_text(valid, encoding="utf-8") + assert manager.reload(strict=False) == {} + assert app[SETTINGS_APP_KEY].password == "private-new" + + +@pytest.mark.asyncio +async def test_management_hot_reload_explicit_default_password_rejects_login( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + path, manager, app = _manager_and_app(tmp_path, monkeypatch) + path.write_text('[webui]\npassword="changeme"\n', encoding="utf-8") + manager.reload(strict=False) + assert app[SETTINGS_APP_KEY].using_default_password + request = make_mocked_request("POST", "/api/login", app=app) + + async def _json() -> dict[str, str]: + return {"password": "changeme"} + + monkeypatch.setattr(request, "json", _json) + response = await _auth.login_handler(request) + assert response.status == 403 + payload = cast(web.Response, response).text + assert payload is not None + assert '"code": "default_password"' in payload + + +@pytest.mark.asyncio +async def test_password_change_keeps_validated_settings_after_external_write( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + path, manager, app = _manager_and_app(tmp_path, monkeypatch) + monkeypatch.setattr(_auth, "get_config_manager", lambda: manager) + + def _corrupt_after_webui_callback( + _config: Config, _changes: dict[str, Any] + ) -> None: + path.write_text("[webui\n", encoding="utf-8") + + manager.subscribe(_corrupt_after_webui_callback) + sessions = app[SESSION_STORE_APP_KEY] + token = sessions.create() + request = make_mocked_request( + "POST", "/api/password", headers={"X-Auth-Token": token}, app=app + ) + + async def _json() -> dict[str, str]: + return {"current_password": "private-old", "new_password": "private-new"} + + monkeypatch.setattr(request, "json", _json) + response = await _auth.password_handler(request) + assert response.status == 200 + assert path.read_text(encoding="utf-8") == "[webui\n" + assert app[SETTINGS_APP_KEY].password == "private-new" + assert not app[SETTINGS_APP_KEY].using_default_password + assert not sessions.is_valid(token) diff --git a/tests/test_webui_settings.py b/tests/test_webui_settings.py index 2e689d26..bdad95da 100644 --- a/tests/test_webui_settings.py +++ b/tests/test_webui_settings.py @@ -7,7 +7,10 @@ from pathlib import Path -from Undefined.config.webui_settings import load_webui_settings +import pytest + +from Undefined.config import Config +from Undefined.config.webui_settings import WebUISettings, load_webui_settings def test_load_webui_settings_with_autostart_bot_true(tmp_path: Path) -> None: @@ -188,3 +191,41 @@ def test_load_webui_settings_check_updates_can_be_disabled(tmp_path: Path) -> No settings = load_webui_settings(config_file) assert settings.check_updates is False + + +def test_load_webui_settings_explicit_default_password_stays_disabled( + tmp_path: Path, +) -> None: + config_file = tmp_path / "config.toml" + config_file.write_text('[webui]\npassword="changeme"\n', encoding="utf-8") + settings = load_webui_settings(config_file) + assert settings.password == "changeme" + assert settings.using_default_password is True + assert settings.config_exists is True + + +@pytest.mark.parametrize("password", ["private-password", "changeme"]) +@pytest.mark.parametrize("config_exists", [False, True]) +def test_settings_from_config_preserves_snapshot_metadata( + password: str, config_exists: bool +) -> None: + config = Config.from_mapping( + { + "webui": { + "url": "::1", + "port": 18787, + "password": password, + "autostart_bot": True, + "check_updates": False, + } + }, + strict=False, + ) + settings = WebUISettings.from_config(config, config_exists=config_exists) + assert settings.url == "::1" + assert settings.port == 18787 + assert settings.password == password + assert settings.autostart_bot is True + assert settings.check_updates is False + assert settings.using_default_password is (password == "changeme") + assert settings.config_exists is config_exists From 9dbf6f3d78072018a845f96b5958a0bd23396f4a Mon Sep 17 00:00:00 2001 From: Null <1708213363@qq.com> Date: Tue, 6 Oct 2026 12:39:41 +0800 Subject: [PATCH 029/121] fix(api): use a typed application key for runtime context --- docs/development.md | 2 ++ src/Undefined/api/app.py | 4 +++- tests/test_runtime_api_app.py | 31 +++++++++++++++++++++++++++++++ 3 files changed, 36 insertions(+), 1 deletion(-) create mode 100644 tests/test_runtime_api_app.py diff --git a/docs/development.md b/docs/development.md index 94fa2249..b6345220 100644 --- a/docs/development.md +++ b/docs/development.md @@ -63,6 +63,8 @@ src/Undefined/ - **Toolsets 开发专版**:[toolsets/README.md](../src/Undefined/skills/toolsets/README.md) - **Commands 开发专版**:[详细斜杠指令开发指南](slash-commands.md) +Runtime API 和 WebUI 向 aiohttp `Application` 存储上下文等共享对象时,使用模块级 `web.AppKey` 常量并在所有读写处复用,保持类型安全并避免 `NotAppKeyWarning`。Runtime API 的应用构造回归测试会将该警告视为错误。 + ### CHANGELOG 维护约定 - 仓库根目录的 `CHANGELOG.md` 是正式版本历史的唯一事实来源。 diff --git a/src/Undefined/api/app.py b/src/Undefined/api/app.py index ea47393b..15bbb0a5 100644 --- a/src/Undefined/api/app.py +++ b/src/Undefined/api/app.py @@ -44,6 +44,8 @@ logger = logging.getLogger(__name__) +RUNTIME_API_CONTEXT_APP_KEY = web.AppKey("runtime_api_context", RuntimeAPIContext) + class RuntimeAPIServer: def __init__( @@ -145,7 +147,7 @@ async def _auth_middleware( return response app = web.Application(middlewares=[_auth_middleware]) - app["runtime_api_context"] = self._context + app[RUNTIME_API_CONTEXT_APP_KEY] = self._context app.add_routes( [ web.get("/health", self._health_handler), diff --git a/tests/test_runtime_api_app.py b/tests/test_runtime_api_app.py new file mode 100644 index 00000000..a53a5358 --- /dev/null +++ b/tests/test_runtime_api_app.py @@ -0,0 +1,31 @@ +"""Regression coverage for Runtime API application construction.""" + +from __future__ import annotations + +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from Undefined.api import RuntimeAPIContext, RuntimeAPIServer, webchat_store +from Undefined.api.app import RUNTIME_API_CONTEXT_APP_KEY + + +@pytest.mark.filterwarnings("error::aiohttp.web_exceptions.NotAppKeyWarning") +def test_application_preserves_context_without_app_key_warning( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(webchat_store, "WEBCHAT_CONVERSATIONS_DIR", tmp_path) + context = RuntimeAPIContext( + config_getter=SimpleNamespace, + onebot=None, + ai=None, + command_dispatcher=None, + queue_manager=None, + history_manager=None, + ) + server = RuntimeAPIServer(context, host="127.0.0.1", port=0) + + app = server._create_app() + + assert app[RUNTIME_API_CONTEXT_APP_KEY] is context From 61a9d4bc5501f6886aa2a47a734ceda8ee8d9379 Mon Sep 17 00:00:00 2001 From: Null <1708213363@qq.com> Date: Tue, 6 Oct 2026 13:00:06 +0800 Subject: [PATCH 030/121] fix(batcher): bind inflight replies to their batch token --- docs/message-batching.md | 5 +- src/Undefined/services/coordinator/group.py | 18 ++- src/Undefined/services/coordinator/private.py | 16 ++- .../services/message_batcher/scheduler.py | 30 +++-- tests/test_message_batcher.py | 66 +++++++-- tests/test_message_batcher_integration.py | 126 ++++++++++++++++++ 6 files changed, 233 insertions(+), 28 deletions(-) diff --git a/docs/message-batching.md b/docs/message-batching.md index 726ef228..04f7b34a 100644 --- a/docs/message-batching.md +++ b/docs/message-batching.md @@ -145,10 +145,11 @@ allow_cancel_after_send = false - T2 预发送会给队列请求附带 `BatchDispatchToken`。新消息抢占时先取消旧 token;若旧请求已入队但尚未执行,`AICoordinator.execute_reply()` 会直接跳过,避免队列拥堵窗口里的陈旧回复。 - T2 的 `flush_callback` 若异常或被取消,桶会从 `SPECULATING` 回滚到 `TYPING` 并换新 token,保留原 items 等 T1 正常重试,避免静默丢消息。 - T1 到期时如果 batch 已经被 T2 投机发出,只负责结束 bucket/等待已知 inflight,不会再次调用 `flush_callback`,避免同一批消息重复入队。 -- `unregister_inflight(scope, sender_id, task)` 必须携带 task 身份并校验;旧任务的 `finally` 不会误清理新一轮已注册的 inflight。 +- inflight 注册必须携带当前批次的 `BatchDispatchToken`,并校验 token 对象身份与取消状态;同一发送者独立旁路的 `@bot`、拍一拍回复不会覆盖投机任务,也不会被后续批次补充误取消。 +- `unregister_inflight(scope, sender_id, task, token=...)` 同时校验 token 和 task 身份;旧任务的 `finally` 不会误清理新一轮已注册的 inflight。 - `flush_all()` 在关停时设置 shutdown 标记,循环遍历所有桶执行等价 T1 路径,并 `await` 所有未完成的 flush task;若收尾过程中又出现新桶,会继续清空直到没有 pending bucket。shutdown 之后的新消息直接发车,不再开缓冲桶。 - `MessageHandler.close()` 的顺序是:停止自动管线热重载 → `message_batcher.flush_all()` → `queue_manager.drain()` 等待已入队/在途回复自然收敛 → `queue_manager.stop()` → flush 历史落盘。 -- coordinator 在 `execute_reply` 入口调用 `register_inflight(scope, sender_id, task, ctx)`,在 `finally` 调 `unregister_inflight(...)`;`asyncio.CancelledError` 被识别为 "投机抢占",仅记录信息日志且不重试。 +- coordinator 在执行回复时调用 `register_inflight(scope, sender_id, task, ctx, token=...)`,仅注册成功后才在 `finally` 注销同一 token 的任务;`asyncio.CancelledError` 被识别为 "投机抢占",仅记录信息日志且不重试。 - 主调用取消或等待超时会同步取消其 LLM 子请求:尚未执行或正在等待重试的子请求会跳过模型调用,已执行的模型任务会收到取消。已经发到上游的请求无法撤回,但不会继续排队重试或产生无人消费的新调用。 ### 兼容回退 diff --git a/src/Undefined/services/coordinator/group.py b/src/Undefined/services/coordinator/group.py index e07ea97d..4c88d518 100644 --- a/src/Undefined/services/coordinator/group.py +++ b/src/Undefined/services/coordinator/group.py @@ -13,7 +13,11 @@ from Undefined.context import RequestContext from Undefined.context_resource_registry import collect_context_resources from Undefined.render import render_html_to_image, render_markdown_to_html -from Undefined.services.message_batcher import BufferedMessage, make_scope +from Undefined.services.message_batcher import ( + BatchDispatchToken, + BufferedMessage, + make_scope, +) from Undefined.utils.recent_messages import get_recent_messages_prefer_local from Undefined.utils.xml import ( escape_xml_attr, @@ -207,6 +211,10 @@ async def _execute_auto_reply(self, request: dict[str, Any]) -> None: ) # 用于向 batcher 注册 inflight 任务(仅当本请求源自合并桶时生效) batcher_scope: str | None = make_scope(group_id=group_id) if group_id else None + batch_token_raw = request.get("_message_batcher_token") + batch_token = ( + batch_token_raw if isinstance(batch_token_raw, BatchDispatchToken) else None + ) async with RequestContext( request_type="group", @@ -290,10 +298,10 @@ async def send_like_cb(uid: int, times: int = 1) -> None: batcher is not None and batcher_scope is not None and current_task is not None - ): - batcher.register_inflight( - batcher_scope, sender_id, current_task, ctx + and batcher.register_inflight( + batcher_scope, sender_id, current_task, ctx, token=batch_token ) + ): registered_task = current_task try: await self.ai.ask( @@ -331,7 +339,7 @@ async def send_like_cb(uid: int, times: int = 1) -> None: and registered_task is not None ): batcher.unregister_inflight( - batcher_scope, sender_id, registered_task + batcher_scope, sender_id, registered_task, token=batch_token ) except asyncio.CancelledError: # 投机预发送被新消息抢占取消:不写错误日志、不重试 diff --git a/src/Undefined/services/coordinator/private.py b/src/Undefined/services/coordinator/private.py index cdb6cf14..58635e62 100644 --- a/src/Undefined/services/coordinator/private.py +++ b/src/Undefined/services/coordinator/private.py @@ -19,7 +19,11 @@ from Undefined.context import RequestContext from Undefined.context_resource_registry import collect_context_resources from Undefined.render import render_html_to_image, render_markdown_to_html -from Undefined.services.message_batcher import BufferedMessage, make_scope +from Undefined.services.message_batcher import ( + BatchDispatchToken, + BufferedMessage, + make_scope, +) from Undefined.utils.recent_messages import get_recent_messages_prefer_local from Undefined.utils.message_targets import DeliveryAddress, parse_delivery_address from Undefined.utils.message_reply import ReplyContext @@ -187,6 +191,10 @@ async def _execute_private_reply(self, request: dict[str, Any]) -> None: raise ValueError("私聊投递地址与逻辑 QQ 身份不一致") channel = address.channel batcher_scope = str(request.get("batch_scope") or make_scope(user_id=user_id)) + batch_token_raw = request.get("_message_batcher_token") + batch_token = ( + batch_token_raw if isinstance(batch_token_raw, BatchDispatchToken) else None + ) async with RequestContext( request_type="private", @@ -315,8 +323,10 @@ async def send_private_cb( batcher is not None and batcher_scope is not None and current_task is not None + and batcher.register_inflight( + batcher_scope, user_id, current_task, ctx, token=batch_token + ) ): - batcher.register_inflight(batcher_scope, user_id, current_task, ctx) registered_task = current_task typing_started = False try: @@ -375,7 +385,7 @@ async def send_private_cb( and registered_task is not None ): batcher.unregister_inflight( - batcher_scope, user_id, registered_task + batcher_scope, user_id, registered_task, token=batch_token ) if result: scope_key = build_attachment_scope( diff --git a/src/Undefined/services/message_batcher/scheduler.py b/src/Undefined/services/message_batcher/scheduler.py index 30dcd4a0..f97d808f 100644 --- a/src/Undefined/services/message_batcher/scheduler.py +++ b/src/Undefined/services/message_batcher/scheduler.py @@ -305,34 +305,46 @@ def register_inflight( sender_id: int, task: asyncio.Task[Any], request_context: Any = None, - ) -> None: + *, + token: BatchDispatchToken | None, + ) -> bool: """coordinator 在 ``execute_reply`` 开头上报 inflight LLM 任务。 - 如果桶不存在或 phase 不是 SPECULATING,则忽略(说明这次 fire 不是投机的)。 + 只接受当前投机批次的 token,独立旁路请求或过期批次不能占用此桶。 """ key = (scope, sender_id) state = self._buckets.get(key) - if state is None: - return - if state.phase is not BatchPhase.SPECULATING: - return + if ( + state is None + or state.phase is not BatchPhase.SPECULATING + or token is None + or token.cancelled + or state.dispatch_token is not token + ): + return False state.inflight = _InflightInfo(task=task, request_context=request_context) logger.debug( "[MessageBatcher] 注册 inflight 任务: scope=%s sender=%s", scope, sender_id, ) + return True # 注销 inflight 任务 def unregister_inflight( - self, scope: str, sender_id: int, task: asyncio.Task[Any] + self, + scope: str, + sender_id: int, + task: asyncio.Task[Any], + *, + token: BatchDispatchToken | None, ) -> None: """coordinator 在 ``execute_reply`` 结束(含异常/取消)时上报。""" key = (scope, sender_id) state = self._buckets.get(key) - if state is None: + if state is None or token is None or state.dispatch_token is not token: return - if state.inflight is not None and state.inflight.task is not task: + if state.inflight is None or state.inflight.task is not task: logger.debug( "[MessageBatcher] 忽略过期 inflight 注销: scope=%s sender=%s phase=%s", scope, diff --git a/tests/test_message_batcher.py b/tests/test_message_batcher.py index 5970bb74..98d48158 100644 --- a/tests/test_message_batcher.py +++ b/tests/test_message_batcher.py @@ -3,6 +3,7 @@ from __future__ import annotations import asyncio +from dataclasses import replace import time import pytest @@ -42,6 +43,12 @@ def _make_item( ) +def _current_token(batcher: MessageBatcher) -> BatchDispatchToken: + token = batcher._buckets[("group:1", 100)].dispatch_token + assert token is not None + return token + + class _Recorder: def __init__(self) -> None: self.batches: list[list[BufferedMessage]] = [] @@ -388,7 +395,9 @@ async def slow_flush(items: list[BufferedMessage]) -> None: await asyncio.sleep(0.1) # 模拟 coordinator 上报 inflight inflight_task = next(iter(batcher._pending_tasks)) - batcher.register_inflight("group:1", 100, inflight_task, fake_ctx) + batcher.register_inflight( + "group:1", 100, inflight_task, fake_ctx, token=_current_token(batcher) + ) # 第二条消息到达,应取消 inflight await batcher.submit(_make_item(text="m2")) await asyncio.wait_for(cancelled.wait(), timeout=1.0) @@ -421,7 +430,9 @@ async def flush(items: list[BufferedMessage]) -> None: await batcher.submit(_make_item(text="m1")) await asyncio.sleep(0.08) inflight_task = next(iter(batcher._pending_tasks)) - batcher.register_inflight("group:1", 100, inflight_task, fake_ctx) + batcher.register_inflight( + "group:1", 100, inflight_task, fake_ctx, token=_current_token(batcher) + ) # 新消息到达:投机已发过消息,inflight 不应被 cancel await batcher.submit(_make_item(text="m2")) # 等 inflight 自然完成 @@ -449,9 +460,10 @@ async def test_completed_speculation_does_not_reprocess_answered_items( await batcher.submit(_make_item(text="m1")) await batcher._handle_t2(("group:1", 100)) inflight = asyncio.create_task(asyncio.sleep(0)) - batcher.register_inflight("group:1", 100, inflight, fake_ctx) + token = _current_token(batcher) + batcher.register_inflight("group:1", 100, inflight, fake_ctx, token=token) await inflight - batcher.unregister_inflight("group:1", 100, inflight) + batcher.unregister_inflight("group:1", 100, inflight, token=token) await batcher.submit(_make_item(text="m2")) await batcher.flush_sender("group:1", 100) @@ -494,7 +506,9 @@ async def flush(items: list[BufferedMessage]) -> None: await asyncio.sleep(0.08) assert batcher._pending_tasks inflight_task = next(iter(batcher._pending_tasks)) - batcher.register_inflight("group:1", 100, inflight_task, fake_ctx) + batcher.register_inflight( + "group:1", 100, inflight_task, fake_ctx, token=_current_token(batcher) + ) await batcher.submit(_make_item(text="m2")) # 正常情况下 cancel 信号会在 50ms 内传到 callback;超时即视为未取消 @@ -564,7 +578,9 @@ async def flush(items: list[BufferedMessage]) -> None: await batcher.submit(_make_item(text="m1")) await asyncio.wait_for(inflight_started.wait(), timeout=0.5) inflight_task = next(iter(batcher._pending_tasks)) - batcher.register_inflight("group:1", 100, inflight_task, fake_ctx) + batcher.register_inflight( + "group:1", 100, inflight_task, fake_ctx, token=_current_token(batcher) + ) # 等到 T1 触发,桶切到 FINALIZING 等 inflight await asyncio.sleep(0.12) # 此刻新消息进入 FINALIZING 分支,开新桶 @@ -701,13 +717,16 @@ async def enqueue_only(items: list[BufferedMessage]) -> None: try: await batcher.submit(_make_item(text="m1")) await asyncio.sleep(0.08) - batcher.register_inflight("group:1", 100, old_task, fake_ctx) + old_token = _current_token(batcher) + batcher.register_inflight("group:1", 100, old_task, fake_ctx, token=old_token) await batcher.submit(_make_item(text="m2")) await asyncio.sleep(0.08) - batcher.register_inflight("group:1", 100, new_task, fake_ctx) + new_token = _current_token(batcher) + batcher.register_inflight("group:1", 100, new_task, fake_ctx, token=new_token) - batcher.unregister_inflight("group:1", 100, old_task) + batcher.unregister_inflight("group:1", 100, old_task, token=old_token) + batcher.unregister_inflight("group:1", 100, new_task, token=old_token) snap = batcher.snapshot() assert snap["buckets"][0]["has_inflight"] is True finally: @@ -716,6 +735,35 @@ async def enqueue_only(items: list[BufferedMessage]) -> None: await asyncio.gather(old_task, new_task, return_exceptions=True) +@pytest.mark.asyncio +@pytest.mark.parametrize("invalid_token", ["missing", "same_fields", "cancelled"]) +async def test_inflight_registration_requires_current_token_identity( + invalid_token: str, +) -> None: + batcher = MessageBatcher( + MessageBatcherConfig(enabled=True, window_seconds=60.0, pre_send_seconds=10.0), + _Recorder(), + ) + task = asyncio.create_task(asyncio.sleep(60.0)) + try: + await batcher.submit(_make_item()) + await batcher._handle_t2(("group:1", 100)) + current_token = _current_token(batcher) + token: BatchDispatchToken | None = current_token + if invalid_token == "missing": + token = None + elif invalid_token == "same_fields": + token = replace(current_token) + else: + current_token.cancel() + assert not batcher.register_inflight("group:1", 100, task, token=token) + assert batcher._buckets[("group:1", 100)].inflight is None + finally: + task.cancel() + await asyncio.gather(task, return_exceptions=True) + await batcher.flush_all() + + @pytest.mark.asyncio async def test_flush_all_loops_until_concurrent_bucket_is_flushed() -> None: """flush_all 快照后若 callback 又创建新桶,也应继续清空。""" diff --git a/tests/test_message_batcher_integration.py b/tests/test_message_batcher_integration.py index f638bcd7..4c8745a8 100644 --- a/tests/test_message_batcher_integration.py +++ b/tests/test_message_batcher_integration.py @@ -20,6 +20,7 @@ from Undefined.handlers import MessageHandler from Undefined.services.coordinator import AICoordinator from Undefined.services.message_batcher import BatchDispatchToken, MessageBatcher +from Undefined.services.queue_manager import QueueManager def _make_coordinator( @@ -391,6 +392,131 @@ async def test_execute_reply_skips_cancelled_batch_token() -> None: execute_auto.assert_not_called() +@pytest.mark.asyncio +@pytest.mark.parametrize("bypass", ["group_at", "group_poke", "private_poke"]) +async def test_bypass_reply_preserves_speculative_inflight_ownership( + bypass: str, +) -> None: + coordinator, _mock_queue, batcher = _make_coordinator(window_seconds=60.0) + batcher.update_config( + MessageBatcherConfig(enabled=True, window_seconds=60.0, pre_send_seconds=10.0) + ) + coordinator.config.bot_qq = 999 + coordinator.config.get_context_recent_messages_limit = lambda: 0 + coordinator.scheduler = None + sent: list[str] = [] + + async def send_message(_target: Any, message: str, **_kwargs: Any) -> None: + sent.append(message) + + coordinator.sender = SimpleNamespace( + send_group_message=send_message, + send_address_message=send_message, + send_private_message=AsyncMock(), + ) + coordinator.onebot = SimpleNamespace( + get_image=AsyncMock(), get_forward_msg=AsyncMock(), send_like=AsyncMock() + ) + coordinator.ai.memory_storage = None + coordinator.ai.runtime_config = coordinator.config + started = {name: asyncio.Event() for name in ("old", "bypass", "new")} + release = {name: asyncio.Event() for name in started} + old_cancelled = asyncio.Event() + finish_old_cancel = asyncio.Event() + tasks: dict[str, asyncio.Task[Any]] = {} + requests: dict[str, dict[str, Any]] = {} + + def request_name(question: str) -> str: + if "独立旁路" in question: + return "bypass" + return "new" if "批次补充" in question else "old" + + async def ask(question: str, **kwargs: Any) -> None: + name = request_name(question) + current_task = asyncio.current_task() + assert current_task is not None + tasks[name] = current_task + started[name].set() + try: + await release[name].wait() + except asyncio.CancelledError: + if name == "old": + old_cancelled.set() + # 让旧请求的 finally 在新 token 注册之后执行。 + await finish_old_cancel.wait() + raise + await kwargs["send_message_callback"](f"{name} reply") + + async def execute(request: dict[str, Any]) -> None: + requests[request_name(request["full_question"])] = request + await coordinator.execute_reply(request) + + coordinator.ai.ask = ask + coordinator._is_at_bot = lambda content: any( + item.get("type") == "at" for item in content + ) + queue = QueueManager(ai_request_interval=0.0) + coordinator.queue_manager = queue + queue.start(execute) + scope = "private:100" if bypass == "private_poke" else "group:1" + key = (scope, 100) + + async def submit(text: str, *, independent: bool = False) -> None: + if bypass == "private_poke": + await coordinator.handle_private_reply(100, text, [], is_poke=independent) + else: + content = ( + [{"type": "at", "data": {"qq": "999"}}] + if independent and bypass == "group_at" + else [] + ) + await coordinator.handle_auto_reply( + 1, + 100, + text, + content, + is_poke=independent and bypass == "group_poke", + ) + + try: + await submit("旧批次") + await batcher._handle_t2(key) + await asyncio.wait_for(started["old"].wait(), timeout=1.0) + await submit("独立旁路", independent=True) + await asyncio.wait_for(started["bypass"].wait(), timeout=1.0) + assert "_message_batcher_token" not in requests["bypass"] + state = batcher._buckets[key] + assert state.inflight is not None + assert state.inflight.task is tasks["old"] + + await submit("批次补充") + await asyncio.wait_for(old_cancelled.wait(), timeout=1.0) + assert not tasks["bypass"].done() + assert requests["old"]["_message_batcher_token"].cancelled + await batcher._handle_t2(key) + await asyncio.wait_for(started["new"].wait(), timeout=1.0) + finish_old_cancel.set() + with pytest.raises(asyncio.CancelledError): + await tasks["old"] + state = batcher._buckets[key] + assert state.inflight is not None + assert state.inflight.task is tasks["new"] + + release["bypass"].set() + await tasks["bypass"] + assert state.inflight is not None + assert state.inflight.task is tasks["new"] + release["new"].set() + await tasks["new"] + assert sent == ["bypass reply", "new reply"] + finally: + finish_old_cancel.set() + for state in list(batcher._buckets.values()): + batcher._cancel_t1(state) + batcher._cancel_t2(state) + await queue.stop() + + @pytest.mark.asyncio async def test_message_handler_close_flushes_batcher_then_drains_queue() -> None: handler: Any = object.__new__(MessageHandler) From 186f53ae7e4382bfb32ad82b3cefacf9803a9027 Mon Sep 17 00:00:00 2001 From: Null <1708213363@qq.com> Date: Tue, 6 Oct 2026 13:00:22 +0800 Subject: [PATCH 031/121] fix(cognitive): confine profile paths to their storage directory --- docs/python-api.md | 2 + src/Undefined/api/routes/cognitive.py | 5 +- src/Undefined/cognitive/profile_storage.py | 32 +++- .../skills/toolsets/cognitive/README.md | 5 +- .../toolsets/cognitive/get_profile/handler.py | 11 +- tests/test_cognitive_profile_paths.py | 155 ++++++++++++++++++ 6 files changed, 199 insertions(+), 11 deletions(-) create mode 100644 tests/test_cognitive_profile_paths.py diff --git a/docs/python-api.md b/docs/python-api.md index e42b380f..c86360bf 100644 --- a/docs/python-api.md +++ b/docs/python-api.md @@ -297,6 +297,8 @@ server = RuntimeAPIServer(RuntimeAPIContext(...)) | `Undefined.api` | stable | `RuntimeAPIServer`, `RuntimeAPIContext` | | `Undefined.mcp` | subpackage | `MCPToolRegistry`, `MCPToolSetRegistry` | +`ProfileStorage` 的所有读写、历史版本和合并锁操作只接受 `user` / `group` 实体类型,以及非空的单层实体 ID(不接受路径分隔符、`.` / `..` 或 Windows 盘符)。侧写文件与历史版本解析符号链接后必须仍位于 `base_path` 内,否则抛出 `ValueError`;Runtime 的侧写查询接口将这类错误转换为 HTTP 400。 + --- ## 相关文档 diff --git a/src/Undefined/api/routes/cognitive.py b/src/Undefined/api/routes/cognitive.py index 66096994..57d803cc 100644 --- a/src/Undefined/api/routes/cognitive.py +++ b/src/Undefined/api/routes/cognitive.py @@ -75,7 +75,10 @@ async def cognitive_profile_handler( if not entity_type or not entity_id: return _json_error("entity_type/entity_id are required", status=400) - profile = await cognitive_service.get_profile(entity_type, entity_id) + try: + profile = await cognitive_service.get_profile(entity_type, entity_id) + except ValueError as exc: + return _json_error(str(exc), status=400) return web.json_response( { "entity_type": entity_type, diff --git a/src/Undefined/cognitive/profile_storage.py b/src/Undefined/cognitive/profile_storage.py index ac657363..e8798642 100644 --- a/src/Undefined/cognitive/profile_storage.py +++ b/src/Undefined/cognitive/profile_storage.py @@ -7,7 +7,7 @@ import os import tempfile from datetime import datetime -from pathlib import Path +from pathlib import Path, PureWindowsPath from typing import Any from Undefined.utils.io import run_cancellation_safe @@ -30,6 +30,7 @@ def __init__(self, base_path: str | Path, revision_keep: int = 5) -> None: ) def _get_lock(self, entity_type: str, entity_id: str) -> asyncio.Lock: + self._validate_entity(entity_type, entity_id) # setdefault 原子插入:并发首次进入同一实体时双方拿到同一把锁, # 不会出现各自建锁、后建覆盖先建导致互斥失效 return self._locks.setdefault(f"{entity_type}:{entity_id}", asyncio.Lock()) @@ -41,15 +42,38 @@ def merge_guard(self, entity_type: str, entity_id: str) -> asyncio.Lock: (后写覆盖先写,先前的观察永久丢失)。与文件写入锁分开,避免与 `write_profile` 的锁重入死锁。锁的创建经 `dict.setdefault` 原子完成。 """ + self._validate_entity(entity_type, entity_id) return self._merge_locks.setdefault( f"{entity_type}:{entity_id}", asyncio.Lock() ) def _profile_path(self, entity_type: str, entity_id: str) -> Path: - return self._base / f"{entity_type}s" / f"{entity_id}.md" + self._validate_entity(entity_type, entity_id) + return self._contained_path(self._base / f"{entity_type}s" / f"{entity_id}.md") def _history_dir(self, entity_type: str, entity_id: str) -> Path: - return self._base / "history" / entity_type / entity_id + self._validate_entity(entity_type, entity_id) + return self._contained_path(self._base / "history" / entity_type / entity_id) + + @staticmethod + def _validate_entity(entity_type: str, entity_id: str) -> None: + if entity_type not in {"user", "group"}: + raise ValueError(f"非法的侧写实体类型: {entity_type!r}") + if ( + not entity_id.strip() + or entity_id in {".", ".."} + or "/" in entity_id + or "\\" in entity_id + or "\x00" in entity_id + or PureWindowsPath(entity_id).drive + ): + raise ValueError(f"非法的侧写实体 ID: {entity_id!r}") + + def _contained_path(self, path: Path) -> Path: + # 同时检查已存在的目录/文件符号链接,不能只过滤输入中的 ../。 + if not path.resolve().is_relative_to(self._base.resolve()): + raise ValueError("侧写路径超出存储目录") + return path async def read_profile(self, entity_type: str, entity_id: str) -> str | None: p = self._profile_path(entity_type, entity_id) @@ -169,7 +193,7 @@ async def read_revision( ) -> str | None: """读取指定历史版本内容;版本名取 `list_revisions` 的返回值。""" name = self._normalize_revision_name(revision) - path = self._history_dir(entity_type, entity_id) / name + path = self._contained_path(self._history_dir(entity_type, entity_id) / name) def _read() -> str | None: if not path.exists(): diff --git a/src/Undefined/skills/toolsets/cognitive/README.md b/src/Undefined/skills/toolsets/cognitive/README.md index 3cd96298..8565738a 100644 --- a/src/Undefined/skills/toolsets/cognitive/README.md +++ b/src/Undefined/skills/toolsets/cognitive/README.md @@ -7,7 +7,9 @@ | 参数 | 类型 | 必填 | 说明 | |------|------|------|------| | `entity_type` | `string(user/group)` | 是 | 实体类型 | -| `entity_id` | `string` | 是 | 用户ID或群ID | +| `entity_id` | `string` | 是 | 用户ID或群ID,不接受路径、空值或 Windows 盘符 | + +实体类型仅支持 `user` / `group`。侧写与历史版本都只能访问配置的侧写存储目录;指向目录外的符号链接会被拒绝。非法查询在工具中返回参数错误,在 Runtime API 中返回 HTTP 400。 ## search_events @@ -33,4 +35,3 @@ | `query` | `string` | 是 | 搜索关键词 | | `entity_type` | `string(user/group)` | 否 | 限定类型(可选) | | `top_k` | `integer` | 否 | 返回条数,默认8 | - diff --git a/src/Undefined/skills/toolsets/cognitive/get_profile/handler.py b/src/Undefined/skills/toolsets/cognitive/get_profile/handler.py index 1a126634..77fdfb2a 100644 --- a/src/Undefined/skills/toolsets/cognitive/get_profile/handler.py +++ b/src/Undefined/skills/toolsets/cognitive/get_profile/handler.py @@ -5,8 +5,11 @@ async def execute(args: dict[str, Any], context: dict[str, Any]) -> str: cognitive_service = context.get("cognitive_service") if not cognitive_service or not cognitive_service.enabled: return "认知记忆系统未启用" - profile = await cognitive_service.get_profile( - entity_type=args["entity_type"], - entity_id=args["entity_id"], - ) + try: + profile = await cognitive_service.get_profile( + entity_type=args["entity_type"], + entity_id=args["entity_id"], + ) + except ValueError as exc: + return f"侧写查询参数无效: {exc}" return profile if profile else "暂无该实体的侧写信息" diff --git a/tests/test_cognitive_profile_paths.py b/tests/test_cognitive_profile_paths.py new file mode 100644 index 00000000..6daf45d8 --- /dev/null +++ b/tests/test_cognitive_profile_paths.py @@ -0,0 +1,155 @@ +from __future__ import annotations + +import json +from pathlib import Path +from types import SimpleNamespace +from typing import Any + +import pytest +from aiohttp.test_utils import make_mocked_request + +from Undefined.api import RuntimeAPIContext +from Undefined.api.routes.cognitive import cognitive_profile_handler +from Undefined.cognitive.profile_storage import ProfileStorage +from Undefined.skills.toolsets.cognitive.get_profile.handler import execute + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("entity_type", "entity_id"), + [ + ("other", "10001"), + ("../user", "10001"), + ("user", ""), + ("user", " "), + ("user", "."), + ("user", ".."), + ("user", "../../outside"), + ("user", "sub/name"), + ("user", "sub\\name"), + ("user", "C:outside"), + ("user", "name\x00"), + ], +) +async def test_profile_operations_reject_invalid_entities( + tmp_path: Path, entity_type: str, entity_id: str +) -> None: + storage = ProfileStorage(tmp_path / "profiles") + outside = tmp_path / "outside.md" + outside.write_text("private", encoding="utf-8") + with pytest.raises(ValueError): + await storage.read_profile(entity_type, entity_id) + with pytest.raises(ValueError): + await storage.write_profile(entity_type, entity_id, "changed") + with pytest.raises(ValueError): + await storage.list_revisions(entity_type, entity_id) + with pytest.raises(ValueError): + await storage.read_revision(entity_type, entity_id, "revision.md") + with pytest.raises(ValueError): + await storage.restore_revision(entity_type, entity_id, "revision.md") + with pytest.raises(ValueError): + storage.merge_guard(entity_type, entity_id) + assert outside.read_text(encoding="utf-8") == "private" + assert not storage._locks + assert not storage._merge_locks + assert not (tmp_path / "profiles").exists() + + +@pytest.mark.asyncio +async def test_profile_operations_reject_absolute_id(tmp_path: Path) -> None: + outside = tmp_path / "outside.md" + outside.write_text("private", encoding="utf-8") + storage = ProfileStorage(tmp_path / "profiles") + entity_id = str(outside.with_suffix("")) + with pytest.raises(ValueError): + await storage.read_profile("user", entity_id) + with pytest.raises(ValueError): + await storage.write_profile("user", entity_id, "changed") + assert outside.read_text(encoding="utf-8") == "private" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("link_kind", ["directory", "file", "history", "revision"]) +async def test_profiles_reject_links_outside_storage( + tmp_path: Path, link_kind: str +) -> None: + base = tmp_path / "profiles" + outside = tmp_path / "outside" + outside.mkdir() + secret = outside / "10001.md" + secret.write_text("private", encoding="utf-8") + base.mkdir() + storage = ProfileStorage(base) + if link_kind == "directory": + (base / "users").symlink_to(outside, target_is_directory=True) + elif link_kind == "file": + (base / "users").mkdir() + (base / "users" / "10001.md").symlink_to(secret) + else: + history = base / "history" / "user" / "10001" + history.parent.mkdir(parents=True) + if link_kind == "history": + history.symlink_to(outside, target_is_directory=True) + else: + history.mkdir() + (history / "revision.md").symlink_to(secret) + + if link_kind in {"directory", "file"}: + with pytest.raises(ValueError): + await storage.read_profile("user", "10001") + with pytest.raises(ValueError): + await storage.write_profile("user", "10001", "changed") + else: + with pytest.raises(ValueError): + await storage.read_revision("user", "10001", "revision.md") + with pytest.raises(ValueError): + await storage.restore_revision("user", "10001", "revision.md") + if link_kind == "history": + with pytest.raises(ValueError): + await storage.list_revisions("user", "10001") + with pytest.raises(ValueError): + await storage.write_profile("user", "10001", "changed") + assert secret.read_text(encoding="utf-8") == "private" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("entity_type", ["user", "group"]) +async def test_profiles_keep_valid_non_numeric_ids( + tmp_path: Path, entity_type: str +) -> None: + storage = ProfileStorage(tmp_path) + await storage.write_profile(entity_type, "new-0", "v1") + await storage.write_profile(entity_type, "new-0", "v2") + revisions = await storage.list_revisions(entity_type, "new-0") + await storage.restore_revision(entity_type, "new-0", revisions[0]) + assert await storage.read_profile(entity_type, "new-0") == "v1" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("entity_type", ["user", "invalid"]) +async def test_profile_query_reports_invalid_parameters_without_reading_secret( + tmp_path: Path, entity_type: str +) -> None: + storage = ProfileStorage(tmp_path / "profiles") + (tmp_path / "outside.md").write_text("private", encoding="utf-8") + service = SimpleNamespace(enabled=True, get_profile=storage.read_profile) + context = RuntimeAPIContext( + config_getter=lambda: None, + onebot=None, + ai=None, + command_dispatcher=None, + queue_manager=None, + history_manager=None, + cognitive_service=service, + ) + args: dict[str, Any] = { + "entity_type": entity_type, + "entity_id": "../../outside", + } + request = make_mocked_request("GET", "/profile", match_info=args) + response = await cognitive_profile_handler(context, request) + assert response.status == 400 + assert "private" not in json.loads(response.text or "{}").get("error", "") + result = await execute(args, {"cognitive_service": service}) + assert result.startswith("侧写查询参数无效:") + assert "private" not in result From 0d72a78b9ee4344228751151f496566038f4f351 Mon Sep 17 00:00:00 2001 From: Null <1708213363@qq.com> Date: Tue, 6 Oct 2026 13:02:12 +0800 Subject: [PATCH 032/121] fix(knowledge): skip cancelled queued model requests --- docs/knowledge.md | 2 + src/Undefined/knowledge/embedder.py | 2 + src/Undefined/knowledge/reranker.py | 2 + tests/test_knowledge.py | 65 +++++++++++++++++++++++++++++ 4 files changed, 71 insertions(+) diff --git a/docs/knowledge.md b/docs/knowledge.md index b27be0fc..79a29d9d 100644 --- a/docs/knowledge.md +++ b/docs/knowledge.md @@ -194,6 +194,8 @@ chunk_size=4, chunk_overlap=1 → step=3 嵌入与重排请求都通过内置队列串行发送,按各自 `queue_interval_seconds` 控制发车间隔;embedding/rerank 默认值均为 `0.0`,非正数都表示请求到达立即发车。多行文本按 `embed_batch_size` 分批,每批一次 API 调用。 +调用方取消仍在排队的请求时,队列会在出队时跳过该请求,不调用模型,也不消耗发车间隔;已经发给模型的请求不在这个排队取消规则内。 + ``` texts → split_lines → [batch 1, batch 2, ...] → Queue → API (间隔发车) ``` diff --git a/src/Undefined/knowledge/embedder.py b/src/Undefined/knowledge/embedder.py index 29988bc0..cb929759 100644 --- a/src/Undefined/knowledge/embedder.py +++ b/src/Undefined/knowledge/embedder.py @@ -60,6 +60,8 @@ async def _process_loop(self) -> None: """站台/发车:按固定间隔从队列取任务并执行。""" while True: job = await self._queue.get() + if job.future.cancelled(): + continue try: result = await self._requester.embed(self._config, job.texts) if not job.future.done(): diff --git a/src/Undefined/knowledge/reranker.py b/src/Undefined/knowledge/reranker.py index a29e63f3..06276193 100644 --- a/src/Undefined/knowledge/reranker.py +++ b/src/Undefined/knowledge/reranker.py @@ -56,6 +56,8 @@ async def stop(self) -> None: async def _process_loop(self) -> None: while True: job = await self._queue.get() + if job.future.cancelled(): + continue try: result = await self._requester.rerank( model_config=self._config, diff --git a/tests/test_knowledge.py b/tests/test_knowledge.py index 137a6b73..e042358d 100644 --- a/tests/test_knowledge.py +++ b/tests/test_knowledge.py @@ -154,6 +154,71 @@ async def fake_rerank( assert sorted(order) == ["q1", "q2", "q3"] +@pytest.mark.parametrize("kind", ["embedding", "rerank"]) +async def test_cancelled_queued_requests_do_not_reach_provider(kind: str) -> None: + entered = asyncio.Event() + release = asyncio.Event() + calls: list[str] = [] + + async def record_call(value: str) -> None: + calls.append(value) + if value == "first": + entered.set() + await release.wait() + + async def fake_embed(config: Any, texts: list[str]) -> list[list[float]]: + await record_call(texts[0]) + return [[1.0]] + + async def fake_rerank( + model_config: Any, + query: str, + documents: list[str], + top_n: int | None = None, + ) -> list[dict[str, Any]]: + await record_call(query) + return [{"index": 0, "relevance_score": 1.0, "document": documents[0]}] + + worker: Embedder | Reranker + if kind == "embedding": + worker, requester = _make_embedder() + requester.embed = fake_embed + else: + worker, requester = _make_reranker() + requester.rerank = fake_rerank + + async def submit(value: str) -> Any: + if isinstance(worker, Embedder): + return await worker.embed([value]) + return await worker.rerank(value, [value]) + + worker.start() + tasks: list[asyncio.Task[Any]] = [] + try: + first = asyncio.create_task(submit("first")) + tasks.append(first) + await asyncio.wait_for(entered.wait(), timeout=2) + cancelled = asyncio.create_task(submit("cancelled")) + tasks.append(cancelled) + await asyncio.sleep(0) # 让第二个请求进入真实队列。 + assert worker._queue.qsize() == 1 + cancelled.cancel() + with pytest.raises(asyncio.CancelledError): + await cancelled + third = asyncio.create_task(submit("third")) + tasks.append(third) + release.set() + results = await asyncio.wait_for(asyncio.gather(first, third), timeout=2) + assert all(results) + assert calls == ["first", "third"] + finally: + release.set() + for task in tasks: + task.cancel() + await asyncio.gather(*tasks, return_exceptions=True) + await worker.stop() + + # ── manager ────────────────────────────────────────────────────────────────── From fafbe82458af034d8bb5b61f10ce7451e9ac1725 Mon Sep 17 00:00:00 2001 From: Null <1708213363@qq.com> Date: Tue, 6 Oct 2026 13:02:57 +0800 Subject: [PATCH 033/121] fix(scripts): synchronize restored profile search vectors --- docs/cognitive-memory.md | 13 +- scripts/README.md | 12 +- scripts/reembed_cognitive.py | 1 + scripts/restore_profile.py | 131 +++++++++++++-- tests/test_restore_profile_script.py | 229 +++++++++++++++++++++++++++ 5 files changed, 367 insertions(+), 19 deletions(-) create mode 100644 tests/test_restore_profile_script.py diff --git a/docs/cognitive-memory.md b/docs/cognitive-memory.md index d37b460e..d8d772ca 100644 --- a/docs/cognitive-memory.md +++ b/docs/cognitive-memory.md @@ -447,6 +447,8 @@ data/cognitive/ 更换嵌入模型(维度变化或模型升级)后,需要对向量库进行全量重嵌入。详见 [`scripts/reembed_cognitive.py`](../scripts/reembed_cognitive.py)。 +重嵌入读取 Chroma 中现有的检索正文,不从侧写 Markdown 重建内容。恢复历史侧写或补同步当前侧写时,使用下方的 `restore_profile.py restore` / `sync`。 + 向量维度发生变化时脚本先把新向量写入临时 collection,全部完成并校验后才删除旧库并换名(ChromaDB 定维后无法原地改维,直接 upsert 异维向量会失败)。建议先 `--dry-run` 确认维度变化与记录数:预览在系统临时目录的完整副本上读取,源 Chroma 文件和 collection 不会被修改,也不会清理或换名遗留的临时 collection;仍会调用嵌入 API,并可能记录 token 用量。请预留副本所需的临时磁盘空间,并按以下步骤停止写入后运行。 ```bash @@ -482,13 +484,18 @@ uv run python scripts/restore_profile.py list --entity-type user --entity-id {us # 预览某个版本内容(不改动文件) uv run python scripts/restore_profile.py show --entity-type user --entity-id {user_id} --revision {timestamp}.md -# 恢复该版本(先 dry-run 确认,再实际恢复) +# 停止机器人及其他写入侧写/向量库的进程后,先预览,再恢复并同步检索 uv run python scripts/restore_profile.py restore --entity-type user --entity-id {user_id} --revision {timestamp}.md --dry-run uv run python scripts/restore_profile.py restore --entity-type user --entity-id {user_id} --revision {timestamp}.md + +# 若向量同步失败,修复错误后从当前 Markdown 重试,不必再次恢复快照 +uv run python scripts/restore_profile.py sync --entity-type user --entity-id {user_id} --dry-run +uv run python scripts/restore_profile.py sync --entity-type user --entity-id {user_id} ``` -恢复只改侧写 Markdown 与历史快照,不会更新 ChromaDB 中的侧写向量;若同一实体在 -`cognitive_profiles` 里有旧向量,请按[更换嵌入模型](#更换嵌入模型)的方式重嵌入侧写。 +`restore` 默认同时恢复 Markdown、保存当前内容的历史快照,并用当前 cognitive 嵌入模型同步 `cognitive_profiles` 中该实体的检索正文和向量,其他实体保持不变。检索正文沿用运行时格式,包含恢复版本中的名称、标签、评价、锐评与正文。模型调用失败时尚未恢复文件;若文件已恢复而向量写入失败,命令会报错,可用 `sync` 从当前文件补同步。文件和向量库之间没有跨存储事务,请保持机器人及其他写入进程停止,直到同步成功。 + +`--dry-run` 只展示将恢复的内容、检索正文和两个目录,不写盘、不打开 Chroma、不调用模型。目录默认读取 `cognitive.profiles_path` 与 `cognitive.vector_store_path`;操作独立副本时同时指定 `--profiles-path` 和 `--db-path`。实际恢复/同步可能调用嵌入 API 并记录 token 用量;若已更换嵌入模型维度,请先按[更换嵌入模型](#更换嵌入模型)重建整个向量库,再恢复或同步单个侧写。 **级别 3:完整移除** diff --git a/scripts/README.md b/scripts/README.md index 98b514e0..1fd047f6 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -65,6 +65,8 @@ python scripts/sync_config_template.py --stdout **原理**:ChromaDB 存储了完整的原文本(`documents`),脚本读取所有记录,用新模型重新计算向量后 upsert 覆写,metadata 保持不变。 +脚本不重新读取侧写 Markdown,也不用于历史侧写恢复。恢复版本或从当前 Markdown 更新检索正文与向量,请使用下方的 `restore_profile.py restore` / `sync`。 + **维度变化**:collection 在首次写入时定维,异维 upsert 会直接报 `InvalidArgumentError`。脚本把新向量写入临时 collection,全部写完并校验记录数后才删除旧库、把临时库换名为正式库。实际运行会清理未完成的临时库,或在正式库缺失时恢复临时库。 `--dry-run` 先复制向量库到系统临时目录,仅在副本上打开 Chroma 客户端,源向量库的数据和文件均不修改。它不会创建缺失的 collection、删除残留临时库或把临时库换名;若只有临时库,会读取其中的记录预览恢复后的重嵌入。预览仍会调用嵌入 API 计算向量,并可能记录 token 用量;需要足够临时磁盘空间保存完整副本。运行前停止机器人及其他写入该库的进程,保证复制和正式迁移读取一致的数据。 @@ -107,14 +109,20 @@ uv run python scripts/restore_profile.py list --entity-type user --entity-id 123 # 查看某个历史版本内容 uv run python scripts/restore_profile.py show --entity-type user --entity-id 123456 --revision 20260101000000000000.md -# 恢复某个历史版本(恢复前会把当前内容另存为新快照,可再次回退) +# 停止机器人及其他写入进程后恢复,并同步该实体的检索正文与向量 uv run python scripts/restore_profile.py restore --entity-type user --entity-id 123456 --revision 20260101000000000000.md # 仅预览,不写盘 uv run python scripts/restore_profile.py restore --entity-type user --entity-id 123456 --revision 20260101000000000000.md --dry-run + +# 补同步当前 Markdown(例如恢复成功但向量写入失败后重试) +uv run python scripts/restore_profile.py sync --entity-type user --entity-id 123456 --dry-run +uv run python scripts/restore_profile.py sync --entity-type user --entity-id 123456 ``` -**注意**:恢复只改侧写 Markdown 与历史快照,不会更新 ChromaDB 中的侧写向量;需要同步检索结果时按 `docs/cognitive-memory.md` 的说明重嵌入。 +`restore` 默认恢复 Markdown、把当前内容另存为历史快照,并使用当前 cognitive 嵌入配置更新 `cognitive_profiles` 中该实体的检索正文与向量。`sync` 只从当前 Markdown 补同步检索,不创建快照。模型请求失败时文件保持原状;向量写入失败时命令报错,已恢复文件保留,可修复错误后重跑 `sync`,无需依赖可能已被清理的旧快照。文件与向量库没有跨存储事务,实际恢复/同步前需停止机器人和其他写入进程,成功后再启动。 + +`--dry-run` 展示文件内容、检索正文与路径,不打开 Chroma、不调用模型、不写盘。目录默认来自 `cognitive.profiles_path` 与 `cognitive.vector_store_path`;操作独立副本时同时传入 `--profiles-path` 和 `--db-path`。实际恢复/同步会调用嵌入 API,可能记录 token 用量。若新模型维度不同于现有库,先完成全库重嵌入再恢复/同步单个侧写。 ### release_notes.py — 发布版本校验与 Release notes 生成 diff --git a/scripts/reembed_cognitive.py b/scripts/reembed_cognitive.py index da368b88..48ea62f9 100755 --- a/scripts/reembed_cognitive.py +++ b/scripts/reembed_cognitive.py @@ -6,6 +6,7 @@ 原理:ChromaDB 存储了完整的原文本(documents),本脚本读取所有记录, 用新模型重新计算向量,然后通过 upsert 覆写回去。metadata 保持不变。 +它不会重新读取侧写 Markdown;历史恢复与当前文件同步使用 restore_profile.py。 维度变化:ChromaDB 的 collection 在首次写入时定维,异维向量 upsert 会直接失败 (InvalidArgumentError)。脚本会先比较新旧向量维度,检测到变化时把新向量全部 diff --git a/scripts/restore_profile.py b/scripts/restore_profile.py index c18993d4..b577b1c7 100644 --- a/scripts/restore_profile.py +++ b/scripts/restore_profile.py @@ -16,8 +16,9 @@ # 仅预览将要恢复的内容(不写盘) uv run python scripts/restore_profile.py restore --entity-type user --entity-id 123456 --revision 20260101000000000000.md --dry-run -恢复只改侧写 Markdown 文件与历史快照,不会重建 ChromaDB 中的侧写向量;如需同时刷新 -检索向量,请按 docs/cognitive-memory.md 的说明重嵌入。 +恢复同时更新侧写 Markdown、历史快照和该实体的 ChromaDB 检索正文与向量。 +若向量写入失败,可用 sync 动作从当前 Markdown 重试,不必再次恢复历史版本。 +实际恢复或同步前,应停止机器人及其他写入侧写/向量库的进程。 """ from __future__ import annotations @@ -27,15 +28,26 @@ import logging import sys from pathlib import Path +from typing import Any # 将项目 src 加入 sys.path,使脚本可以直接 uv run 执行 _PROJECT_ROOT = Path(__file__).resolve().parent.parent +_SCRIPTS_DIR = _PROJECT_ROOT / "scripts" +if str(_SCRIPTS_DIR) not in sys.path: + sys.path.insert(0, str(_SCRIPTS_DIR)) _SRC_DIR = _PROJECT_ROOT / "src" if str(_SRC_DIR) not in sys.path: sys.path.insert(0, str(_SRC_DIR)) from Undefined.cognitive.profile_storage import ProfileStorage # noqa: E402 +from Undefined.cognitive.service.helpers import ( # noqa: E402 + _build_profile_vector_payload, + _current_profile_name, + _normalize_profile_tags, + _parse_profile_markdown, +) from Undefined.config.loader import Config # noqa: E402 +from reembed_cognitive import _build_embedder, _open_chroma # noqa: E402 logger = logging.getLogger("restore_profile") @@ -46,8 +58,13 @@ def _parse_args() -> argparse.Namespace: ) parser.add_argument( "action", - choices=("list", "show", "restore"), - help="list=列出历史版本;show=查看版本内容;restore=恢复版本", + choices=("list", "show", "restore", "sync"), + help="list=列出历史版本;show=查看版本;restore=恢复并同步检索;sync=同步当前侧写检索", + ) + parser.add_argument( + "--db-path", + default="", + help="ChromaDB 目录(默认读取 config.toml 中的 cognitive.vector_store_path)", ) parser.add_argument( "--entity-type", @@ -86,6 +103,67 @@ def _print_revisions(revisions: list[str]) -> None: print(f"{index:>3}. {name}") +def _vector_payload( + content: str, entity_type: str, entity_id: str +) -> tuple[str, dict[str, Any]]: + """沿用运行时侧写格式;无 frontmatter 的旧文件仍可同步正文。""" + parsed = _parse_profile_markdown(content) + if parsed is None: + frontmatter: dict[str, Any] = {} + evaluation, summary, roast = "", content, "" + else: + frontmatter, evaluation, summary, roast = parsed + return _build_profile_vector_payload( + entity_type=entity_type, + entity_id=entity_id, + effective_name=_current_profile_name(entity_type, frontmatter), + tags=_normalize_profile_tags(frontmatter.get("tags")), + summary=summary, + evaluation=evaluation, + roast=roast, + ) + + +async def _restore_and_sync( + config: Config, + storage: ProfileStorage, + *, + entity_type: str, + entity_id: str, + revision: str | None, + document: str, + metadata: dict[str, Any], + db_path: Path, +) -> None: + embedder = _build_embedder(config) + try: + # 模型请求失败时,尚未改动侧写文件和向量库。 + embeddings = await embedder.embed([document]) + if len(embeddings) != 1 or not embeddings[0]: + raise RuntimeError("嵌入模型没有返回该侧写的有效向量,未恢复侧写") + async with _open_chroma(db_path, dry_run=False) as client: + collection = client.get_or_create_collection( + "cognitive_profiles", metadata={"hnsw:space": "cosine"} + ) + if revision is not None: + await storage.restore_revision(entity_type, entity_id, revision) + try: + await asyncio.to_thread( + collection.upsert, + ids=[f"{entity_type}:{entity_id}"], + documents=[document], + embeddings=embeddings, + metadatas=[metadata], + ) + except Exception as exc: + raise RuntimeError( + "侧写检索同步失败;当前 Markdown 已保留。修复向量库/模型配置后," + "使用 sync 动作重试当前侧写,无需再次恢复历史版本。" + ) from exc + finally: + await embedder.stop() + + async def _main(args: argparse.Namespace) -> None: config = Config.load(strict=False) base_path = args.profiles_path or config.cognitive.profiles_path @@ -113,22 +191,47 @@ async def _main(args: argparse.Namespace) -> None: print(content) return - # restore - content = await storage.read_revision(entity_type, entity_id, args.revision) + revision = args.revision if args.action == "restore" else None + content = ( + await storage.read_revision(entity_type, entity_id, revision) + if revision is not None + else await storage.read_profile(entity_type, entity_id) + ) if content is None: - logger.error("历史版本不存在: %s:%s/%s", entity_type, entity_id, args.revision) + logger.error( + "侧写或历史版本不存在: %s:%s/%s", entity_type, entity_id, revision or "当前" + ) sys.exit(1) + document, metadata = _vector_payload(content, entity_type, entity_id) + db_path = Path(args.db_path or config.cognitive.vector_store_path) + print(f"侧写目录: {base_path}") + print(f"ChromaDB 目录: {db_path}") if args.dry_run: - print( - f"[dry-run] 将把 {entity_type}:{entity_id} 恢复为 {args.revision}," - f"当前内容会先存为新快照。" - ) + if revision is not None: + print( + f"[dry-run] 将把 {entity_type}:{entity_id} 恢复为 {revision}," + "当前内容会先存为新快照。" + ) + print(f"[dry-run] 将同步 cognitive_profiles 中的 {entity_type}:{entity_id}") print("---- 恢复后的内容 ----") print(content) + print("---- 检索正文 ----") + print(document) return - restored = await storage.restore_revision(entity_type, entity_id, args.revision) - print(f"已恢复 {entity_type}:{entity_id} 到历史版本 {restored}") - print("当前内容已另存为新快照,可再次用 list / restore 回退。") + await _restore_and_sync( + config, + storage, + entity_type=entity_type, + entity_id=entity_id, + revision=revision, + document=document, + metadata=metadata, + db_path=db_path, + ) + if revision is not None: + print(f"已恢复 {entity_type}:{entity_id} 到历史版本 {revision}") + print("当前内容已另存为新快照,可再次用 list / restore 回退。") + print(f"已同步 {entity_type}:{entity_id} 的检索正文与向量。") def main() -> None: diff --git a/tests/test_restore_profile_script.py b/tests/test_restore_profile_script.py new file mode 100644 index 00000000..72eca75d --- /dev/null +++ b/tests/test_restore_profile_script.py @@ -0,0 +1,229 @@ +from __future__ import annotations + +import argparse +import importlib.util +from pathlib import Path +from types import ModuleType +from typing import Any + +import chromadb +import pytest + +from Undefined.cognitive.profile_storage import ProfileStorage +from Undefined.cognitive.service.helpers import _serialize_profile_markdown +from Undefined.config.loader import Config + + +def _load_script() -> ModuleType: + path = Path(__file__).resolve().parent.parent / "scripts" / "restore_profile.py" + spec = importlib.util.spec_from_file_location("restore_profile_script", path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class _Embedder: + def __init__(self, *, fail: bool = False) -> None: + self.fail = fail + self.calls: list[list[str]] = [] + self.stopped = False + + async def embed(self, documents: list[str]) -> list[list[float]]: + self.calls.append(documents) + if self.fail: + raise RuntimeError("embedding unavailable") + return [[1.0, 0.0, 0.0] for _ in documents] + + async def stop(self) -> None: + self.stopped = True + + +def _args( + revision: str, *, action: str = "restore", dry_run: bool = False +) -> argparse.Namespace: + return argparse.Namespace( + action=action, + entity_type="user", + entity_id="123", + revision=revision, + profiles_path="", + db_path="", + dry_run=dry_run, + ) + + +async def _seed_profiles(tmp_path: Path) -> tuple[Config, ProfileStorage, str, str]: + config = Config.from_mapping( + { + "cognitive": { + "vector_store": {"path": str(tmp_path / "chroma")}, + "profile": {"path": str(tmp_path / "profiles"), "revision_keep": 1}, + } + }, + strict=False, + ) + assert Path(config.cognitive.profiles_path).resolve().is_relative_to(tmp_path) + assert Path(config.cognitive.vector_store_path).resolve().is_relative_to(tmp_path) + assert ( + (Path(config.cognitive.profiles_path) / "history") + .resolve() + .is_relative_to(tmp_path) + ) + storage = ProfileStorage(config.cognitive.profiles_path, revision_keep=1) + recovered = _serialize_profile_markdown( + { + "entity_type": "user", + "entity_id": "123", + "nickname": "恢复的名称", + "tags": ["原标签"], + }, + "GOOD recovered profile", + evaluation="恢复的评价", + roast="恢复的锐评", + ) + await storage.write_profile("user", "123", recovered) + await storage.write_profile("user", "123", "BAD mistaken profile") + revision = (await storage.list_revisions("user", "123"))[0] + return config, storage, revision, recovered + + +def _seed_vectors(config: Config, tmp_path: Path) -> Any: + assert Path(config.cognitive.vector_store_path).resolve().is_relative_to(tmp_path) + client: Any = chromadb.PersistentClient(path=config.cognitive.vector_store_path) + collection = client.get_or_create_collection( + "cognitive_profiles", metadata={"hnsw:space": "cosine"} + ) + collection.upsert( + ids=["user:123", "group:999"], + documents=["BAD mistaken profile", "unrelated profile"], + embeddings=[[0.0, 1.0, 0.0], [0.0, 0.0, 1.0]], + metadatas=[ + {"entity_type": "user", "entity_id": "123", "nickname": "错误名称"}, + {"entity_type": "group", "entity_id": "999"}, + ], + ) + return collection + + +def _files(path: Path) -> dict[Path, bytes]: + return { + file.relative_to(path): file.read_bytes() + for file in path.rglob("*") + if file.is_file() + } + + +@pytest.fixture(autouse=True) +def _isolate_cwd(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.chdir(tmp_path) + + +@pytest.mark.asyncio +async def test_restore_syncs_actual_markdown_and_vectors_without_touching_other_entities( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + module = _load_script() + config, storage, revision, recovered = await _seed_profiles(tmp_path) + collection = _seed_vectors(config, tmp_path) + embedder = _Embedder() + monkeypatch.setattr(module.Config, "load", lambda **kwargs: config) + monkeypatch.setattr(module, "_build_embedder", lambda cfg: embedder) + + await module._main(_args(revision)) + + assert await storage.read_profile("user", "123") == recovered + result = collection.get( + ids=["user:123"], include=["documents", "embeddings", "metadatas"] + ) + document = result["documents"][0] + assert "GOOD recovered profile" in document + assert "BAD mistaken profile" not in document + assert "恢复的评价" in document and "恢复的锐评" in document + assert "原标签" in document + assert embedder.calls == [[document]] + assert list(result["embeddings"][0]) == [1.0, 0.0, 0.0] + assert result["metadatas"][0]["nickname"] == "恢复的名称" + other = collection.get(ids=["group:999"], include=["documents", "embeddings"]) + assert other["documents"] == ["unrelated profile"] + assert list(other["embeddings"][0]) == [0.0, 0.0, 1.0] + assert embedder.stopped + + +@pytest.mark.asyncio +async def test_restore_dry_run_does_not_open_chroma_or_call_model( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + module = _load_script() + config, storage, revision, recovered = await _seed_profiles(tmp_path) + before = _files(tmp_path) + monkeypatch.setattr(module.Config, "load", lambda **kwargs: config) + + def forbidden(*args: Any, **kwargs: Any) -> Any: + raise AssertionError("dry-run must not construct a client or model") + + monkeypatch.setattr(module, "_open_chroma", forbidden) + monkeypatch.setattr(module, "_build_embedder", forbidden) + await module._main(_args(revision, dry_run=True)) + + assert _files(tmp_path) == before + assert await storage.read_profile("user", "123") == "BAD mistaken profile" + assert recovered in capsys.readouterr().out + assert not Path(config.cognitive.vector_store_path).exists() + + +@pytest.mark.asyncio +async def test_embedding_failure_leaves_files_and_vectors_unchanged( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + module = _load_script() + config, _, revision, _ = await _seed_profiles(tmp_path) + _seed_vectors(config, tmp_path) + before = _files(tmp_path) + embedder = _Embedder(fail=True) + monkeypatch.setattr(module.Config, "load", lambda **kwargs: config) + monkeypatch.setattr(module, "_build_embedder", lambda cfg: embedder) + + with pytest.raises(RuntimeError, match="embedding unavailable"): + await module._main(_args(revision)) + + assert _files(tmp_path) == before + assert embedder.stopped + + +@pytest.mark.asyncio +async def test_sync_retries_vector_failure_after_revision_was_pruned( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + module = _load_script() + config, storage, revision, recovered = await _seed_profiles(tmp_path) + collection = _seed_vectors(config, tmp_path) + embedder = _Embedder() + monkeypatch.setattr(module.Config, "load", lambda **kwargs: config) + monkeypatch.setattr(module, "_build_embedder", lambda cfg: embedder) + original_upsert = type(collection).upsert + fail_next = True + + def upsert_once(self: Any, *args: Any, **kwargs: Any) -> None: + nonlocal fail_next + if fail_next: + fail_next = False + raise RuntimeError("Chroma temporarily unavailable") + original_upsert(self, *args, **kwargs) + + monkeypatch.setattr(type(collection), "upsert", upsert_once) + with pytest.raises(RuntimeError, match="使用 sync 动作重试"): + await module._main(_args(revision)) + + assert "已同步" not in capsys.readouterr().out + assert await storage.read_profile("user", "123") == recovered + assert revision not in await storage.list_revisions("user", "123") + assert collection.get(ids=["user:123"])["documents"] == ["BAD mistaken profile"] + history_before = _files(tmp_path / "profiles" / "history") + + await module._main(_args("", action="sync")) + + result = collection.get(ids=["user:123"], include=["documents", "embeddings"]) + assert "GOOD recovered profile" in result["documents"][0] + assert list(result["embeddings"][0]) == [1.0, 0.0, 0.0] + assert _files(tmp_path / "profiles" / "history") == history_before From 7e4f80803137fff81b987d9e6d8ab49d47c6ada9 Mon Sep 17 00:00:00 2001 From: Null <1708213363@qq.com> Date: Tue, 6 Oct 2026 13:04:12 +0800 Subject: [PATCH 034/121] fix(bilibili): settle download threads before cancelled cleanup --- src/Undefined/bilibili/downloader.py | 22 ++-- .../skills/tools/bilibili_video/README.md | 2 + tests/test_bilibili_downloader_adapter.py | 103 ++++++++++++++++++ 3 files changed, 118 insertions(+), 9 deletions(-) diff --git a/src/Undefined/bilibili/downloader.py b/src/Undefined/bilibili/downloader.py index ffacff78..b2faa80a 100644 --- a/src/Undefined/bilibili/downloader.py +++ b/src/Undefined/bilibili/downloader.py @@ -13,6 +13,7 @@ from Undefined.bilibili.download_core import QUALITY_MAP as QUALITY_MAP from Undefined.bilibili.download_core import download_video as download_video_core from Undefined.bilibili.models import VideoInfo as VideoInfo +from Undefined.utils import io from Undefined.utils.paths import DOWNLOAD_CACHE_DIR, ensure_dir logger = logging.getLogger(__name__) @@ -114,13 +115,15 @@ async def download_video( work_dir = ensure_dir(output_dir / uuid.uuid4().hex) try: - downloaded_path, video_info, actual_qn = await asyncio.to_thread( - partial( - _download_video_sync, - bvid, - work_dir=work_dir, - cookie=cookie, - prefer_quality=prefer_quality, + downloaded_path, video_info, actual_qn = await io.run_cancellation_safe( + asyncio.to_thread( + partial( + _download_video_sync, + bvid, + work_dir=work_dir, + cookie=cookie, + prefer_quality=prefer_quality, + ) ) ) @@ -131,8 +134,9 @@ async def download_video( actual_qn, ) return downloaded_path, video_info, actual_qn - except Exception: - _cleanup_dir(work_dir) + except BaseException: + # 下载线程已收敛,取消/失败时在释放所有权前清理,避免线程稍后重建产物。 + await io.run_cancellation_safe(asyncio.to_thread(_cleanup_dir, work_dir)) raise diff --git a/src/Undefined/skills/tools/bilibili_video/README.md b/src/Undefined/skills/tools/bilibili_video/README.md index d89aa7f1..bdb0df5e 100644 --- a/src/Undefined/skills/tools/bilibili_video/README.md +++ b/src/Undefined/skills/tools/bilibili_video/README.md @@ -33,6 +33,8 @@ 配置依赖: - `config.toml` 中的 `[bilibili]` 段控制清晰度、时长限制、文件大小限制等 +下载超时或取消时,会等待正在运行的下载/合并线程完成,再清理整个工作目录并传播取消;等待期间不会发送下载结果。工具超时或停机收尾因此可能超过触发取消的时间点。正常下载完成后,产物由发送或附件注册流程接管并清理;自动提取和图文嵌套视频共用这一下载路径。 + 自动提取行为: - 自动处理管线命中 B 站链接、BV 号或 AV 号后,会发送一次外层合并转发,包含视频信息、视频文件或视频状态、弹幕列表三个节点。 - 弹幕通过 Bilibili protobuf 接口分段拉取;弹幕列表会按每 100 条弹幕拆成一个内层合并转发。 diff --git a/tests/test_bilibili_downloader_adapter.py b/tests/test_bilibili_downloader_adapter.py index fdc7a401..eba083c1 100644 --- a/tests/test_bilibili_downloader_adapter.py +++ b/tests/test_bilibili_downloader_adapter.py @@ -1,12 +1,18 @@ from __future__ import annotations +import asyncio from pathlib import Path +import threading from types import TracebackType +from types import SimpleNamespace +from typing import Any, cast +from unittest.mock import AsyncMock import pytest from Undefined.bilibili import downloader from Undefined.bilibili.models import DownloadResult, VideoInfo, VideoStats +from Undefined.bilibili.sender import send_bilibili_video @pytest.mark.asyncio @@ -181,3 +187,100 @@ def _fake_download_core( assert called["bvid"] == "BV1xx411c7mD" assert called["prefer_quality"] == 64 assert called["overwrite"] is True + + +@pytest.mark.asyncio +@pytest.mark.parametrize("via_sender", [False, True]) +async def test_cancelled_download_waits_for_thread_then_cleans_artifacts( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, via_sender: bool +) -> None: + started = threading.Event() + release = threading.Event() + finished = threading.Event() + sender = SimpleNamespace(send_group_forward_message=AsyncMock()) + + def download_sync( + bvid: str, *, work_dir: Path, cookie: str, prefer_quality: int + ) -> tuple[Path, VideoInfo, int]: + (work_dir / "stream.part").write_bytes(b"partial") + started.set() + try: + if not release.wait(timeout=5.0): + raise TimeoutError("test download was not released") + output = work_dir / "video.mp4" + output.write_bytes(b"finished after caller cancellation") + info = VideoInfo( + bvid=bvid, + aid=1, + title="test", + duration=1, + cover_url="", + up_name="test", + desc="", + cid=1, + page_duration=1, + stats=VideoStats(), + ) + return output, info, prefer_quality + finally: + finished.set() + + monkeypatch.setattr(downloader, "_download_video_sync", download_sync) + monkeypatch.setattr(downloader, "DOWNLOAD_CACHE_DIR", tmp_path) + if via_sender: + monkeypatch.setattr( + "Undefined.bilibili.sender.normalize_to_bvid", + AsyncMock(return_value="BV1xx411c7mD"), + ) + task: asyncio.Task[Any] = asyncio.create_task( + send_bilibili_video( + "BV1xx411c7mD", + cast(Any, sender), + cast(Any, SimpleNamespace()), + "group", + 1, + max_duration=0, + danmaku_enabled=False, + ) + ) + else: + task = asyncio.create_task( + downloader.download_video("BV1xx411c7mD", output_dir=tmp_path) + ) + try: + assert await asyncio.to_thread(started.wait, 2.0) + task.cancel() + await asyncio.sleep(0) + task.cancel() + await asyncio.sleep(0) + assert not task.done() + assert not finished.is_set() + assert list(tmp_path.rglob("stream.part")) + release.set() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(task, timeout=2.0) + assert finished.is_set() + assert list(tmp_path.iterdir()) == [] + sender.send_group_forward_message.assert_not_awaited() + finally: + release.set() + await asyncio.gather(task, return_exceptions=True) + + +@pytest.mark.asyncio +async def test_failed_download_cleans_partial_artifacts_and_preserves_error( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + error = RuntimeError("download failed") + + def download_sync( + bvid: str, *, work_dir: Path, cookie: str, prefer_quality: int + ) -> tuple[Path, VideoInfo, int]: + (work_dir / "stream.part").write_bytes(b"partial") + raise error + + monkeypatch.setattr(downloader, "_download_video_sync", download_sync) + with pytest.raises(RuntimeError) as exc_info: + await downloader.download_video("BV1xx411c7mD", output_dir=tmp_path) + assert exc_info.value is error + assert list(tmp_path.iterdir()) == [] From 55a8c963cb4eed38e5a979c1660587ddde4f7d6e Mon Sep 17 00:00:00 2001 From: Null <1708213363@qq.com> Date: Tue, 6 Oct 2026 13:04:41 +0800 Subject: [PATCH 035/121] fix(runtime): drain background workers before closing model queues --- docs/deployment.md | 2 +- src/Undefined/main.py | 40 ++++++++++++---- tests/test_main_shutdown.py | 96 ++++++++++++++++++++++++++++++++++++- 3 files changed, 127 insertions(+), 11 deletions(-) diff --git a/docs/deployment.md b/docs/deployment.md index 6b5d47c7..ead48817 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -156,7 +156,7 @@ uv run Undefined-webui #### 优雅停机 -`SIGINT`(Ctrl+C)与 `SIGTERM` 都会触发同一套优雅停机流程:停止 OneBot 连接、关闭 Runtime API 与微信服务、停止史官 / memes worker、断开连接、关闭 AI 客户端与检索运行时、停止配置热重载并释放渲染资源。容器、systemd、supervisor 等默认以 `SIGTERM` 停止进程,无需额外配置;请避免使用 `SIGKILL`(`docker kill -s KILL`),否则会跳过全部落盘清理。 +`SIGINT`(Ctrl+C)与 `SIGTERM` 都会触发同一套优雅停机流程:停止 OneBot 连接、关闭 Runtime API 与微信服务、停止史官 / memes worker,再由消息处理器排空并关闭 AI 队列、落盘历史,最后断开连接、关闭 AI 客户端与检索运行时、停止配置热重载并释放渲染资源。等待 worker 收尾期间模型队列保持可用,史官在向量写入后继续提交侧写模型请求也能完成;消息处理器收尾时新落盘的认知任务留待下次启动处理。容器、systemd、supervisor 等默认以 `SIGTERM` 停止进程,无需额外配置;请避免使用 `SIGKILL`(`docker kill -s KILL`),否则会跳过全部落盘清理。 #### 自动启动选项 diff --git a/src/Undefined/main.py b/src/Undefined/main.py index 4535e56f..ec269b34 100644 --- a/src/Undefined/main.py +++ b/src/Undefined/main.py @@ -1,11 +1,13 @@ """程序入口""" +from __future__ import annotations + import asyncio import logging import signal import time import sys -from typing import Any +from typing import TYPE_CHECKING, Any from logging.handlers import RotatingFileHandler from pathlib import Path @@ -40,6 +42,10 @@ from Undefined.render import close_browser as close_render_browser from Undefined.utils.render_cache import close_render_cache +if TYPE_CHECKING: + from Undefined.cognitive import HistorianWorker + from Undefined.memes import MemeWorker + def ensure_runtime_dirs() -> None: """确保运行时目录存在""" @@ -528,14 +534,12 @@ def _apply_config_updates( await weixin_service.stop() except Exception: logger.exception("[清理] WeixinService stop 失败") - try: - await handler.close() - except Exception: - logger.exception("[清理] MessageHandler close 失败") - if meme_worker is not None: - await meme_worker.stop() - if historian_worker: - await historian_worker.stop() + await _close_message_workers( + handler, + meme_worker=meme_worker, + historian_worker=historian_worker, + logger=logger, + ) await onebot.disconnect() await ai.close() if retrieval_registry is not None: @@ -547,6 +551,24 @@ def _apply_config_updates( logger.info("[退出] 机器人已停止运行") +async def _close_message_workers( + handler: MessageHandler, + *, + meme_worker: MemeWorker | None, + historian_worker: HistorianWorker | None, + logger: logging.Logger, +) -> None: + """先收敛仍会提交模型调用的 worker,再由 handler 排空并关闭队列。""" + if meme_worker is not None: + await meme_worker.stop() + if historian_worker is not None: + await historian_worker.stop() + try: + await handler.close() + except Exception: + logger.exception("[清理] MessageHandler close 失败") + + class ShutdownSignalGuard: """优雅停机信号注册的句柄:携带停机事件并支持恢复安装前的信号状态。""" diff --git a/tests/test_main_shutdown.py b/tests/test_main_shutdown.py index ff2fdb07..e1015c0e 100644 --- a/tests/test_main_shutdown.py +++ b/tests/test_main_shutdown.py @@ -4,13 +4,107 @@ import logging import os import signal +from types import SimpleNamespace +from typing import Any, cast +from unittest.mock import AsyncMock import pytest -from Undefined.main import _run_until_shutdown, install_shutdown_signal_handlers +from Undefined.ai import AIClient +from Undefined.cognitive import HistorianWorker +from Undefined.handlers import MessageHandler +from Undefined.main import ( + _close_message_workers, + _run_until_shutdown, + install_shutdown_signal_handlers, +) +from Undefined.services.queue_manager import QueueManager logger = logging.getLogger("test-main-shutdown") +@pytest.mark.asyncio +async def test_shutdown_keeps_model_queue_alive_between_historian_phases() -> None: + queue = QueueManager(ai_request_interval=0, max_retries=0) + ai = AIClient.__new__(AIClient) + ai._queue_manager = queue + ai._pending_llm_calls = {} + ai.runtime_config = cast(Any, SimpleNamespace(ai_request_max_retries=0)) + model = SimpleNamespace(model_name="historian", max_tokens=100) + calls: list[str] = [] + + async def execute_model(request: dict[str, Any]) -> None: + calls.append(request["call_type"]) + ai.set_llm_call_result(request["request_id"], {"ok": True}) + + queue.start(execute_model) + between_phases = asyncio.Event() + release_io = asyncio.Event() + + async def phase() -> None: + await ai.submit_background_llm_call(model, [], call_type="rewrite") + between_phases.set() + await release_io.wait() # 向量/磁盘阶段完成后还需要一次模型调用。 + await ai.submit_background_llm_call(model, [], call_type="merge") + + worker = HistorianWorker( + job_queue=SimpleNamespace( + recover_stale=AsyncMock(), list_jobs=AsyncMock(return_value=[]) + ), + vector_store=None, + profile_storage=None, + ai_client=ai, + config_getter=lambda: SimpleNamespace( + poll_interval_seconds=1, stale_job_timeout_seconds=60 + ), + ) + await worker.start() + phase_task = asyncio.create_task(phase()) + worker._inflight_tasks.add(phase_task) + handler = MessageHandler.__new__(MessageHandler) + handler._background_tasks = set() + cast(Any, handler).pipeline_registry = SimpleNamespace(stop_hot_reload=AsyncMock()) + cast(Any, handler).ai_coordinator = SimpleNamespace(queue_manager=queue) + flush_batches = AsyncMock() + flush_history = AsyncMock() + cast(Any, handler).message_batcher = SimpleNamespace(flush_all=flush_batches) + cast(Any, handler).history_manager = SimpleNamespace( + flush_pending_saves=flush_history + ) + meme = SimpleNamespace(stop=AsyncMock()) + close_task: asyncio.Task[None] | None = None + try: + await asyncio.wait_for(between_phases.wait(), timeout=2) + close_task = asyncio.create_task( + _close_message_workers( + handler, + meme_worker=cast(Any, meme), + historian_worker=worker, + logger=logger, + ) + ) + await asyncio.sleep(0) + assert not close_task.done() + assert queue._processor_tasks + assert calls == ["rewrite"] + flush_batches.assert_not_awaited() + release_io.set() + await asyncio.wait_for(close_task, timeout=2) + assert calls == ["rewrite", "merge"] + assert not ai._pending_llm_calls + assert not queue._processor_tasks + meme.stop.assert_awaited_once() + flush_history.assert_awaited_once() + finally: + release_io.set() + phase_task.cancel() + if close_task is not None: + close_task.cancel() + await asyncio.gather(close_task, return_exceptions=True) + await asyncio.gather(phase_task, return_exceptions=True) + await worker.stop() + await queue.stop() + + class _SlowOneBot: def __init__(self) -> None: self.cancelled = False From f7b07893be045755d456aa27c573535f3f23bde0 Mon Sep 17 00:00:00 2001 From: Null <1708213363@qq.com> Date: Tue, 6 Oct 2026 13:05:42 +0800 Subject: [PATCH 036/121] fix(deploy): preserve distinct configuration backups --- docs/docker-deploy.md | 2 + src/Undefined/deploy/config_patch.py | 18 ++++++- tests/test_deploy_config_patch.py | 77 ++++++++++++++++++++++++++++ 3 files changed, 95 insertions(+), 2 deletions(-) diff --git a/docs/docker-deploy.md b/docs/docker-deploy.md index c7498c20..ff1b38c4 100644 --- a/docs/docker-deploy.md +++ b/docs/docker-deploy.md @@ -251,6 +251,8 @@ uv run deploy up --pull always | `deploy/backup/` | 部署工具修改 `config.toml` 前的备份 | | `deploy/searxng/`、`deploy/firecrawl/`、`deploy/lxmusic2api/` | 已选服务的配置,以及音乐服务的音源、数据和下载文件 | +每次实际修改已有 `config.toml` 前,部署工具会保存独立备份,文件名为 `config__<唯一后缀>.toml`。同一秒内的多次更新也会保留各次修改前的内容;命令输出会显示本次备份的完整路径。配置无变更或 `--dry-run` 时不创建备份。 + 备份时保存 `config.toml`、`deploy/` 和你使用的自定义目录。Firecrawl 等服务还使用 Docker 命名卷,备份这些服务时需另行备份对应卷;仅复制 `deploy/` 不能保存全部数据库数据。 普通停止使用 `uv run deploy down` 即可。**`down --volumes` 会删除 Compose 管理的数据卷;`down --purge` 还会删除整个 `deploy/`,包括登录态、凭据、配置备份和容器模式下的 Bot 数据,不可恢复。** diff --git a/src/Undefined/deploy/config_patch.py b/src/Undefined/deploy/config_patch.py index 2bec12cf..295c66c2 100644 --- a/src/Undefined/deploy/config_patch.py +++ b/src/Undefined/deploy/config_patch.py @@ -16,6 +16,8 @@ from __future__ import annotations import tomllib +import os +import tempfile from dataclasses import dataclass, field from pathlib import Path from typing import Any @@ -207,9 +209,21 @@ def apply_plan( backup_path: Path | None = None if plan.config_path.is_file(): + original = plan.config_path.read_text(encoding="utf-8") backup_dir.mkdir(parents=True, exist_ok=True) - backup_path = backup_dir / f"{BACKUP_FILE_PREFIX}{utc_stamp_compact()}.toml" - write_text(backup_path, plan.config_path.read_text(encoding="utf-8")) + # 独占创建预留名称;秒级时间戳相同或并发备份也不会覆盖旧文件。 + descriptor, name = tempfile.mkstemp( + prefix=f"{BACKUP_FILE_PREFIX}{utc_stamp_compact()}_", + suffix=".toml", + dir=backup_dir, + ) + os.close(descriptor) + backup_path = Path(name) + try: + write_text(backup_path, original) + except BaseException: + backup_path.unlink(missing_ok=True) + raise write_text(plan.config_path, rendered) return PatchOutcome( diff --git a/tests/test_deploy_config_patch.py b/tests/test_deploy_config_patch.py index e10362c5..bc763576 100644 --- a/tests/test_deploy_config_patch.py +++ b/tests/test_deploy_config_patch.py @@ -3,6 +3,7 @@ from __future__ import annotations import tomllib +from concurrent.futures import ThreadPoolExecutor from pathlib import Path import pytest @@ -233,6 +234,82 @@ def test_apply_is_noop_when_nothing_changes( assert not backup_dir.exists() +def test_same_second_updates_keep_each_previous_config( + config_dir: tuple[Path, Path, Path], monkeypatch: pytest.MonkeyPatch +) -> None: + config, example, backup_dir = config_dir + monkeypatch.chdir(config.parent) + assert backup_dir.resolve().is_relative_to(config.parent) + monkeypatch.setattr(config_patch, "utc_stamp_compact", lambda: "20261006T123456Z") + original = config.read_text(encoding="utf-8") + first = config_patch.apply_plan( + _plan(config, {"webui.port": 8788}), example_path=example, backup_dir=backup_dir + ) + intermediate = config.read_text(encoding="utf-8") + second = config_patch.apply_plan( + _plan(config, {"webui.port": 8789}), example_path=example, backup_dir=backup_dir + ) + + assert first.backup_path is not None and second.backup_path is not None + assert first.backup_path != second.backup_path + assert first.backup_path.read_text(encoding="utf-8") == original + assert second.backup_path.read_text(encoding="utf-8") == intermediate + assert len(list(backup_dir.glob("config_*.toml"))) == 2 + assert config_patch.load_toml(config)["webui"]["port"] == 8789 + + +def test_concurrent_backups_reserve_distinct_names( + config_dir: tuple[Path, Path, Path], monkeypatch: pytest.MonkeyPatch +) -> None: + config, example, backup_dir = config_dir + monkeypatch.chdir(config.parent) + assert backup_dir.resolve().is_relative_to(config.parent) + other = config.with_name("other.toml") + other.write_text(EXAMPLE.replace("8787", "8790"), encoding="utf-8") + originals = {path.read_text(encoding="utf-8") for path in (config, other)} + monkeypatch.setattr(config_patch, "utc_stamp_compact", lambda: "20261006T123456Z") + + def apply(path: Path) -> config_patch.PatchOutcome: + return config_patch.apply_plan( + _plan(path, {"webui.port": 8788}), + example_path=example, + backup_dir=backup_dir, + ) + + with ThreadPoolExecutor(max_workers=2) as executor: + futures = [executor.submit(apply, path) for path in (config, other)] + outcomes = [future.result() for future in futures] + + paths = {outcome.backup_path for outcome in outcomes} + assert None not in paths + assert len(paths) == 2 + assert { + path.read_text(encoding="utf-8") for path in paths if path is not None + } == originals + + +def test_failed_backup_keeps_config_and_removes_empty_reserved_file( + config_dir: tuple[Path, Path, Path], monkeypatch: pytest.MonkeyPatch +) -> None: + config, example, backup_dir = config_dir + monkeypatch.chdir(config.parent) + assert backup_dir.resolve().is_relative_to(config.parent) + original = config.read_bytes() + + def fail_write(path: Path, content: str) -> None: + raise OSError("backup write failed") + + monkeypatch.setattr(config_patch, "write_text", fail_write) + with pytest.raises(OSError, match="backup write failed"): + config_patch.apply_plan( + _plan(config, {"webui.port": 8788}), + example_path=example, + backup_dir=backup_dir, + ) + assert config.read_bytes() == original + assert list(backup_dir.glob("config_*.toml")) == [] + + def test_apply_is_idempotent(config_dir: tuple[Path, Path, Path]) -> None: config, example, backup_dir = config_dir plan = _plan(config, {"onebot.ws_url": "ws://napcat:3001"}) From bde8ee83cf928a2ef538b18d2992c373e5d9e6d2 Mon Sep 17 00:00:00 2001 From: Null <1708213363@qq.com> Date: Tue, 6 Oct 2026 13:08:06 +0800 Subject: [PATCH 037/121] fix(chat): isolate runtime state and stale asynchronous responses --- apps/undefined-chat/src/App.test.tsx | 111 +++++++++ apps/undefined-chat/src/App.tsx | 19 +- .../src/chat-store/runtime-scope.test.ts | 217 ++++++++++++++++++ apps/undefined-chat/src/chat-store/store.ts | 98 +++++++- .../rendering/AttachmentImageContext.test.tsx | 58 +++++ .../src/rendering/AttachmentImageContext.tsx | 36 +-- docs/undefined-chat.md | 2 + 7 files changed, 518 insertions(+), 23 deletions(-) create mode 100644 apps/undefined-chat/src/chat-store/runtime-scope.test.ts diff --git a/apps/undefined-chat/src/App.test.tsx b/apps/undefined-chat/src/App.test.tsx index d0b519a9..1758373f 100644 --- a/apps/undefined-chat/src/App.test.tsx +++ b/apps/undefined-chat/src/App.test.tsx @@ -1,5 +1,6 @@ import { open } from "@tauri-apps/plugin-dialog"; import { + act, fireEvent, render, screen, @@ -12,6 +13,10 @@ import { beforeEach, describe, expect, test, vi } from "vitest"; import { App } from "./App"; import { LanguageProvider, LOCALE_STORAGE_KEY } from "./i18n"; import { createTauriRuntimeClient } from "./runtime-client/tauri"; +import type { + AttachmentPreviewResult, + RuntimeConfig, +} from "./runtime-client/types"; import { conversation, historyItem, runtimeClientStub } from "./test-fixtures"; vi.mock("./runtime-client/tauri", () => ({ @@ -30,6 +35,14 @@ function renderApp(ui: ReactElement = ) { return render({ui}); } +async function connectRuntime(runtimeUrl: string): Promise { + fireEvent.keyDown(window, { key: ",", ctrlKey: true }); + await userEvent.clear(screen.getByLabelText("Runtime URL")); + await userEvent.type(screen.getByLabelText("Runtime URL"), runtimeUrl); + await userEvent.click(screen.getByRole("button", { name: "保存并连接" })); + await screen.findByText(runtimeUrl); +} + describe("App", () => { beforeEach(() => { vi.resetAllMocks(); @@ -425,6 +438,104 @@ describe("App", () => { }); }); + test("ignores a file picker opened on the previous Runtime", async () => { + let resolvePicker!: (path: string) => void; + vi.mocked(open).mockImplementationOnce( + () => + new Promise((resolve) => { + resolvePicker = resolve; + }), + ); + let config: RuntimeConfig = { + runtimeUrl: "http://runtime-a.test", + hasApiKey: true, + }; + const client = runtimeClientStub({ + getRuntimeConfig: vi.fn(async () => config), + saveRuntimeConfig: vi.fn(async (runtimeUrl) => { + config = { ...config, runtimeUrl }; + return config; + }), + }); + vi.mocked(createTauriRuntimeClient).mockReturnValue(client); + renderApp(); + await screen.findByLabelText("消息输入"); + await userEvent.click(screen.getByRole("button", { name: "添加附件" })); + await connectRuntime("http://runtime-b.test"); + await waitFor(() => { + expect(client.getHistory).toHaveBeenCalledTimes(2); + }); + resolvePicker("/tmp/runtime-a-private.txt"); + await waitFor(() => { + expect(screen.getByLabelText("消息输入")).toHaveValue(""); + }); + expect(client.uploadAttachment).not.toHaveBeenCalled(); + }); + + test("ignores a direct attachment preview returned from the previous Runtime", async () => { + let resolvePreview!: (result: AttachmentPreviewResult) => void; + let config: RuntimeConfig = { + runtimeUrl: "http://runtime-a.test", + hasApiKey: true, + }; + const client = runtimeClientStub(); + const getHistory = client.getHistory; + client.getRuntimeConfig = vi.fn(async () => config); + client.saveRuntimeConfig = vi.fn(async (runtimeUrl) => { + config = { ...config, runtimeUrl }; + return config; + }); + client.getHistory = vi.fn(async (request) => ({ + ...(await getHistory(request)), + items: + config.runtimeUrl === "http://runtime-a.test" + ? [ + historyItem({ + attachments: [ + { + id: "pic_a", + name: "private-a.png", + kind: "image", + mediaType: "image/png", + size: 20 * 1024 * 1024, + downloadUrl: "/api/v1/chat/attachments/pic_a", + previewUrl: "/api/v1/chat/attachments/pic_a/preview", + discarded: false, + }, + ], + }), + ] + : [], + })); + client.previewAttachment = vi.fn( + () => + new Promise((resolve) => { + resolvePreview = resolve; + }), + ); + vi.mocked(createTauriRuntimeClient).mockReturnValue(client); + const createSpy = vi.spyOn(URL, "createObjectURL"); + renderApp(); + await screen.findByText("private-a.png"); + await userEvent.click(screen.getByRole("button", { name: "预览" })); + await connectRuntime("http://runtime-b.test"); + await waitFor(() => { + expect(screen.queryByText("private-a.png")).toBeNull(); + }); + await act(async () => { + resolvePreview({ + status: 200, + ok: true, + mediaType: "image/png", + bytes: [1, 2, 3, 4], + body: null, + }); + }); + expect(screen.queryByRole("dialog", { name: "图片查看器" })).toBeNull(); + expect(createSpy).not.toHaveBeenCalled(); + createSpy.mockRestore(); + }); + test("uses native attachment bridges for timeline actions", async () => { const client = runtimeClientStub({ getHistory: vi.fn(async () => ({ diff --git a/apps/undefined-chat/src/App.tsx b/apps/undefined-chat/src/App.tsx index ceee074c..375b3e42 100644 --- a/apps/undefined-chat/src/App.tsx +++ b/apps/undefined-chat/src/App.tsx @@ -129,7 +129,7 @@ export function App() { ): Promise { setSetupError(null); try { - await client.saveRuntimeConfig(url); + const config = await client.saveRuntimeConfig(url); if (allowInsecure) { await client.confirmInsecureStorageFallback(); } @@ -138,7 +138,7 @@ export function App() { await client.saveApiKey(apiKey); } setIsSettingsOpen(false); - await store.bootstrap(); + await store.bootstrap({ runtimeUrl: config.runtimeUrl }); } catch (err) { setSetupError(err instanceof Error ? err.message : String(err)); } @@ -165,10 +165,12 @@ export function App() { } async function previewAttachment(attachment: Attachment): Promise { + const runtimeConfig = store.getSnapshot().runtimeConfig; try { const result = await client.previewAttachment({ attachmentId: attachment.id, }); + if (store.getSnapshot().runtimeConfig !== runtimeConfig) return; if (!result.ok || result.bytes.length === 0) { window.alert(result.body ?? t("attachment.previewFailed")); return; @@ -188,11 +190,13 @@ export function App() { window.open(url, "_blank", "noopener,noreferrer"); window.setTimeout(() => URL.revokeObjectURL(url), 60_000); } catch (err) { + if (store.getSnapshot().runtimeConfig !== runtimeConfig) return; window.alert(err instanceof Error ? err.message : String(err)); } } async function addAttachment(conversationId: string): Promise { + const runtimeConfig = store.getSnapshot().runtimeConfig; try { const selected = await open({ multiple: false, @@ -201,10 +205,12 @@ export function App() { pickerMode: "document", fileAccessMode: "copy", }); + if (store.getSnapshot().runtimeConfig !== runtimeConfig) return; if (typeof selected === "string") { await store.addAttachmentPath(conversationId, selected); } } catch (err) { + if (store.getSnapshot().runtimeConfig !== runtimeConfig) return; window.alert(err instanceof Error ? err.message : String(err)); } } @@ -215,7 +221,9 @@ export function App() { return; } // 尚无会话:先自动创建再把示例填入草稿 + const runtimeConfig = store.getSnapshot().runtimeConfig; void store.createConversation().then(() => { + if (store.getSnapshot().runtimeConfig !== runtimeConfig) return; const conversationId = store.getSnapshot().selectedConversationId; if (conversationId) { store.updateDraft(conversationId, prompt); @@ -293,7 +301,7 @@ export function App() { previewImageUrlRef.current = null; } }; - }, []); + }, [state.runtimeConfig?.runtimeUrl]); useEffect(() => { if (!isMobile || !isMobileSidebarActive) { @@ -460,7 +468,10 @@ export function App() { ]); return ( - +
{/* 移动端侧边栏背景遮罩 */}
({ invoke: vi.fn() })); +vi.mock("@tauri-apps/api/event", () => ({ listen: vi.fn() })); + +function deferred() { + let resolve!: (value: T) => void; + let reject!: (reason: Error) => void; + const promise = new Promise((resolvePromise, rejectPromise) => { + resolve = resolvePromise; + reject = rejectPromise; + }); + return { promise, resolve, reject }; +} + +describe("Runtime instance isolation", () => { + beforeEach(() => vi.resetAllMocks()); + + test("a new Runtime sharing the default conversation sends only its own editor state through Tauri", async () => { + const id = "legacy-system-42"; + let runtimeUrl = "http://runtime-a.example"; + let invokedOrigin = ""; + vi.mocked(invoke).mockImplementation(async () => { + invokedOrigin = runtimeUrl; + return { + status: 202, + ok: true, + body: { job_id: "new", conversation_id: id }, + }; + }); + const client = runtimeClientStub({ + getRuntimeConfig: vi.fn(async () => ({ runtimeUrl, hasApiKey: true })), + listConversations: vi.fn(async () => ({ + conversations: [conversation({ id })], + activeJob: null, + defaultConversationId: id, + virtualUserId: "webchat", + })), + getHistory: vi.fn(async () => ({ + conversationId: id, + virtualUserId: "webchat", + permission: "superadmin", + count: 1, + limit: 50, + before: null, + hasMore: false, + nextBefore: null, + total: 1, + items: [ + historyItem({ + messageId: runtimeUrl, + role: "bot", + content: runtimeUrl, + }), + ], + })), + sendMessage: createTauriRuntimeClient().sendMessage, + }); + const store = createChatStore({ client }); + await store.bootstrap(); + store.updateDraft(id, "A private draft"); + store.addReferenceFromMessageId(id, runtimeUrl); + await store.addAttachmentPath(id, "/mock/a-private.pdf"); + store.dispatch({ type: "autoScroll/set", enabled: false }); + runtimeUrl = "http://runtime-b.example"; + await store.bootstrap({ runtimeUrl }); + const state = store.getSnapshot(); + expect(state.draftsByConversation).toEqual({}); + expect(state.referencesByConversation).toEqual({}); + expect(state.attachmentsByConversation).toEqual({}); + expect(state.historyByConversation[id].items[0].content).toBe(runtimeUrl); + expect(state.autoScrollEnabled).toBe(false); + store.updateDraft(id, "B request"); + await store.sendSelectedMessage(); + expect(invokedOrigin).toBe(runtimeUrl); + expect(invoke).toHaveBeenCalledWith("send_message", { + input: { + conversationId: id, + message: { text: "B request", attachment_ids: [], references: [] }, + }, + }); + }); + + test("reconnecting to the same Runtime preserves drafts, references, and ready attachments", async () => { + const client = runtimeClientStub(); + const store = createChatStore({ client }); + await store.bootstrap(); + store.updateDraft("default", "keep draft"); + store.addReference("default", { messageId: "a", quote: "keep quote" }); + await store.addAttachmentPath("default", "/mock/a.pdf"); + const before = store.getSnapshot(); + await store.bootstrap({ + runtimeUrl: before.runtimeConfig?.runtimeUrl, + resuming: true, + }); + const after = store.getSnapshot(); + expect(after.draftsByConversation).toEqual(before.draftsByConversation); + expect(after.referencesByConversation).toEqual( + before.referencesByConversation, + ); + expect(after.attachmentsByConversation).toEqual( + before.attachmentsByConversation, + ); + }); + + test("old history, uploads, and failed sends cannot repopulate a replacement Runtime", async () => { + let runtimeUrl = "http://runtime-a.example"; + const client = runtimeClientStub({ + getRuntimeConfig: vi.fn(async () => ({ runtimeUrl, hasApiKey: true })), + }); + const store = createChatStore({ client }); + await store.bootstrap(); + const history = deferred(); + const upload = deferred(); + const send = deferred>(); + vi.mocked(client.getHistory).mockImplementationOnce(() => history.promise); + vi.mocked(client.uploadAttachment).mockImplementationOnce( + () => upload.promise, + ); + vi.mocked(client.sendMessage).mockImplementationOnce(() => send.promise); + store.updateDraft("default", "A pending send"); + const oldSend = store.sendSelectedMessage(); + const oldUpload = store.addAttachmentPath("default", "/mock/a.pdf"); + const oldHistory = store.reloadHistory("default"); + runtimeUrl = "http://runtime-b.example"; + await store.bootstrap({ runtimeUrl }); + store.updateDraft("default", "B new draft"); + history.resolve({ + conversationId: "default", + virtualUserId: "webchat", + permission: "superadmin", + count: 1, + limit: 50, + before: null, + hasMore: false, + nextBefore: null, + total: 1, + items: [historyItem({ content: "A late history" })], + }); + upload.resolve({ + id: "A-uid", + name: "a.pdf", + size: 1, + mediaType: "application/pdf", + kind: "file", + downloadUrl: "", + previewUrl: null, + discarded: false, + }); + send.reject(new Error("A late send failure")); + await Promise.all([oldHistory, oldUpload, oldSend]); + expect(store.getSnapshot().draftsByConversation.default).toBe( + "B new draft", + ); + expect(store.getSnapshot().attachmentsByConversation).toEqual({}); + expect(store.getSnapshot().historyByConversation.default.items).toEqual([]); + expect(store.getSnapshot().sendError).toBeNull(); + await store.sendSelectedMessage(); + expect(client.sendMessage).toHaveBeenLastCalledWith({ + conversationId: "default", + message: { text: "B new draft", attachmentIds: [], references: [] }, + }); + }); + + test("a replacement Runtime preempts an old bootstrap and ignores its late SSE callback", async () => { + let runtimeUrl = "http://runtime-a.example"; + const history = deferred(); + const client = runtimeClientStub({ + getRuntimeConfig: vi.fn(async () => ({ runtimeUrl, hasApiKey: true })), + }); + vi.mocked(client.getHistory).mockImplementationOnce(() => history.promise); + const store = createChatStore({ client }); + const oldBootstrap = store.bootstrap(); + await vi.waitFor(() => expect(client.getHistory).toHaveBeenCalledTimes(1)); + const oldStatus = vi.mocked(client.listenRuntimeSse).mock.calls[0][1]; + runtimeUrl = "http://runtime-b.example"; + const newJob = job(); + vi.mocked(client.getActiveJobs).mockResolvedValue({ + job: newJob, + jobs: [newJob], + }); + await store.bootstrap({ runtimeUrl }); + oldStatus({ + subscriptionId: "old", + jobId: newJob.jobId, + status: "closed", + detail: "late", + }); + expect(client.fetchJobEventsJson).not.toHaveBeenCalled(); + history.resolve({ + conversationId: "default", + virtualUserId: "webchat", + permission: "superadmin", + count: 1, + limit: 50, + before: null, + hasMore: false, + nextBefore: null, + total: 1, + items: [historyItem({ content: "A late bootstrap" })], + }); + await oldBootstrap; + expect(store.getSnapshot().runtimeConfig?.runtimeUrl).toBe(runtimeUrl); + expect(store.getSnapshot().historyByConversation.default.items).toEqual([]); + expect(store.getSnapshot().connectionState).toBe("streaming"); + }); +}); diff --git a/apps/undefined-chat/src/chat-store/store.ts b/apps/undefined-chat/src/chat-store/store.ts index b62ff717..45c029c4 100644 --- a/apps/undefined-chat/src/chat-store/store.ts +++ b/apps/undefined-chat/src/chat-store/store.ts @@ -26,6 +26,7 @@ export type { export type ChatStore = { bootstrap: (opts?: { + runtimeUrl?: string; preserveSelectionId?: string; resuming?: boolean; }) => Promise; @@ -669,6 +670,8 @@ export function createChatStore({ client: RuntimeClient; }): ChatStore { let state = createInitialChatState(); + let runtimeUrl: string | null = null; + let runtimeGeneration = 0; const listeners = new Set<() => void>(); const subscriptionsByJob = new Map(); const fallbackTimersByJob = new Map>(); @@ -711,15 +714,32 @@ export function createChatStore({ } fallbackTimersByJob.clear(); latestFallbackRequestByJob.clear(); + fallbackBackoffByJob.clear(); + } + + function resetRuntimeScope(nextUrl: string): void { + runtimeUrl = nextUrl; + runtimeGeneration += 1; + clearNativeSubscriptions(); + sendInFlight = false; + lastCommandsFetchedAt = 0; + state = { + ...createInitialChatState(), + autoScrollEnabled: state.autoScrollEnabled, + platform: state.platform, + settings: state.settings, + }; } async function loadHistory(conversationId: string): Promise { + const generation = runtimeGeneration; dispatch({ type: "history/loading", conversationId }); try { const history = await client.getHistory({ conversationId, limit: HISTORY_LIMIT, }); + if (generation !== runtimeGeneration) return; dispatch({ type: "history/set", conversationId, @@ -729,6 +749,7 @@ export function createChatStore({ total: history.total, }); } catch (err) { + if (generation !== runtimeGeneration) return; // 单会话历史加载失败降级为会话级错误,不升级为全局致命错误 dispatch({ type: "history/error", @@ -739,6 +760,7 @@ export function createChatStore({ } async function startStreamForJob(job: ChatJob): Promise { + const generation = runtimeGeneration; if (subscriptionsByJob.has(job.jobId)) { return; } @@ -748,10 +770,17 @@ export function createChatStore({ afterSeq: state.eventCursorByJob[job.jobId] ?? job.lastSeq, conversationId: job.conversationId, }); + if (generation !== runtimeGeneration) { + if (subscription.subscriptionId) { + void client.stopJobEventStream(subscription.subscriptionId); + } + return; + } if (subscription.subscriptionId) { subscriptionsByJob.set(job.jobId, subscription.subscriptionId); } } catch { + if (generation !== runtimeGeneration) return; // job 已建立;订阅失败只切换传输方式,不能回滚成功发送的输入。 fetchJobEventsFallback(job.jobId); } @@ -800,7 +829,9 @@ export function createChatStore({ if (fallbackTimersByJob.has(jobId)) { return; } + const generation = runtimeGeneration; const timer = setTimeout(() => { + if (generation !== runtimeGeneration) return; fallbackTimersByJob.delete(jobId); fetchJobEventsFallback(jobId); }, delayMs ?? JSON_FALLBACK_BASE_MS); @@ -808,6 +839,7 @@ export function createChatStore({ } function fetchJobEventsFallback(jobId: string): void { + const generation = runtimeGeneration; const conversationId = state.jobConversationById[jobId]; if ( !conversationId || @@ -818,6 +850,7 @@ export function createChatStore({ const requestGeneration = ++fallbackRequestGeneration; latestFallbackRequestByJob.set(jobId, requestGeneration); const isCurrentRequest = (): boolean => + generation === runtimeGeneration && latestFallbackRequestByJob.get(jobId) === requestGeneration && state.activeJobsByConversation[conversationId]?.jobId === jobId; const timer = fallbackTimersByJob.get(jobId); @@ -892,6 +925,7 @@ export function createChatStore({ preserveSelectionId?: string, resuming = false, ): Promise { + let generation = runtimeGeneration; dispatch({ type: "bootstrap/start" }); // Android 切后台返回等续接场景:标记为 resuming,连接成功后由后续状态覆盖 if (resuming) { @@ -899,6 +933,15 @@ export function createChatStore({ } try { const runtimeConfig = await client.getRuntimeConfig(); + if (generation !== runtimeGeneration) return; + if ( + runtimeConfig?.runtimeUrl && + runtimeConfig.runtimeUrl !== runtimeUrl + ) { + resetRuntimeScope(runtimeConfig.runtimeUrl); + generation = runtimeGeneration; + dispatch({ type: "bootstrap/start" }); + } if (!runtimeConfig?.runtimeUrl || !runtimeConfig.hasApiKey) { dispatch({ type: "bootstrap/error", @@ -915,6 +958,7 @@ export function createChatStore({ client.getActiveJobs(), client.listCommands(), ]); + if (generation !== runtimeGeneration) return; lastCommandsFetchedAt = Date.now(); // 仅选取真实存在于会话列表中的会话:defaultConversationId 是后端常量 // (legacy-system-42),未必对应已存在的会话,盲目用它会触发 getHistory 404 @@ -944,16 +988,27 @@ export function createChatStore({ commands: commands.commands, }); clearNativeSubscriptions(); - unlistenRuntimeSse = await client.listenRuntimeSse( - handleRuntimeEvent, - handleRuntimeStatus, + const unlisten = await client.listenRuntimeSse( + (event) => { + if (generation === runtimeGeneration) handleRuntimeEvent(event); + }, + (status) => { + if (generation === runtimeGeneration) handleRuntimeStatus(status); + }, ); + if (generation !== runtimeGeneration) { + unlisten(); + return; + } + unlistenRuntimeSse = unlisten; for (const activeJob of activeJobs.jobs) { + if (generation !== runtimeGeneration) return; await startStreamForJob(activeJob); } if (selected) { await loadHistory(selected); } + if (generation !== runtimeGeneration) return; if ( activeJobs.jobs.length > 0 && state.connectionState !== "json_fallback" @@ -961,14 +1016,20 @@ export function createChatStore({ dispatch({ type: "connection/set", connectionState: "streaming" }); } } catch (err) { + if (generation !== runtimeGeneration) return; dispatch({ type: "bootstrap/error", error: errorMessage(err) }); } } async function bootstrap(opts?: { + runtimeUrl?: string; preserveSelectionId?: string; resuming?: boolean; }): Promise { + if (opts?.runtimeUrl && opts.runtimeUrl !== runtimeUrl) { + resetRuntimeScope(opts.runtimeUrl); + inFlightBootstrap = null; + } // 重入保护:App 挂载与 Android resume 可能并发触发 bootstrap, // 复用进行中的 promise,避免重复 clearNativeSubscriptions / listenRuntimeSse 造成监听泄漏 if (inFlightBootstrap) { @@ -976,7 +1037,7 @@ export function createChatStore({ } const run = runBootstrap(opts?.preserveSelectionId, opts?.resuming).finally( () => { - inFlightBootstrap = null; + if (inFlightBootstrap === run) inFlightBootstrap = null; }, ); inFlightBootstrap = run; @@ -988,6 +1049,7 @@ export function createChatStore({ * 使 Skills 热重载新增的命令可见。未连接或未配置时跳过。 */ async function refreshCommandsIfStale(): Promise { + const generation = runtimeGeneration; if (!state.runtimeConfig?.hasApiKey || state.bootstrapping) { return; } @@ -996,6 +1058,7 @@ export function createChatStore({ } try { const commands = await client.listCommands(); + if (generation !== runtimeGeneration) return; lastCommandsFetchedAt = Date.now(); dispatch({ type: "commands/set", commands: commands.commands }); } catch { @@ -1004,19 +1067,25 @@ export function createChatStore({ } async function createConversation(title?: string): Promise { + const generation = runtimeGeneration; dispatch({ type: "conversation/creating", creating: true }); try { const conversation = await client.createConversation(title); + if (generation !== runtimeGeneration) return; dispatch({ type: "conversation/upsert", conversation }); await loadHistory(conversation.id); } catch (err) { + if (generation !== runtimeGeneration) return; dispatch({ type: "send/error", error: errorMessage(err) }); } finally { - dispatch({ type: "conversation/creating", creating: false }); + if (generation === runtimeGeneration) { + dispatch({ type: "conversation/creating", creating: false }); + } } } async function deleteConversation(conversationId: string): Promise { + const generation = runtimeGeneration; const wasSelected = state.selectedConversationId === conversationId; const nextSelectedId = wasSelected ? (state.conversations.find((item) => item.id !== conversationId)?.id ?? @@ -1024,6 +1093,7 @@ export function createChatStore({ : state.selectedConversationId; try { await client.deleteConversation(conversationId); + if (generation !== runtimeGeneration) return; dispatch({ type: "conversation/remove", conversationId, nextSelectedId }); if ( wasSelected && @@ -1033,6 +1103,7 @@ export function createChatStore({ await loadHistory(nextSelectedId); } } catch (err) { + if (generation !== runtimeGeneration) return; dispatch({ type: "send/error", error: errorMessage(err) }); } } @@ -1041,8 +1112,10 @@ export function createChatStore({ conversationId: string, newTitle: string, ): Promise { + const generation = runtimeGeneration; try { await client.renameConversation(conversationId, newTitle); + if (generation !== runtimeGeneration) return; // 更新本地 state 中的会话标题 const conversation = state.conversations.find( (item) => item.id === conversationId, @@ -1054,6 +1127,7 @@ export function createChatStore({ }); } } catch (err) { + if (generation !== runtimeGeneration) return; dispatch({ type: "send/error", error: errorMessage(err) }); } } @@ -1073,6 +1147,7 @@ export function createChatStore({ } async function sendSelectedMessage(): Promise { + const generation = runtimeGeneration; // in-flight 锁:从点击到 job 建立期间阻止重复发送, // 避免快速双击重复插入乐观消息 / 重复请求后端 if (sendInFlight) { @@ -1142,10 +1217,12 @@ export function createChatStore({ try { dispatch({ type: "send/error", error: null }); const job = await client.sendMessage(input); + if (generation !== runtimeGeneration) return; dispatch({ type: "job/upsert", job }); dispatch({ type: "connection/set", connectionState: "streaming" }); await startStreamForJob(job); } catch (err) { + if (generation !== runtimeGeneration) return; // 发送失败:移除乐观用户消息并恢复草稿/附件/引用 const currentHistory = state.historyByConversation[conversationId]; if (currentHistory) { @@ -1165,17 +1242,19 @@ export function createChatStore({ dispatch({ type: "references/set", conversationId, references }); dispatch({ type: "send/error", error: errorMessage(err) }); } finally { - sendInFlight = false; + if (generation === runtimeGeneration) sendInFlight = false; } } async function cancelJob(jobId: string): Promise { + const generation = runtimeGeneration; const conversationId = state.jobConversationById[jobId]; if (!conversationId) { return; } try { const cancelled = await client.cancelJob(jobId); + if (generation !== runtimeGeneration) return; const current = state.activeJobsByConversation[conversationId]; if (current?.jobId !== jobId) { return; @@ -1195,6 +1274,7 @@ export function createChatStore({ finishJob(jobId, conversationId); } } catch (err) { + if (generation !== runtimeGeneration) return; if (state.activeJobsByConversation[conversationId]?.jobId !== jobId) { return; } @@ -1206,6 +1286,7 @@ export function createChatStore({ conversationId: string, filePath: string, ): Promise { + const generation = runtimeGeneration; const trimmed = filePath.trim(); if (!trimmed) { return; @@ -1224,6 +1305,7 @@ export function createChatStore({ dispatch({ type: "attachments/set", conversationId, attachments: queued }); try { const attachment = await client.uploadAttachment({ filePath: trimmed }); + if (generation !== runtimeGeneration) return; const current = state.attachmentsByConversation[conversationId] ?? []; dispatch({ type: "attachments/set", @@ -1241,6 +1323,7 @@ export function createChatStore({ ), }); } catch (err) { + if (generation !== runtimeGeneration) return; const current = state.attachmentsByConversation[conversationId] ?? []; dispatch({ type: "attachments/set", @@ -1552,6 +1635,7 @@ export function createChatStore({ } async function loadMoreHistory(conversationId: string): Promise { + const generation = runtimeGeneration; const historyState = state.historyByConversation[conversationId]; if (!historyState?.hasMore || historyState.loading) { return; @@ -1565,6 +1649,7 @@ export function createChatStore({ historyState.nextBefore ?? undefined, HISTORY_LIMIT, ); + if (generation !== runtimeGeneration) return; const newItems = [...response.items, ...historyState.items]; @@ -1577,6 +1662,7 @@ export function createChatStore({ total: response.total, }); } catch (error) { + if (generation !== runtimeGeneration) return; dispatch({ type: "history/error", conversationId, diff --git a/apps/undefined-chat/src/rendering/AttachmentImageContext.test.tsx b/apps/undefined-chat/src/rendering/AttachmentImageContext.test.tsx index adf67af6..7b97fd67 100644 --- a/apps/undefined-chat/src/rendering/AttachmentImageContext.test.tsx +++ b/apps/undefined-chat/src/rendering/AttachmentImageContext.test.tsx @@ -130,4 +130,62 @@ describe("AttachmentImageContext", () => { expect(revokeSpy).toHaveBeenCalledTimes(2); revokeSpy.mockRestore(); }); + + it("Provider 卸载后迟到的预览不再创建或缓存旧实例 blob", async () => { + let resolve!: (result: AttachmentPreviewResult) => void; + const createSpy = vi.spyOn(URL, "createObjectURL"); + const { client } = makeClient( + () => + new Promise((done) => { + resolve = done; + }), + ); + const { result, unmount } = renderHook(() => useAttachmentImage(), { + wrapper: makeWrapper(client), + }); + const pending = result.current.loadAttachmentBlob("pic_old"); + unmount(); + resolve(imageResult()); + expect(await pending).toBeNull(); + expect(createSpy).not.toHaveBeenCalled(); + createSpy.mockRestore(); + }); + + it("切换 Runtime 释放旧缓存,相同 UID 在新实例重新加载并拒绝旧回包", async () => { + let resolve!: (result: AttachmentPreviewResult) => void; + let runtimeUrl = "http://runtime-a.test"; + const { client, previewAttachment } = makeClient(); + previewAttachment.mockImplementationOnce( + () => + new Promise((done) => { + resolve = done; + }), + ); + function Wrapper({ children }: { children: ReactNode }) { + return ( + + {children} + + ); + } + const { result, rerender } = renderHook(() => useAttachmentImage(), { + wrapper: Wrapper, + }); + const pending = result.current.loadAttachmentBlob("pic_pending"); + const cached = await result.current.loadAttachmentBlob("pic_shared"); + const revokeSpy = vi.spyOn(URL, "revokeObjectURL"); + runtimeUrl = "http://runtime-b.test"; + rerender(); + expect(revokeSpy).toHaveBeenCalledWith(cached?.url); + const current = await result.current.loadAttachmentBlob("pic_shared"); + expect(current?.url).not.toBe(cached?.url); + const createSpy = vi.spyOn(URL, "createObjectURL"); + resolve(imageResult()); + expect(await pending).toBeNull(); + expect(createSpy).not.toHaveBeenCalled(); + expect(await result.current.loadAttachmentBlob("pic_shared")).toBe(current); + expect(previewAttachment).toHaveBeenCalledTimes(3); + revokeSpy.mockRestore(); + createSpy.mockRestore(); + }); }); diff --git a/apps/undefined-chat/src/rendering/AttachmentImageContext.tsx b/apps/undefined-chat/src/rendering/AttachmentImageContext.tsx index b41a55e0..c8484434 100644 --- a/apps/undefined-chat/src/rendering/AttachmentImageContext.tsx +++ b/apps/undefined-chat/src/rendering/AttachmentImageContext.tsx @@ -5,7 +5,6 @@ import { useContext, useEffect, useMemo, - useRef, } from "react"; import type { RuntimeClient } from "../runtime-client/types"; @@ -26,6 +25,8 @@ const DEFAULT_MAX_CACHE_SIZE = 60; export type AttachmentImageProviderProps = { /** 仅依赖 previewAttachment,便于测试以最小桩注入 */ client: Pick; + /** 按 Runtime URL 隔离 UID 缓存,保持消费组件与 DOM 的生命周期。 */ + runtimeUrl?: string | null; /** blob 缓存上限,超出按 LRU 淘汰并 revoke。默认 60。 */ maxCacheSize?: number; children: ReactNode; @@ -41,23 +42,28 @@ export type AttachmentImageProviderProps = { * - LRU 缓存 uid→blobUrl:缩略图/正文图/全屏大图复用同一 blob,避免重复拉取。 * - inflight 去重:同一 UID 并发请求只发一次。 * - blob URL 归本 Provider 所有:消费组件**不**自行 revoke(缩略图与大图共享同一 - * URL,组件卸载即 revoke 会让仍打开的大图破图),仅在 LRU 淘汰或 Provider 卸载 - * 时 revoke。 + * URL,组件卸载即 revoke 会让仍打开的大图破图),仅在 LRU 淘汰、切换 Runtime + * 或 Provider 卸载时 revoke。 */ export function AttachmentImageProvider({ client, + runtimeUrl, maxCacheSize = DEFAULT_MAX_CACHE_SIZE, children, }: AttachmentImageProviderProps) { - const cacheRef = useRef>(new Map()); - const inflightRef = useRef>>( - new Map(), + const scope = useMemo( + () => ({ + runtimeUrl, + cache: new Map(), + inflight: new Map>(), + generation: 0, + }), + [runtimeUrl], ); const loadAttachmentBlob = useCallback( (uid: string): Promise => { - const cache = cacheRef.current; - const inflight = inflightRef.current; + const { cache, inflight } = scope; const cached = cache.get(uid); if (cached) { @@ -71,8 +77,10 @@ export function AttachmentImageProvider({ if (pending) return pending; const promise = (async (): Promise => { + const generation = scope.generation; try { const result = await client.previewAttachment({ attachmentId: uid }); + if (generation !== scope.generation) return null; if (!result.ok || result.bytes.length === 0) { return null; } @@ -97,26 +105,28 @@ export function AttachmentImageProvider({ } catch { return null; } finally { - inflight.delete(uid); + if (generation === scope.generation) inflight.delete(uid); } })(); inflight.set(uid, promise); return promise; }, - [client, maxCacheSize], + [client, maxCacheSize, scope], ); - // Provider 卸载时释放全部 blob URL + // 切换 Runtime 或卸载时释放全部 blob URL,拒绝旧作用域迟到的响应。 useEffect(() => { - const cache = cacheRef.current; + const { cache, inflight } = scope; return () => { + scope.generation += 1; + inflight.clear(); for (const { url } of cache.values()) { URL.revokeObjectURL(url); } cache.clear(); }; - }, []); + }, [scope]); const value = useMemo( () => ({ loadAttachmentBlob }), diff --git a/docs/undefined-chat.md b/docs/undefined-chat.md index b05ebe0d..d6b4de95 100644 --- a/docs/undefined-chat.md +++ b/docs/undefined-chat.md @@ -50,6 +50,8 @@ Undefined Chat 不持久化权威聊天数据。以下内容都以 Runtime 返 本地只保存 UI 相关状态,例如当前会话、草稿、事件游标、窗口布局、连接档案和是否允许不安全存储降级。刷新、重启或换端之后,客户端必须重新从 Runtime 恢复会话和任务状态。 +连接到不同 Runtime 时,会清理旧实例的草稿、附件 UID、消息引用、历史、任务和图片预览缓存,并丢弃旧实例迟到的请求或订阅通知,避免相同会话 ID 造成跨实例误发送。同一 Runtime 的重连及 Android 恢复保留编辑状态和 UI 偏好;自动滚动、平台与布局偏好在切换实例时也保留。 + ## 连接与事件 Runtime 请求必须由 Tauri command 拼接到已配置 origin 下,避免 React 传入任意 URL 后绕过目标主机限制。React 不直接持有 API Key;受保护请求由 Rust 侧注入 `X-Undefined-API-Key`。 From 36e2ab1a2deca57fde3349b2c902ca880404c154 Mon Sep 17 00:00:00 2001 From: Null <1708213363@qq.com> Date: Tue, 6 Oct 2026 13:10:20 +0800 Subject: [PATCH 038/121] fix(scripts): retry incomplete version synchronization --- scripts/README.md | 1 + scripts/bump_version.py | 32 ++++-- tests/test_bump_version_script.py | 176 ++++++++++++++++++++++++++++++ 3 files changed, 200 insertions(+), 9 deletions(-) diff --git a/scripts/README.md b/scripts/README.md index 1fd047f6..aaca81e9 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -179,6 +179,7 @@ uv run python scripts/bump_version.py 3.6.0 --commit - `apps/undefined-chat/src-tauri/Cargo.lock` 非 dry-run 时脚本还会执行 `uv sync`,并分别在 Console / Chat 下执行 `npm install --package-lock-only` 与 `cargo update --workspace`,保证 lock 文件和 manifest 不漂移。 +每次运行都会逐一核对所有版本文件,即使 `pyproject.toml` 已是目标版本也会补齐其余目标;非 dry-run 时仍完整执行锁文件同步。若写文件或同步依赖中途失败,修复原因后重跑同一个版本即可继续,不会因主版本已更新而跳过。锁同步实际修改的 `uv.lock`、`package-lock.json` 和 `Cargo.lock` 也会列入变更清单,`--commit` 会包含这些文件。`--dry-run` 只预览版本文件差异,不写盘或执行依赖同步。 脚本更新 JSON manifest 时只替换顶层 `version` 字段,保留现有格式,避免 Tauri 配置与 Biome 格式化规则漂移。 ### prepare_tauri_android.py — 生成后 Android 修补 diff --git a/scripts/bump_version.py b/scripts/bump_version.py index 72ee5064..421ad6db 100644 --- a/scripts/bump_version.py +++ b/scripts/bump_version.py @@ -247,13 +247,7 @@ def bump_project_versions( raise ValueError(f"'{version}' 不是合法的语义版本号 (x.y.z)") current = _read_current_version(root) - if version == current: - return BumpResult( - old_version=current, - new_version=version, - changed_paths=(), - ) - + # pyproject 可能已在上次失败前写完;同版本也必须逐一核对其余目标。 changed: list[str] = [] text_targets: tuple[tuple[Path, str, str], ...] = ( ( @@ -274,8 +268,25 @@ def bump_project_versions( for app in NATIVE_APPS: changed.extend(_update_native_app_versions(root, app, version, dry_run)) - if changed and not dry_run: + if not dry_run: + lock_paths = [root / "uv.lock"] + for app in NATIVE_APPS: + lock_paths.extend( + [ + app.app_root(root) / "package-lock.json", + app.tauri_root(root) / "Cargo.lock", + ] + ) + before_sync = { + path: path.read_bytes() if path.is_file() else None for path in lock_paths + } + # manifest 已一致也可能是上次同步 lock 失败后的重跑。 sync_lock_files(root, dry_run=False) + for path, previous in before_sync.items(): + content = path.read_bytes() if path.is_file() else None + relative = _relative(root, path) + if content != previous and relative not in changed: + changed.append(relative) return BumpResult( old_version=current, @@ -310,7 +321,10 @@ def main(argv: list[str] | None = None) -> int: return 1 if not result.changed_paths: - print(f"版本已经是 {result.old_version},无需更新") + if dry_run: + print(f"版本文件已经是 {result.old_version};实际执行时仍会同步 lock 文件") + else: + print(f"版本文件已经是 {result.old_version},lock 文件同步完成") return 0 prefix = "(dry-run) " if dry_run else "" diff --git a/tests/test_bump_version_script.py b/tests/test_bump_version_script.py index a13cf30f..26f8ff3c 100644 --- a/tests/test_bump_version_script.py +++ b/tests/test_bump_version_script.py @@ -2,6 +2,7 @@ import importlib.util import json +import subprocess from pathlib import Path import sys from types import ModuleType @@ -26,6 +27,16 @@ def _load_script() -> ModuleType: bump_version = _load_script() +@pytest.fixture(autouse=True) +def _isolate_commands(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.chdir(tmp_path) + + def forbidden(*args: Any, **kwargs: Any) -> Any: + raise AssertionError("tests must not run external commands") + + monkeypatch.setattr(bump_version.subprocess, "run", forbidden) + + def _write_bump_project(root: Path, *, version: str = "1.2.3") -> None: (root / "src" / "Undefined").mkdir(parents=True) (root / "pyproject.toml").write_text( @@ -214,3 +225,168 @@ def test_bump_project_versions_rejects_invalid_version(tmp_path: Path) -> None: project_root=tmp_path, dry_run=True, ) + + +def _assert_all_versions(root: Path, version: str) -> None: + assert ( + tomllib_loads((root / "pyproject.toml").read_text(encoding="utf-8"))["project"][ + "version" + ] + == version + ) + assert f'__version__: str = "{version}"' in ( + root / "src" / "Undefined" / "__init__.py" + ).read_text(encoding="utf-8") + for app in bump_version.NATIVE_APPS: + app_root = app.app_root(root) + tauri_root = app.tauri_root(root) + assert _json_version(app_root / "package.json") == version + assert _package_lock_versions(app_root / "package-lock.json") == ( + version, + version, + ) + assert ( + tomllib_loads((tauri_root / "Cargo.toml").read_text(encoding="utf-8"))[ + "package" + ]["version"] + == version + ) + assert _json_version(tauri_root / "tauri.conf.json") == version + assert ( + _cargo_lock_root_version(tauri_root / "Cargo.lock", app.cargo_package) + == version + ) + + +@pytest.mark.parametrize( + "failed_path", + ["src/Undefined/__init__.py", "apps/undefined-chat/package.json"], +) +def test_same_version_retry_finishes_all_targets_after_partial_write( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, failed_path: str +) -> None: + _write_bump_project(tmp_path) + assert Path.cwd() == tmp_path + original_write = bump_version._write_if_changed + sync_calls: list[Path] = [] + monkeypatch.setattr( + bump_version, + "sync_lock_files", + lambda root, *, dry_run: sync_calls.append(root), + ) + + def fail_write(path: Path, text: str, dry_run: bool) -> bool: + assert path.resolve().is_relative_to(tmp_path) + if path == tmp_path / failed_path: + raise OSError("temporarily unavailable") + result: bool = original_write(path, text, dry_run) + return result + + monkeypatch.setattr(bump_version, "_write_if_changed", fail_write) + with pytest.raises(OSError, match="temporarily unavailable"): + bump_version.bump_project_versions( + "2.0.0", project_root=tmp_path, dry_run=False + ) + assert bump_version._read_current_version(tmp_path) == "2.0.0" + assert not sync_calls + monkeypatch.setattr(bump_version, "_write_if_changed", original_write) + + result = bump_version.bump_project_versions( + "2.0.0", project_root=tmp_path, dry_run=False + ) + + assert result.old_version == result.new_version == "2.0.0" + assert result.changed_paths + assert sync_calls == [tmp_path] + _assert_all_versions(tmp_path, "2.0.0") + + +def test_same_version_retry_reruns_all_lock_commands_after_sync_failure( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _write_bump_project(tmp_path) + calls: list[tuple[tuple[str, ...], Path]] = [] + fail_next_npm = True + uv_runs = 0 + + def run( + command: list[str], *, cwd: Path, check: bool + ) -> subprocess.CompletedProcess[bytes]: + nonlocal fail_next_npm, uv_runs + assert cwd.resolve().is_relative_to(tmp_path) + assert check + calls.append((tuple(command), cwd)) + if command[0] == "npm" and fail_next_npm: + fail_next_npm = False + raise subprocess.CalledProcessError(1, command) + if command[0] == "uv": + uv_runs += 1 + (cwd / "uv.lock").write_text(f"sync_count={uv_runs}\n", encoding="utf-8") + return subprocess.CompletedProcess(command, 0) + + monkeypatch.setattr(bump_version.subprocess, "run", run) + with pytest.raises(subprocess.CalledProcessError): + bump_version.bump_project_versions( + "2.0.0", project_root=tmp_path, dry_run=False + ) + _assert_all_versions(tmp_path, "2.0.0") + calls.clear() + + result = bump_version.bump_project_versions( + "2.0.0", project_root=tmp_path, dry_run=False + ) + + assert [command for command, _ in calls] == [ + ("uv", "sync"), + ("npm", "install", "--package-lock-only"), + ("cargo", "update", "--workspace"), + ("npm", "install", "--package-lock-only"), + ("cargo", "update", "--workspace"), + ] + assert result.changed_paths == ("uv.lock",) + _assert_all_versions(tmp_path, "2.0.0") + + +def test_same_version_dry_run_detects_other_target_without_writing_or_syncing( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _write_bump_project(tmp_path, version="2.0.0") + target = tmp_path / "src" / "Undefined" / "__init__.py" + target.write_text('__version__: str = "1.2.3"\n', encoding="utf-8") + before = {path: path.read_bytes() for path in tmp_path.rglob("*") if path.is_file()} + monkeypatch.setattr( + bump_version, + "sync_lock_files", + lambda *args, **kwargs: pytest.fail("dry-run must not sync"), + ) + + result = bump_version.bump_project_versions( + "2.0.0", project_root=tmp_path, dry_run=True + ) + + assert result.changed_paths == ("src/Undefined/__init__.py",) + assert { + path: path.read_bytes() for path in tmp_path.rglob("*") if path.is_file() + } == before + + +def test_same_version_cli_reports_completed_lock_sync( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + _write_bump_project(tmp_path, version="2.0.0") + sync_calls: list[Path] = [] + monkeypatch.setattr(bump_version, "_PROJECT_ROOT", tmp_path) + monkeypatch.setattr( + bump_version, + "sync_lock_files", + lambda root, *, dry_run: sync_calls.append(root), + ) + + assert bump_version.main(["2.0.0"]) == 0 + + assert sync_calls == [tmp_path] + output = capsys.readouterr().out + assert "lock 文件同步完成" in output + assert "无需更新" not in output From 132852d8edecec923bed80d019c839a80df545f6 Mon Sep 17 00:00:00 2001 From: Null <1708213363@qq.com> Date: Tue, 6 Oct 2026 13:11:00 +0800 Subject: [PATCH 039/121] fix(weixin): flatten nested forward content before delivery --- docs/wechat-ilink.md | 2 + .../skills/tools/bilibili_opus/README.md | 2 + src/Undefined/utils/sender.py | 7 + tests/test_bilibili_opus_sender.py | 75 +++++- tests/test_sender.py | 241 ++++++++++++++++++ 5 files changed, 325 insertions(+), 2 deletions(-) diff --git a/docs/wechat-ilink.md b/docs/wechat-ilink.md index 14b21e42..74cd2938 100644 --- a/docs/wechat-ilink.md +++ b/docs/wechat-ilink.md @@ -109,6 +109,8 @@ iLink 帐号只接受登录结果声明的那个私聊来源。若收到帐号 I 微信文本中的 `<`、`>`、`&` 等特殊符号和 `` 附件标签应原样发送,不使用 HTML/XML 实体。微信没有 QQ 合并转发卡片的等价合同。默认情况下,Undefined 会把当前微信私聊的转发节点展开为保留原顺序的文本与媒体项目,再优先通过一个或多个 iLink `sendmessage.item_list` 请求投递;这是一种实验性多项目消息,不会生成可展开的 QQ 式转发卡片。单次项目数由 `multi_item_max_items` 限制,媒体合计仍受 `media_max_size_mb` 约束。若上游明确拒绝多项目结构,会从当前批次起自动回退为逐段发送;网络超时、会话暂停或服务端错误等结果不确定的失败不会降级重发,以免产生重复消息。设置 `multi_item_messages_enabled=false` 可直接使用逐段模式。包含本地 CQ 图片、文件、视频或音频的消息会走统一附件发送层,媒体本地路径不会作为正文发给用户。 +转发节点中的嵌套节点会递归展开,保留每层的发送者标题、正文以及文本和媒体的先后顺序;Bilibili 长正文分段和嵌套卡片也按这一规则发送。逐段模式与多项目模式都会在首个项目发出前校验所有层级的本地媒体,并完成全部语音准备。任一层媒体不存在、超限、使用当前不支持的远程来源或语音准备失败时,整个转发在投递前失败。 + 媒体上传由 SDK 在 `getuploadurl` 和 CDN 上传阶段处理临时错误重试;`media_upload_attempts` 控制总尝试次数。多项目消息会先按 `media_upload_concurrency` 有界并发上传媒体,再严格按原始项目顺序组装发送请求,因此并发只缩短准备时间,不改变用户看到的顺序。 Bot 层通过 `messages.send_voice(uid, address?)` 显式选择“作为语音发送”。普通 `` 始终保留原始文件语义,即使扩展名是 `.wav`;`CQ:record`、`CQ:audio` 和内部明确标记为 voice 的媒体回调则发送原生语音。QQ 侧继续转换为 `CQ:record`,微信侧先用 FFmpeg 归一化为 24 kHz、16-bit、单声道 PCM,再由 `silk-python` 编码为 Tencent SILK。部署机必须能从 `PATH` 找到 `ffmpeg`;缺失、音频无效或超出 `[weixin].media_max_size_mb` 时会在任何消息段发出前失败,不会自动改成文件。 diff --git a/src/Undefined/skills/tools/bilibili_opus/README.md b/src/Undefined/skills/tools/bilibili_opus/README.md index 9dd8b01a..adec4011 100644 --- a/src/Undefined/skills/tools/bilibili_opus/README.md +++ b/src/Undefined/skills/tools/bilibili_opus/README.md @@ -19,6 +19,8 @@ 4. 发送合并转发:第一条是图文信息(封面、标题、UP主、时间、数据、原文链接),第二条是正文(文本与图片按原始顺序混排;单节点上限 1200 字 / 9 张图,超出时切成 `正文 1/N` 并整体收进顶层 `正文` 的嵌套内容,避免顶层卡片预览携带全文而被 QQ 拒收) 5. 正文里的图文/视频卡片各自成为独立嵌套合并转发,按 `opus_nested_depth` / `opus_nested_max_cards` 递归展开,超界降级为一行文本 + 链接 +微信私聊发送时,这些嵌套节点会递归展开成有序文本与本地媒体项目,保留长正文分段、嵌套卡片内容和发送者标题。发送前统一校验所有层级的媒体并准备语音;逐段模式和多项目模式都使用这一流程。微信目前仅支持本地媒体来源,任一媒体准备失败时不会先投递部分正文。 + `uid` 模式流程: 1. 拉取图文并收集正文图片 URL 2. 按 `max_images` 上限逐张 `register_remote_url`(超过附件大小上限时自动降级为 URL 引用) diff --git a/src/Undefined/utils/sender.py b/src/Undefined/utils/sender.py index 6e07869a..2e1feb8c 100644 --- a/src/Undefined/utils/sender.py +++ b/src/Undefined/utils/sender.py @@ -1963,6 +1963,13 @@ def flush_text() -> None: if not isinstance(segment, dict): continue segment_type = str(segment.get("type", "") or "").strip().lower() + if segment_type == "node": + nested_units = await self._build_weixin_forward_units([segment]) + if nested_units: + flush_text() + node_units.extend(nested_units) + has_content = True + continue if segment_type not in { "text", "image", diff --git a/tests/test_bilibili_opus_sender.py b/tests/test_bilibili_opus_sender.py index cede3dbc..d74d7e9f 100644 --- a/tests/test_bilibili_opus_sender.py +++ b/tests/test_bilibili_opus_sender.py @@ -1,12 +1,13 @@ from __future__ import annotations -from collections.abc import Iterator +from collections.abc import Iterator, Sequence from pathlib import Path from types import SimpleNamespace -from typing import Any +from typing import Any, cast from unittest.mock import AsyncMock, MagicMock import pytest +from weixin_ilink_client import OutboundMessageItem, OutboundTextItem import Undefined.bilibili.opus_sender as opus_sender from Undefined.bilibili.errors import OpusUnavailableError @@ -23,6 +24,9 @@ VideoStats, ) from Undefined.onebot.client import OneBotDeliveryUncertainError +from Undefined.utils.history import MessageHistoryManager +from Undefined.utils.message_targets import DeliveryAddress +from Undefined.utils.sender import AddressBoundSender, MessageSender def _author(name: str = "测试UP") -> OpusAuthor: @@ -534,6 +538,73 @@ async def test_send_opus_sends_forward_with_history( assert call.kwargs["history_message"].startswith("[Bilibili 图文] 「测试图文」") +@pytest.mark.asyncio +@pytest.mark.parametrize("multi_item_enabled", [False, True]) +async def test_send_opus_to_wechat_preserves_nested_long_body( + monkeypatch: pytest.MonkeyPatch, + multi_item_enabled: bool, +) -> None: + body = "A" * NODE_TEXT_BUDGET + "B" * NODE_TEXT_BUDGET + "C" * 80 + info = _info(TextBlock(body), cover_url="") + monkeypatch.setattr(opus_sender, "_fetch_opus_info", AsyncMock(return_value=info)) + config = _config() + config.weixin = SimpleNamespace( + multi_item_messages_enabled=multi_item_enabled, + multi_item_max_items=2, + media_max_size_mb=100, + ) + + def is_private_allowed(_user_id: int) -> bool: + return True + + config.is_private_allowed = is_private_allowed + history = SimpleNamespace(add_private_message=AsyncMock()) + sender = MessageSender( + MagicMock(), cast(MessageHistoryManager, history), bot_qq=10000, config=config + ) + delivered: list[str] = [] + + async def send_text(_user_id: int, text: str, **_kwargs: Any) -> str: + delivered.append(text) + return f"client-{len(delivered)}" + + async def send_items(_user_id: int, items: Sequence[OutboundMessageItem]) -> str: + for item in items: + assert isinstance(item, OutboundTextItem) + delivered.append(item.text) + return f"client-{len(delivered)}" + + sender.set_weixin_service( + SimpleNamespace(send_text=send_text, send_items=send_items) + ) + bound_sender = AddressBoundSender(sender, DeliveryAddress("wechat", 20001)) + + result = await opus_sender.send_opus( + info.opus_id, + sender=cast(MessageSender, bound_sender), + target_type="private", + target_id=20001, + config=config, + ) + + assert result.startswith("已发送 Bilibili 图文合并转发") + assert "图文信息" in delivered[0] + joined = "".join(delivered) + assert joined.count("A") == NODE_TEXT_BUDGET + assert joined.count("B") == NODE_TEXT_BUDGET + assert joined.count("C") == 80 + assert ( + joined.index("A" * NODE_TEXT_BUDGET) + < joined.index("B" * NODE_TEXT_BUDGET) + < joined.index("C" * 80) + ) + history.add_private_message.assert_awaited_once() + assert ( + history.add_private_message.await_args.kwargs["transport"]["channel"] + == "wechat" + ) + + @pytest.mark.asyncio async def test_send_opus_propagates_delivery_uncertain( monkeypatch: pytest.MonkeyPatch, diff --git a/tests/test_sender.py b/tests/test_sender.py index 0d2df237..2e2a311a 100644 --- a/tests/test_sender.py +++ b/tests/test_sender.py @@ -2,6 +2,8 @@ import logging import nturl2path +from collections.abc import Sequence +from copy import deepcopy from pathlib import Path from types import SimpleNamespace from typing import Any, cast @@ -12,6 +14,7 @@ ApiError, HttpError, OutboundMediaItem, + OutboundMessageItem, OutboundTextItem, RefMessage, RequestTimeoutError, @@ -38,6 +41,7 @@ _local_path_from_segment_source, ) from Undefined.weixin.audio import PreparedWeixinVoice +from Undefined.weixin.service import WeixinService, WeixinServiceError @pytest.fixture @@ -1569,6 +1573,243 @@ async def send_file(_user_id: int, path: str | Path, **_kwargs: Any) -> str: assert "type=at" in caplog.text +@pytest.mark.asyncio +@pytest.mark.parametrize("multi_item_enabled", [False, True]) +async def test_send_wechat_nested_forward_preserves_text_and_media_order( + sender: MessageSender, + tmp_path: Path, + multi_item_enabled: bool, +) -> None: + image_path = tmp_path / "nested.png" + voice_path = tmp_path / "nested.wav" + video_path = tmp_path / "nested.mp4" + for path in (image_path, voice_path, video_path): + await async_io.write_bytes(path, path.name.encode()) + weixin_config = cast(Any, sender.config.weixin) + weixin_config.multi_item_messages_enabled = multi_item_enabled + weixin_config.multi_item_max_items = 3 + weixin_config.media_max_size_mb = 100 + events: list[tuple[str, str]] = [] + prepared = PreparedWeixinVoice(content=b"silk", duration_ms=200) + + async def validate_media_files(paths: Sequence[str | Path]) -> None: + assert list(paths) == [image_path, voice_path, video_path] + assert events == [] + events.append(("prepare", "validate")) + + async def prepare_voice(path: Path) -> PreparedWeixinVoice: + assert path == voice_path + assert events == [("prepare", "validate")] + events.append(("prepare", "voice")) + return prepared + + async def send_text(_user_id: int, text: str, **_kwargs: Any) -> str: + events.append(("text", text)) + return f"client-{len(events)}" + + async def send_file(_user_id: int, path: str | Path, **kwargs: Any) -> str: + events.append((str(kwargs["kind"]), Path(path).name)) + return f"client-{len(events)}" + + async def send_prepared_voice( + _user_id: int, value: PreparedWeixinVoice, **_kwargs: Any + ) -> str: + assert value is prepared + events.append(("voice", voice_path.name)) + return f"client-{len(events)}" + + async def send_items(_user_id: int, items: Sequence[OutboundMessageItem]) -> str: + for item in items: + if isinstance(item, OutboundTextItem): + events.append(("text", item.text)) + else: + events.append((str(item.kind), item.file_name)) + return f"client-{len(events)}" + + service = SimpleNamespace( + validate_media_files=validate_media_files, + prepare_voice=prepare_voice, + send_text=send_text, + send_file=send_file, + send_prepared_voice=send_prepared_voice, + send_items=send_items, + ) + sender.set_weixin_service(service) + nodes = [ + { + "type": "node", + "data": { + "name": "Parent", + "content": [ + {"type": "text", "data": {"text": "Before"}}, + { + "type": "node", + "data": { + "name": "Child", + "content": [ + {"type": "text", "data": {"text": "Child-start"}}, + { + "type": "image", + "data": {"file": image_path.as_uri()}, + }, + { + "type": "node", + "data": { + "name": "Grandchild", + "content": [ + { + "type": "text", + "data": {"text": "Deep-start"}, + }, + { + "type": "record", + "data": {"file": voice_path.as_uri()}, + }, + { + "type": "video", + "data": {"file": video_path.as_uri()}, + }, + { + "type": "node", + "data": { + "name": "Leaf", + "content": "Leaf-text", + }, + }, + { + "type": "text", + "data": {"text": "Deep-end"}, + }, + ], + }, + }, + {"type": "text", "data": {"text": "Child-end"}}, + ], + }, + }, + {"type": "text", "data": {"text": "After"}}, + ], + }, + }, + {"type": "node", "data": {"name": "Last", "content": "Last-text"}}, + ] + original_nodes = deepcopy(nodes) + + await AddressBoundSender( + sender, DeliveryAddress("wechat", 12345) + ).send_private_forward_message(12345, nodes, history_message="Nested forward") + + assert events == [ + ("prepare", "validate"), + ("prepare", "voice"), + ("text", "**Parent**\nBefore"), + ("text", "**Child**\nChild-start"), + ("image", image_path.name), + ("text", "**Grandchild**\nDeep-start"), + ("voice", voice_path.name), + ("video", video_path.name), + ("text", "**Leaf**\nLeaf-text"), + ("text", "Deep-end"), + ("text", "Child-end"), + ("text", "After"), + ("text", ("\n" if multi_item_enabled else "") + "---\n**Last**\nLast-text"), + ] + assert nodes == original_nodes + cast(AsyncMock, sender.history_manager.add_private_message).assert_awaited_once() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("multi_item_enabled", [False, True]) +@pytest.mark.parametrize("failure", ["oversized", "missing", "remote", "voice"]) +async def test_send_wechat_nested_forward_preflights_all_media_before_delivery( + sender: MessageSender, + tmp_path: Path, + multi_item_enabled: bool, + failure: str, +) -> None: + image_path = tmp_path / "valid.png" + invalid_path = tmp_path / "invalid.bin" + await async_io.write_bytes(image_path, b"image") + if failure != "missing": + await async_io.write_bytes( + invalid_path, + b"x" * (1024 * 1024 + 1) if failure == "oversized" else b"voice", + ) + weixin_config = cast(Any, sender.config.weixin) + weixin_config.multi_item_messages_enabled = multi_item_enabled + weixin_config.media_max_size_mb = 1 + validation_service = cast( + WeixinService, SimpleNamespace(weixin_config=weixin_config) + ) + + async def validate_media_files(paths: Sequence[str | Path]) -> None: + await WeixinService.validate_media_files(validation_service, paths) + + service = SimpleNamespace( + validate_media_files=AsyncMock(side_effect=validate_media_files), + prepare_voice=AsyncMock(side_effect=WeixinServiceError("invalid voice")), + send_text=AsyncMock(), + send_file=AsyncMock(), + send_prepared_voice=AsyncMock(), + send_items=AsyncMock(), + ) + sender.set_weixin_service(service) + invalid_source = ( + "https://example.com/invalid.bin" + if failure == "remote" + else invalid_path.as_uri() + ) + nodes = [ + { + "type": "node", + "data": { + "content": [ + {"type": "text", "data": {"text": "Must not be sent"}}, + { + "type": "node", + "data": { + "content": [ + { + "type": "image", + "data": {"file": image_path.as_uri()}, + }, + { + "type": "node", + "data": { + "content": [ + { + "type": "record" + if failure == "voice" + else "file", + "data": {"file": invalid_source}, + }, + ] + }, + }, + ] + }, + }, + ] + }, + } + ] + + with pytest.raises((WeixinServiceError, ValueError)): + await AddressBoundSender( + sender, DeliveryAddress("wechat", 12345) + ).send_private_forward_message(12345, nodes, history_message="Nested forward") + + service.send_text.assert_not_awaited() + service.send_file.assert_not_awaited() + service.send_prepared_voice.assert_not_awaited() + service.send_items.assert_not_awaited() + cast(AsyncMock, sender.history_manager.add_private_message).assert_not_awaited() + if failure in {"oversized", "voice"}: + service.validate_media_files.assert_awaited_once_with( + [image_path, invalid_path] + ) + + @pytest.mark.asyncio async def test_send_wechat_forward_uses_ordered_multi_item_message( sender: MessageSender, From cb48f925333b2ce37ed2a2ecd3df8961c8d0e1c0 Mon Sep 17 00:00:00 2001 From: Null <1708213363@qq.com> Date: Tue, 6 Oct 2026 13:12:17 +0800 Subject: [PATCH 040/121] docs(scripts): clarify cognitive storage configuration paths --- docs/cognitive-memory.md | 2 +- scripts/README.md | 2 +- scripts/reembed_cognitive.py | 2 +- scripts/restore_profile.py | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/cognitive-memory.md b/docs/cognitive-memory.md index d8d772ca..9c161ec9 100644 --- a/docs/cognitive-memory.md +++ b/docs/cognitive-memory.md @@ -495,7 +495,7 @@ uv run python scripts/restore_profile.py sync --entity-type user --entity-id {us `restore` 默认同时恢复 Markdown、保存当前内容的历史快照,并用当前 cognitive 嵌入模型同步 `cognitive_profiles` 中该实体的检索正文和向量,其他实体保持不变。检索正文沿用运行时格式,包含恢复版本中的名称、标签、评价、锐评与正文。模型调用失败时尚未恢复文件;若文件已恢复而向量写入失败,命令会报错,可用 `sync` 从当前文件补同步。文件和向量库之间没有跨存储事务,请保持机器人及其他写入进程停止,直到同步成功。 -`--dry-run` 只展示将恢复的内容、检索正文和两个目录,不写盘、不打开 Chroma、不调用模型。目录默认读取 `cognitive.profiles_path` 与 `cognitive.vector_store_path`;操作独立副本时同时指定 `--profiles-path` 和 `--db-path`。实际恢复/同步可能调用嵌入 API 并记录 token 用量;若已更换嵌入模型维度,请先按[更换嵌入模型](#更换嵌入模型)重建整个向量库,再恢复或同步单个侧写。 +`--dry-run` 只展示将恢复的内容、检索正文和两个目录,不写盘、不打开 Chroma、不调用模型。目录默认读取 TOML 配置的 `cognitive.profile.path` 与 `cognitive.vector_store.path`;操作独立副本时同时指定 `--profiles-path` 和 `--db-path`。实际恢复/同步可能调用嵌入 API 并记录 token 用量;若已更换嵌入模型维度,请先按[更换嵌入模型](#更换嵌入模型)重建整个向量库,再恢复或同步单个侧写。 **级别 3:完整移除** diff --git a/scripts/README.md b/scripts/README.md index aaca81e9..256994bc 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -122,7 +122,7 @@ uv run python scripts/restore_profile.py sync --entity-type user --entity-id 123 `restore` 默认恢复 Markdown、把当前内容另存为历史快照,并使用当前 cognitive 嵌入配置更新 `cognitive_profiles` 中该实体的检索正文与向量。`sync` 只从当前 Markdown 补同步检索,不创建快照。模型请求失败时文件保持原状;向量写入失败时命令报错,已恢复文件保留,可修复错误后重跑 `sync`,无需依赖可能已被清理的旧快照。文件与向量库没有跨存储事务,实际恢复/同步前需停止机器人和其他写入进程,成功后再启动。 -`--dry-run` 展示文件内容、检索正文与路径,不打开 Chroma、不调用模型、不写盘。目录默认来自 `cognitive.profiles_path` 与 `cognitive.vector_store_path`;操作独立副本时同时传入 `--profiles-path` 和 `--db-path`。实际恢复/同步会调用嵌入 API,可能记录 token 用量。若新模型维度不同于现有库,先完成全库重嵌入再恢复/同步单个侧写。 +`--dry-run` 展示文件内容、检索正文与路径,不打开 Chroma、不调用模型、不写盘。目录默认来自 TOML 配置的 `cognitive.profile.path` 与 `cognitive.vector_store.path`;操作独立副本时同时传入 `--profiles-path` 和 `--db-path`。实际恢复/同步会调用嵌入 API,可能记录 token 用量。若新模型维度不同于现有库,先完成全库重嵌入再恢复/同步单个侧写。 ### release_notes.py — 发布版本校验与 Release notes 生成 diff --git a/scripts/reembed_cognitive.py b/scripts/reembed_cognitive.py index 48ea62f9..2e6de6d4 100755 --- a/scripts/reembed_cognitive.py +++ b/scripts/reembed_cognitive.py @@ -106,7 +106,7 @@ def _parse_args() -> argparse.Namespace: "--db-path", type=str, default=None, - help="ChromaDB 路径(默认读取 config.toml 中的 cognitive.vector_store_path)", + help="ChromaDB 路径(默认读取 config.toml 中的 cognitive.vector_store.path)", ) parser.add_argument( "-v", diff --git a/scripts/restore_profile.py b/scripts/restore_profile.py index b577b1c7..164641ff 100644 --- a/scripts/restore_profile.py +++ b/scripts/restore_profile.py @@ -64,7 +64,7 @@ def _parse_args() -> argparse.Namespace: parser.add_argument( "--db-path", default="", - help="ChromaDB 目录(默认读取 config.toml 中的 cognitive.vector_store_path)", + help="ChromaDB 目录(默认读取 config.toml 中的 cognitive.vector_store.path)", ) parser.add_argument( "--entity-type", @@ -81,7 +81,7 @@ def _parse_args() -> argparse.Namespace: parser.add_argument( "--profiles-path", default="", - help="侧写目录(默认读取 config.toml 中的 cognitive.profiles_path)", + help="侧写目录(默认读取 config.toml 中的 cognitive.profile.path)", ) parser.add_argument( "--dry-run", From 2530c2ca1879bb34d1949df5e183ff76f88c1d6b Mon Sep 17 00:00:00 2001 From: Null <1708213363@qq.com> Date: Tue, 6 Oct 2026 13:15:44 +0800 Subject: [PATCH 041/121] fix(cognitive): validate profile paths inside file IO threads --- docs/python-api.md | 2 +- src/Undefined/cognitive/profile_storage.py | 13 +++--- tests/test_cognitive_profile_paths.py | 52 ++++++++++++++++++++++ 3 files changed, 60 insertions(+), 7 deletions(-) diff --git a/docs/python-api.md b/docs/python-api.md index c86360bf..950892df 100644 --- a/docs/python-api.md +++ b/docs/python-api.md @@ -297,7 +297,7 @@ server = RuntimeAPIServer(RuntimeAPIContext(...)) | `Undefined.api` | stable | `RuntimeAPIServer`, `RuntimeAPIContext` | | `Undefined.mcp` | subpackage | `MCPToolRegistry`, `MCPToolSetRegistry` | -`ProfileStorage` 的所有读写、历史版本和合并锁操作只接受 `user` / `group` 实体类型,以及非空的单层实体 ID(不接受路径分隔符、`.` / `..` 或 Windows 盘符)。侧写文件与历史版本解析符号链接后必须仍位于 `base_path` 内,否则抛出 `ValueError`;Runtime 的侧写查询接口将这类错误转换为 HTTP 400。 +`ProfileStorage` 的所有读写、历史版本和合并锁操作只接受 `user` / `group` 实体类型,以及非空的单层实体 ID(不接受路径分隔符、`.` / `..` 或 Windows 盘符)。侧写文件与历史版本解析符号链接后必须仍位于 `base_path` 内,否则抛出 `ValueError`;路径解析与文件访问一起在线程中执行,避免阻塞事件循环。Runtime 的侧写查询接口将这类错误转换为 HTTP 400。 --- diff --git a/src/Undefined/cognitive/profile_storage.py b/src/Undefined/cognitive/profile_storage.py index e8798642..a064e835 100644 --- a/src/Undefined/cognitive/profile_storage.py +++ b/src/Undefined/cognitive/profile_storage.py @@ -76,12 +76,12 @@ def _contained_path(self, path: Path) -> Path: return path async def read_profile(self, entity_type: str, entity_id: str) -> str | None: - p = self._profile_path(entity_type, entity_id) logger.info( "[认知侧写] 读取侧写: entity_type=%s entity_id=%s", entity_type, entity_id ) def _read() -> str | None: + p = self._profile_path(entity_type, entity_id) if not p.exists(): return None return p.read_text(encoding="utf-8") @@ -106,10 +106,10 @@ async def write_profile( len(content or ""), ) async with self._get_lock(entity_type, entity_id): - p = self._profile_path(entity_type, entity_id) - hist_dir = self._history_dir(entity_type, entity_id) def _write() -> None: + p = self._profile_path(entity_type, entity_id) + hist_dir = self._history_dir(entity_type, entity_id) p.parent.mkdir(parents=True, exist_ok=True) hist_dir.mkdir(parents=True, exist_ok=True) @@ -156,9 +156,8 @@ def _write() -> None: ) async def list_revisions(self, entity_type: str, entity_id: str) -> list[str]: - hist_dir = self._history_dir(entity_type, entity_id) - def _list() -> list[str]: + hist_dir = self._history_dir(entity_type, entity_id) if not hist_dir.exists(): return [] return sorted(f.name for f in hist_dir.glob("*.md")) @@ -193,9 +192,11 @@ async def read_revision( ) -> str | None: """读取指定历史版本内容;版本名取 `list_revisions` 的返回值。""" name = self._normalize_revision_name(revision) - path = self._contained_path(self._history_dir(entity_type, entity_id) / name) def _read() -> str | None: + path = self._contained_path( + self._history_dir(entity_type, entity_id) / name + ) if not path.exists(): return None return path.read_text(encoding="utf-8") diff --git a/tests/test_cognitive_profile_paths.py b/tests/test_cognitive_profile_paths.py index 6daf45d8..a9bb14fc 100644 --- a/tests/test_cognitive_profile_paths.py +++ b/tests/test_cognitive_profile_paths.py @@ -1,6 +1,8 @@ from __future__ import annotations +import asyncio import json +import threading from pathlib import Path from types import SimpleNamespace from typing import Any @@ -14,6 +16,56 @@ from Undefined.skills.toolsets.cognitive.get_profile.handler import execute +@pytest.mark.asyncio +async def test_profile_paths_are_validated_in_file_io_threads( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + storage = ProfileStorage(tmp_path) + event_loop_thread = threading.get_ident() + checked_threads: list[int] = [] + original = storage._contained_path + + def check_path(path: Path) -> Path: + checked_threads.append(threading.get_ident()) + return original(path) + + monkeypatch.setattr(storage, "_contained_path", check_path) + await storage.write_profile("user", "10001", "v1") + await storage.write_profile("user", "10001", "v2") + assert await storage.read_profile("user", "10001") == "v2" + revisions = await storage.list_revisions("user", "10001") + assert await storage.read_revision("user", "10001", revisions[0]) == "v1" + await storage.restore_revision("user", "10001", revisions[0]) + assert checked_threads + assert event_loop_thread not in checked_threads + + +@pytest.mark.asyncio +async def test_profile_read_checks_links_after_dispatching_file_io( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + base = tmp_path / "profiles" + users = base / "users" + users.mkdir(parents=True) + (users / "10001.md").write_text("safe", encoding="utf-8") + outside = tmp_path / "outside" + outside.mkdir() + (outside / "10001.md").write_text("private", encoding="utf-8") + storage = ProfileStorage(base) + original = asyncio.to_thread + + async def dispatch_after_link_change( + func: Any, /, *args: Any, **kwargs: Any + ) -> Any: + users.rename(base / "original-users") + users.symlink_to(outside, target_is_directory=True) + return await original(func, *args, **kwargs) + + monkeypatch.setattr(asyncio, "to_thread", dispatch_after_link_change) + with pytest.raises(ValueError, match="超出存储目录"): + await storage.read_profile("user", "10001") + + @pytest.mark.asyncio @pytest.mark.parametrize( ("entity_type", "entity_id"), From 4e3b173ea45c9197ed515780abbf22f912d2b955 Mon Sep 17 00:00:00 2001 From: Null <1708213363@qq.com> Date: Tue, 6 Oct 2026 13:18:56 +0800 Subject: [PATCH 042/121] fix(scripts): commit all managed version targets on retry --- scripts/README.md | 2 +- scripts/bump_version.py | 68 ++++++++++++---- tests/test_bump_version_script.py | 127 ++++++++++++++++++++++++++++++ 3 files changed, 179 insertions(+), 18 deletions(-) diff --git a/scripts/README.md b/scripts/README.md index 256994bc..a5e4d742 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -179,7 +179,7 @@ uv run python scripts/bump_version.py 3.6.0 --commit - `apps/undefined-chat/src-tauri/Cargo.lock` 非 dry-run 时脚本还会执行 `uv sync`,并分别在 Console / Chat 下执行 `npm install --package-lock-only` 与 `cargo update --workspace`,保证 lock 文件和 manifest 不漂移。 -每次运行都会逐一核对所有版本文件,即使 `pyproject.toml` 已是目标版本也会补齐其余目标;非 dry-run 时仍完整执行锁文件同步。若写文件或同步依赖中途失败,修复原因后重跑同一个版本即可继续,不会因主版本已更新而跳过。锁同步实际修改的 `uv.lock`、`package-lock.json` 和 `Cargo.lock` 也会列入变更清单,`--commit` 会包含这些文件。`--dry-run` 只预览版本文件差异,不写盘或执行依赖同步。 +每次运行都会逐一核对所有版本文件,即使 `pyproject.toml` 已是目标版本也会补齐其余目标;非 dry-run 时仍完整执行锁文件同步。若写文件或同步依赖中途失败,修复原因后重跑同一个版本即可继续,不会因主版本已更新而跳过。锁同步实际修改的 `uv.lock`、`package-lock.json` 和 `Cargo.lock` 也会列入本次变更清单。`--commit` 会核对并暂存上述完整的版本管理目标与 `uv.lock`,包括上次失败前已写入的目标,即使本次没有文件写入差异也会检查;只有这些目标存在待提交差异时才创建提交,已暂存的无关文件保持暂存状态,不纳入版本提交。`--dry-run` 只预览版本文件差异,不写盘、执行依赖同步或运行 Git。 脚本更新 JSON manifest 时只替换顶层 `version` 字段,保留现有格式,避免 Tauri 配置与 Biome 格式化规则漂移。 ### prepare_tauri_android.py — 生成后 Android 修补 diff --git a/scripts/bump_version.py b/scripts/bump_version.py index 421ad6db..018c5b67 100644 --- a/scripts/bump_version.py +++ b/scripts/bump_version.py @@ -303,6 +303,46 @@ def _build_parser() -> argparse.ArgumentParser: return parser +def _commit_version_files(project_root: Path, version: str) -> bool: + """提交全部受管理目标的差异,包括上次失败前已写入的文件。""" + paths = ["pyproject.toml", "src/Undefined/__init__.py", "uv.lock"] + for app in NATIVE_APPS: + paths.extend( + [ + app.package_json, + app.package_lock, + app.cargo_toml, + app.tauri_conf, + app.cargo_lock, + ] + ) + subprocess.run(["git", "add", "--", *paths], cwd=project_root, check=True) + diff = subprocess.run( + ["git", "diff", "--cached", "--quiet", "--", *paths], + cwd=project_root, + check=False, + ) + if diff.returncode == 0: + return False + if diff.returncode != 1: + raise subprocess.CalledProcessError(diff.returncode, diff.args) + # 明确限定提交范围,保留用户已暂存的其他文件。 + subprocess.run( + [ + "git", + "commit", + "--only", + "-m", + f"chore(version): bump version to {version}", + "--", + *paths, + ], + cwd=project_root, + check=True, + ) + return True + + def main(argv: list[str] | None = None) -> int: parser = _build_parser() args = parser.parse_args(argv) @@ -325,26 +365,20 @@ def main(argv: list[str] | None = None) -> int: print(f"版本文件已经是 {result.old_version};实际执行时仍会同步 lock 文件") else: print(f"版本文件已经是 {result.old_version},lock 文件同步完成") - return 0 - - prefix = "(dry-run) " if dry_run else "" - print(f"{prefix}版本 {result.old_version} → {result.new_version}\n") - for path in result.changed_paths: - print(f" ✓ {path}") + if not should_commit or dry_run: + return 0 + else: + prefix = "(dry-run) " if dry_run else "" + print(f"{prefix}版本 {result.old_version} → {result.new_version}\n") + for path in result.changed_paths: + print(f" ✓ {path}") if should_commit and not dry_run: print("\n创建 git commit...") - subprocess.run( - ["git", "add", *result.changed_paths, "uv.lock"], - cwd=_PROJECT_ROOT, - check=True, - ) - subprocess.run( - ["git", "commit", "-m", f"chore(version): bump version to {new_version}"], - cwd=_PROJECT_ROOT, - check=True, - ) - print(f"\n完成! 已提交版本 {new_version}") + if _commit_version_files(_PROJECT_ROOT, new_version): + print(f"\n完成! 已提交版本 {new_version}") + else: + print(f"\n版本 {new_version} 的受管理文件没有待提交差异") elif not dry_run: print(f"\n完成! 版本已更新为 {new_version}") print("提示: 使用 --commit 可自动提交,或手动 git add + commit") diff --git a/tests/test_bump_version_script.py b/tests/test_bump_version_script.py index 26f8ff3c..e502b887 100644 --- a/tests/test_bump_version_script.py +++ b/tests/test_bump_version_script.py @@ -390,3 +390,130 @@ def test_same_version_cli_reports_completed_lock_sync( output = capsys.readouterr().out assert "lock 文件同步完成" in output assert "无需更新" not in output + + +def _record_git( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + *, + staged_difference: bool, +) -> list[list[str]]: + calls: list[list[str]] = [] + + def run( + command: list[str], *, cwd: Path, check: bool + ) -> subprocess.CompletedProcess[bytes]: + assert cwd == tmp_path + assert command[0] == "git" + calls.append(command) + code = int(staged_difference) if command[1] == "diff" else 0 + return subprocess.CompletedProcess(command, code) + + monkeypatch.setattr(bump_version.subprocess, "run", run) + return calls + + +def _assert_complete_git_scope(calls: list[list[str]]) -> None: + expected = {"pyproject.toml", "src/Undefined/__init__.py", "uv.lock"} + for app in bump_version.NATIVE_APPS: + expected.update( + [ + app.package_json, + app.package_lock, + app.cargo_toml, + app.tauri_conf, + app.cargo_lock, + ] + ) + assert [command[1] for command in calls] == ["add", "diff", "commit"] + for command in calls: + assert set(command[command.index("--") + 1 :]) == expected + assert "--cached" in calls[1] and "--quiet" in calls[1] + assert "--only" in calls[2] + + +def test_main_partial_failure_retry_commits_prior_pyproject_write( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _write_bump_project(tmp_path) + monkeypatch.setattr(bump_version, "_PROJECT_ROOT", tmp_path) + monkeypatch.setattr(bump_version, "sync_lock_files", lambda *args, **kwargs: None) + calls = _record_git(tmp_path, monkeypatch, staged_difference=True) + original_write = bump_version._write_if_changed + + def fail_init(path: Path, text: str, dry_run: bool) -> bool: + assert path.resolve().is_relative_to(tmp_path) + if path.name == "__init__.py": + raise OSError("controlled write failure") + result: bool = original_write(path, text, dry_run) + return result + + monkeypatch.setattr(bump_version, "_write_if_changed", fail_init) + assert bump_version.main(["2.0.0", "--commit"]) == 1 + assert bump_version._read_current_version(tmp_path) == "2.0.0" + assert not calls + monkeypatch.setattr(bump_version, "_write_if_changed", original_write) + + assert bump_version.main(["2.0.0", "--commit"]) == 0 + + _assert_all_versions(tmp_path, "2.0.0") + _assert_complete_git_scope(calls) + + +def test_main_lock_failure_retry_commits_when_this_run_changes_no_files( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _write_bump_project(tmp_path) + monkeypatch.setattr(bump_version, "_PROJECT_ROOT", tmp_path) + calls = _record_git(tmp_path, monkeypatch, staged_difference=True) + sync_count = 0 + + def sync(root: Path, *, dry_run: bool) -> None: + nonlocal sync_count + assert root == tmp_path and not dry_run + sync_count += 1 + if sync_count == 1: + raise subprocess.CalledProcessError( + 1, ["npm", "install", "--package-lock-only"] + ) + + monkeypatch.setattr(bump_version, "sync_lock_files", sync) + assert bump_version.main(["2.0.0", "--commit"]) == 1 + _assert_all_versions(tmp_path, "2.0.0") + before = {path: path.read_bytes() for path in tmp_path.rglob("*") if path.is_file()} + assert not calls + + assert bump_version.main(["2.0.0", "--commit"]) == 0 + + assert { + path: path.read_bytes() for path in tmp_path.rglob("*") if path.is_file() + } == before + assert sync_count == 2 + _assert_complete_git_scope(calls) + + +def test_main_commit_skips_creation_without_managed_git_differences( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _write_bump_project(tmp_path, version="2.0.0") + monkeypatch.setattr(bump_version, "_PROJECT_ROOT", tmp_path) + monkeypatch.setattr(bump_version, "sync_lock_files", lambda *args, **kwargs: None) + calls = _record_git(tmp_path, monkeypatch, staged_difference=False) + + assert bump_version.main(["2.0.0", "--commit"]) == 0 + + assert [command[1] for command in calls] == ["add", "diff"] + + +def test_main_dry_run_commit_never_calls_git_or_lock_sync( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _write_bump_project(tmp_path) + monkeypatch.setattr(bump_version, "_PROJECT_ROOT", tmp_path) + monkeypatch.setattr( + bump_version, "sync_lock_files", lambda *args, **kwargs: pytest.fail("dry-run") + ) + + # autouse fixture also forbids every actual subprocess. + assert bump_version.main(["2.0.0", "--commit", "--dry-run"]) == 0 + _assert_all_versions(tmp_path, "1.2.3") From 842fd05a31bdcefababb29a17c526137f929e675 Mon Sep 17 00:00:00 2001 From: Null <1708213363@qq.com> Date: Tue, 6 Oct 2026 13:21:20 +0800 Subject: [PATCH 043/121] fix(chat): suspend requests and recover failed connection changes --- apps/undefined-chat/src/App.test.tsx | 155 ++++++++++++++++-- apps/undefined-chat/src/App.tsx | 83 +++++++++- .../chat-store/connection-recovery.test.ts | 60 +++++++ apps/undefined-chat/src/chat-store/store.ts | 47 +++++- .../src/connection-recovery.test.tsx | 108 ++++++++++++ apps/undefined-chat/src/i18n/index.ts | 10 ++ .../src/runtime-client/tauri.test.ts | 17 ++ .../src/runtime-client/tauri.ts | 15 +- .../src/runtime-client/types.ts | 1 + apps/undefined-chat/src/test-fixtures.ts | 1 + .../tests/e2e/connection-setup.test.tsx | 21 ++- docs/undefined-chat.md | 2 + 12 files changed, 485 insertions(+), 35 deletions(-) create mode 100644 apps/undefined-chat/src/chat-store/connection-recovery.test.ts create mode 100644 apps/undefined-chat/src/connection-recovery.test.tsx diff --git a/apps/undefined-chat/src/App.test.tsx b/apps/undefined-chat/src/App.test.tsx index 1758373f..799a9ffd 100644 --- a/apps/undefined-chat/src/App.test.tsx +++ b/apps/undefined-chat/src/App.test.tsx @@ -165,12 +165,14 @@ describe("App", () => { test("saves initial Runtime config and API key before reconnecting", async () => { const client = runtimeClientStub({ getRuntimeConfig: vi - .fn() + .fn( + async (): Promise => ({ + runtimeUrl: "http://127.0.0.1:8788", + hasApiKey: true, + }), + ) .mockResolvedValueOnce(null) - .mockResolvedValueOnce({ - runtimeUrl: "http://127.0.0.1:8788", - hasApiKey: true, - }), + .mockResolvedValueOnce(null), }); vi.mocked(createTauriRuntimeClient).mockReturnValue(client); @@ -195,12 +197,14 @@ describe("App", () => { test("requires an explicit opt-in before confirming insecure API key storage fallback", async () => { const client = runtimeClientStub({ getRuntimeConfig: vi - .fn() + .fn( + async (): Promise => ({ + runtimeUrl: "http://127.0.0.1:8788", + hasApiKey: true, + }), + ) .mockResolvedValueOnce(null) - .mockResolvedValueOnce({ - runtimeUrl: "http://127.0.0.1:8788", - hasApiKey: true, - }), + .mockResolvedValueOnce(null), }); vi.mocked(createTauriRuntimeClient).mockReturnValue(client); @@ -232,6 +236,137 @@ describe("App", () => { expect(await screen.findByRole("button", { name: /新会话/ })).toBeTruthy(); }); + test("restores the previous Native target and blocks sending after key storage fails", async () => { + let nativeConfig: RuntimeConfig = { + runtimeUrl: "http://runtime-a.test", + hasApiKey: true, + }; + const client = runtimeClientStub({ + getRuntimeConfig: vi.fn(async () => nativeConfig), + saveRuntimeConfig: vi.fn(async (runtimeUrl) => { + nativeConfig = { ...nativeConfig, runtimeUrl }; + return nativeConfig; + }), + saveApiKey: vi.fn(async () => { + throw new Error("keyring denied"); + }), + }); + vi.mocked(createTauriRuntimeClient).mockReturnValue(client); + renderApp(); + await screen.findByLabelText("消息输入"); + await userEvent.type(screen.getByLabelText("消息输入"), "A draft"); + fireEvent.keyDown(window, { key: ",", ctrlKey: true }); + await userEvent.type(screen.getByLabelText("API Key"), "B-key"); + await userEvent.clear(screen.getByLabelText("Runtime URL")); + await userEvent.type( + screen.getByLabelText("Runtime URL"), + "http://runtime-b.test", + ); + await userEvent.click(screen.getByRole("button", { name: "保存并连接" })); + await screen.findByText(/keyring denied/); + expect(nativeConfig.runtimeUrl).toBe("http://runtime-a.test"); + expect(client.saveRuntimeConfig).toHaveBeenLastCalledWith( + "http://runtime-a.test", + ); + expect(screen.getByLabelText("Runtime URL")).toHaveValue( + "http://runtime-a.test", + ); + expect(screen.getByLabelText("消息输入")).toHaveValue("A draft"); + expect(screen.getByLabelText("消息输入")).toBeDisabled(); + fireEvent.keyDown(window, { key: "Escape" }); + expect(screen.getByLabelText("Runtime URL")).toBeInTheDocument(); + expect(client.sendMessage).not.toHaveBeenCalled(); + expect(client.listConversations).toHaveBeenCalledTimes(1); + // 恢复界面不能留空沿用未知状态的密钥;明确重新保存后才解除暂停。 + await userEvent.click(screen.getByRole("button", { name: "保存并连接" })); + expect(await screen.findByText("请输入 API Key")).toBeInTheDocument(); + client.saveApiKey = vi.fn(async () => client.loadApiKeyStatus()); + await userEvent.type(screen.getByLabelText("API Key"), "A-key"); + await userEvent.click(screen.getByRole("button", { name: "保存并连接" })); + await waitFor(() => { + expect(screen.queryByLabelText("Runtime URL")).toBeNull(); + expect(screen.getByLabelText("消息输入")).toBeEnabled(); + }); + expect(screen.getByLabelText("消息输入")).toHaveValue("A draft"); + await userEvent.click(screen.getByRole("button", { name: "发送" })); + expect(client.sendMessage).toHaveBeenCalledWith({ + conversationId: "default", + message: { text: "A draft", attachmentIds: [], references: [] }, + }); + }); + + test("clears a first configuration after key storage fails", async () => { + let nativeConfig: RuntimeConfig | null = null; + const client = runtimeClientStub({ + getRuntimeConfig: vi.fn(async () => nativeConfig), + saveRuntimeConfig: vi.fn(async (runtimeUrl) => { + nativeConfig = { runtimeUrl, hasApiKey: false }; + return nativeConfig; + }), + clearRuntimeConfig: vi.fn(async () => { + nativeConfig = null; + }), + saveApiKey: vi.fn(async () => { + throw new Error("keyring denied"); + }), + }); + vi.mocked(createTauriRuntimeClient).mockReturnValue(client); + renderApp(); + await screen.findByLabelText("Runtime URL"); + await userEvent.type(screen.getByLabelText("API Key"), "B-key"); + await userEvent.click(screen.getByRole("button", { name: "保存并连接" })); + await screen.findByText(/keyring denied/); + expect(nativeConfig).toBeNull(); + expect(client.clearRuntimeConfig).toHaveBeenCalledOnce(); + expect(client.listConversations).not.toHaveBeenCalled(); + expect(screen.getByLabelText("消息输入")).toBeDisabled(); + }); + + test("shows the actual Native target and requires recovery if URL rollback fails", async () => { + let nativeConfig: RuntimeConfig = { + runtimeUrl: "http://runtime-a.test", + hasApiKey: true, + }; + const client = runtimeClientStub({ + getRuntimeConfig: vi.fn(async () => nativeConfig), + saveRuntimeConfig: vi.fn(async (runtimeUrl) => { + if (runtimeUrl === "http://runtime-a.test") { + throw new Error("restore denied"); + } + nativeConfig = { ...nativeConfig, runtimeUrl }; + return nativeConfig; + }), + saveApiKey: vi.fn(async () => { + // 模拟底层已经更新密钥但回执失败;客户端不得推断旧密钥仍安全。 + nativeConfig = { ...nativeConfig, hasApiKey: true }; + throw new Error("key write receipt failed"); + }), + }); + vi.mocked(createTauriRuntimeClient).mockReturnValue(client); + renderApp(); + await screen.findByLabelText("消息输入"); + await userEvent.type(screen.getByLabelText("消息输入"), "private A draft"); + fireEvent.keyDown(window, { key: ",", ctrlKey: true }); + await userEvent.type(screen.getByLabelText("API Key"), "B-key"); + await userEvent.clear(screen.getByLabelText("Runtime URL")); + await userEvent.type( + screen.getByLabelText("Runtime URL"), + "http://runtime-b.test", + ); + await userEvent.click(screen.getByRole("button", { name: "保存并连接" })); + await screen.findByText(/restore denied/); + expect(nativeConfig.runtimeUrl).toBe("http://runtime-b.test"); + expect(screen.getByLabelText("Runtime URL")).toHaveValue( + "http://runtime-b.test", + ); + expect(screen.getByLabelText("消息输入")).toHaveValue(""); + expect(screen.getByLabelText("消息输入")).toBeDisabled(); + fireEvent.keyDown(window, { key: "Escape" }); + expect(screen.getByLabelText("API Key")).toBeInTheDocument(); + expect(client.sendMessage).not.toHaveBeenCalled(); + expect(client.listConversations).toHaveBeenCalledTimes(1); + }); + test("opens and closes the mobile conversation drawer with accessible state", async () => { const originalInnerWidth = window.innerWidth; Object.defineProperty(window, "innerWidth", { diff --git a/apps/undefined-chat/src/App.tsx b/apps/undefined-chat/src/App.tsx index 375b3e42..20a120f2 100644 --- a/apps/undefined-chat/src/App.tsx +++ b/apps/undefined-chat/src/App.tsx @@ -29,7 +29,7 @@ import { } from "./platform/PlatformContext"; import { AttachmentImageProvider } from "./rendering/AttachmentImageContext"; import { createTauriRuntimeClient } from "./runtime-client/tauri"; -import type { Attachment } from "./runtime-client/types"; +import type { Attachment, RuntimeConfig } from "./runtime-client/types"; import { useTheme } from "./theme/use-theme"; import { isImageAttachment } from "./utils/attachment"; @@ -89,6 +89,11 @@ export function App() { : null; const [setupError, setSetupError] = useState(null); + const [isConnecting, setIsConnecting] = useState(false); + const [connectionRecoveryGeneration, setConnectionRecoveryGeneration] = + useState(0); + const connectionRecoveryRequired = connectionRecoveryGeneration > 0; + const connectionUpdateInFlightRef = useRef(false); const needsSetup = !state.runtimeConfig?.runtimeUrl || !state.runtimeConfig.hasApiKey; @@ -127,9 +132,18 @@ export function App() { apiKey: string, allowInsecure: boolean, ): Promise { + if (connectionUpdateInFlightRef.current) return; + connectionUpdateInFlightRef.current = true; + setIsConnecting(true); setSetupError(null); + let previousConfig: RuntimeConfig | null = null; + let savedConfig: RuntimeConfig | null = null; + let writeStarted = false; try { - const config = await client.saveRuntimeConfig(url); + previousConfig = await client.getRuntimeConfig(); + store.suspendRuntime(); + writeStarted = true; + savedConfig = await client.saveRuntimeConfig(url); if (allowInsecure) { await client.confirmInsecureStorageFallback(); } @@ -137,10 +151,50 @@ export function App() { if (apiKey) { await client.saveApiKey(apiKey); } + await store.bootstrap({ + runtimeUrl: savedConfig.runtimeUrl, + resumeSuspended: true, + }); + if (store.getSnapshot().error) { + setSetupError(store.getSnapshot().error); + store.suspendRuntime(savedConfig); + setConnectionRecoveryGeneration((generation) => generation + 1); + return; + } + setConnectionRecoveryGeneration(0); setIsSettingsOpen(false); - await store.bootstrap({ runtimeUrl: config.runtimeUrl }); } catch (err) { - setSetupError(err instanceof Error ? err.message : String(err)); + let detail = err instanceof Error ? err.message : String(err); + if (writeStarted) { + try { + if (previousConfig) { + await client.saveRuntimeConfig(previousConfig.runtimeUrl); + } else { + await client.clearRuntimeConfig(); + } + } catch (rollbackError) { + detail = t("setup.error.rollbackFailed", { + error: detail, + rollbackError: + rollbackError instanceof Error + ? rollbackError.message + : String(rollbackError), + }); + } + let actualConfig = savedConfig ?? previousConfig; + try { + actualConfig = await client.getRuntimeConfig(); + } catch { + // 无法读取时保留最后一次成功保存的目标,仍禁止 Runtime 请求。 + } + store.suspendRuntime(actualConfig); + setConnectionRecoveryGeneration((generation) => generation + 1); + detail = t("setup.error.reconfigure", { error: detail }); + } + setSetupError(detail); + } finally { + connectionUpdateInFlightRef.current = false; + setIsConnecting(false); } } @@ -231,7 +285,7 @@ export function App() { }); } - const showModal = needsSetup || isSettingsOpen; + const showModal = needsSetup || isSettingsOpen || connectionRecoveryRequired; const closeMobileSidebar = useCallback((): void => { setIsMobileSidebarActive(false); @@ -607,13 +661,20 @@ export function App() { {/* 模态框 / 配置面板:统一 ConnectionSetup(首次连接 / 运行期修改两种模式) */} {showModal ? ( { void handleConnect(url, apiKey, allowInsecure); }} onClose={ - needsSetup ? undefined : () => setIsSettingsOpen(false) + needsSetup || isConnecting || connectionRecoveryRequired + ? undefined + : () => setIsSettingsOpen(false) } error={setupError} > @@ -699,7 +760,13 @@ export function App() { : [] } commandSuggestions={state.commands} - disabled={isJobRunning(activeJob)} + disabled={ + isJobRunning(activeJob) || + isConnecting || + connectionRecoveryRequired || + state.bootstrapping || + state.connectionState === "disconnected" + } draft={ selectedConversationId ? (state.draftsByConversation[selectedConversationId] ?? "") diff --git a/apps/undefined-chat/src/chat-store/connection-recovery.test.ts b/apps/undefined-chat/src/chat-store/connection-recovery.test.ts new file mode 100644 index 00000000..97fa5732 --- /dev/null +++ b/apps/undefined-chat/src/chat-store/connection-recovery.test.ts @@ -0,0 +1,60 @@ +import { describe, expect, test, vi } from "vitest"; +import type { RuntimeConfig } from "../runtime-client/types"; +import { runtimeClientStub } from "../test-fixtures"; +import { createChatStore } from "./store"; + +describe("connection configuration suspension", () => { + test("blocks Runtime operations and lifecycle bootstrap until explicit connection recovery", async () => { + let runtimeConfig: RuntimeConfig = { + runtimeUrl: "http://runtime-a.test", + hasApiKey: true, + }; + const client = runtimeClientStub({ + getRuntimeConfig: vi.fn(async () => runtimeConfig), + }); + const store = createChatStore({ client }); + await store.bootstrap(); + store.updateDraft("default", "A draft"); + store.suspendRuntime(); + await Promise.all([ + store.sendSelectedMessage(), + store.createConversation(), + store.deleteConversation("default"), + store.renameConversation("default", "unsafe rename"), + store.selectConversation("default"), + store.addAttachmentPath("default", "/mock/a.txt"), + store.reloadHistory("default"), + store.loadMoreHistory("default"), + store.cancelJob("job-1"), + store.refreshCommandsIfStale(), + store.bootstrap({ resuming: true }), + ]); + expect(client.sendMessage).not.toHaveBeenCalled(); + expect(client.createConversation).not.toHaveBeenCalled(); + expect(client.deleteConversation).not.toHaveBeenCalled(); + expect(client.renameConversation).not.toHaveBeenCalled(); + expect(client.uploadAttachment).not.toHaveBeenCalled(); + expect(client.cancelJob).not.toHaveBeenCalled(); + expect(client.getHistory).toHaveBeenCalledTimes(1); + expect(client.listCommands).toHaveBeenCalledTimes(1); + expect(client.getRuntimeConfig).toHaveBeenCalledTimes(1); + expect(store.getSnapshot().draftsByConversation.default).toBe("A draft"); + await store.bootstrap({ resumeSuspended: true }); + await store.sendSelectedMessage(); + expect(client.sendMessage).toHaveBeenCalledWith({ + conversationId: "default", + message: { text: "A draft", attachmentIds: [], references: [] }, + }); + runtimeConfig = { ...runtimeConfig, runtimeUrl: "http://runtime-b.test" }; + store.suspendRuntime(runtimeConfig); + expect(store.getSnapshot().runtimeConfig).toEqual(runtimeConfig); + expect(store.getSnapshot().conversations).toEqual([]); + expect(store.getSnapshot().draftsByConversation).toEqual({}); + expect(store.getSnapshot().connectionState).toBe("disconnected"); + await store.bootstrap(); + expect(client.getRuntimeConfig).toHaveBeenCalledTimes(2); + await store.bootstrap({ resumeSuspended: true }); + expect(store.getSnapshot().runtimeConfig).toEqual(runtimeConfig); + expect(store.getSnapshot().connectionState).toBe("connected"); + }); +}); diff --git a/apps/undefined-chat/src/chat-store/store.ts b/apps/undefined-chat/src/chat-store/store.ts index 45c029c4..b68de85c 100644 --- a/apps/undefined-chat/src/chat-store/store.ts +++ b/apps/undefined-chat/src/chat-store/store.ts @@ -1,6 +1,7 @@ import type { HistoryItem, RuntimeClient, + RuntimeConfig, RuntimeSseEvent, RuntimeSseStatus, SendMessageInput, @@ -29,7 +30,9 @@ export type ChatStore = { runtimeUrl?: string; preserveSelectionId?: string; resuming?: boolean; + resumeSuspended?: boolean; }) => Promise; + suspendRuntime: (runtimeConfig?: RuntimeConfig | null) => void; refreshCommandsIfStale: () => Promise; createConversation: (title?: string) => Promise; deleteConversation: (conversationId: string) => Promise; @@ -672,6 +675,7 @@ export function createChatStore({ let state = createInitialChatState(); let runtimeUrl: string | null = null; let runtimeGeneration = 0; + let runtimeSuspended = false; const listeners = new Set<() => void>(); const subscriptionsByJob = new Map(); const fallbackTimersByJob = new Map>(); @@ -731,7 +735,27 @@ export function createChatStore({ }; } + function suspendRuntime(runtimeConfig = state.runtimeConfig): void { + if (runtimeConfig?.runtimeUrl && runtimeConfig.runtimeUrl !== runtimeUrl) { + resetRuntimeScope(runtimeConfig.runtimeUrl); + } else { + runtimeGeneration += 1; + clearNativeSubscriptions(); + sendInFlight = false; + } + runtimeSuspended = true; + inFlightBootstrap = null; + state = { + ...state, + runtimeConfig, + bootstrapping: false, + connectionState: "disconnected", + }; + emit(); + } + async function loadHistory(conversationId: string): Promise { + if (runtimeSuspended) return; const generation = runtimeGeneration; dispatch({ type: "history/loading", conversationId }); try { @@ -760,6 +784,7 @@ export function createChatStore({ } async function startStreamForJob(job: ChatJob): Promise { + if (runtimeSuspended) return; const generation = runtimeGeneration; if (subscriptionsByJob.has(job.jobId)) { return; @@ -926,13 +951,14 @@ export function createChatStore({ resuming = false, ): Promise { let generation = runtimeGeneration; + let runtimeConfig: RuntimeConfig | null = null; dispatch({ type: "bootstrap/start" }); // Android 切后台返回等续接场景:标记为 resuming,连接成功后由后续状态覆盖 if (resuming) { dispatch({ type: "connection/set", connectionState: "resuming" }); } try { - const runtimeConfig = await client.getRuntimeConfig(); + runtimeConfig = await client.getRuntimeConfig(); if (generation !== runtimeGeneration) return; if ( runtimeConfig?.runtimeUrl && @@ -960,6 +986,7 @@ export function createChatStore({ ]); if (generation !== runtimeGeneration) return; lastCommandsFetchedAt = Date.now(); + runtimeSuspended = false; // 仅选取真实存在于会话列表中的会话:defaultConversationId 是后端常量 // (legacy-system-42),未必对应已存在的会话,盲目用它会触发 getHistory 404 const conversationIds = new Set( @@ -1017,7 +1044,11 @@ export function createChatStore({ } } catch (err) { if (generation !== runtimeGeneration) return; - dispatch({ type: "bootstrap/error", error: errorMessage(err) }); + dispatch({ + type: "bootstrap/error", + runtimeConfig, + error: errorMessage(err), + }); } } @@ -1025,7 +1056,9 @@ export function createChatStore({ runtimeUrl?: string; preserveSelectionId?: string; resuming?: boolean; + resumeSuspended?: boolean; }): Promise { + if (runtimeSuspended && !opts?.resumeSuspended) return; if (opts?.runtimeUrl && opts.runtimeUrl !== runtimeUrl) { resetRuntimeScope(opts.runtimeUrl); inFlightBootstrap = null; @@ -1049,6 +1082,7 @@ export function createChatStore({ * 使 Skills 热重载新增的命令可见。未连接或未配置时跳过。 */ async function refreshCommandsIfStale(): Promise { + if (runtimeSuspended) return; const generation = runtimeGeneration; if (!state.runtimeConfig?.hasApiKey || state.bootstrapping) { return; @@ -1067,6 +1101,7 @@ export function createChatStore({ } async function createConversation(title?: string): Promise { + if (runtimeSuspended) return; const generation = runtimeGeneration; dispatch({ type: "conversation/creating", creating: true }); try { @@ -1085,6 +1120,7 @@ export function createChatStore({ } async function deleteConversation(conversationId: string): Promise { + if (runtimeSuspended) return; const generation = runtimeGeneration; const wasSelected = state.selectedConversationId === conversationId; const nextSelectedId = wasSelected @@ -1112,6 +1148,7 @@ export function createChatStore({ conversationId: string, newTitle: string, ): Promise { + if (runtimeSuspended) return; const generation = runtimeGeneration; try { await client.renameConversation(conversationId, newTitle); @@ -1133,6 +1170,7 @@ export function createChatStore({ } async function selectConversation(conversationId: string): Promise { + if (runtimeSuspended) return; if (state.selectedConversationId === conversationId) { return; } @@ -1147,6 +1185,7 @@ export function createChatStore({ } async function sendSelectedMessage(): Promise { + if (runtimeSuspended) return; const generation = runtimeGeneration; // in-flight 锁:从点击到 job 建立期间阻止重复发送, // 避免快速双击重复插入乐观消息 / 重复请求后端 @@ -1247,6 +1286,7 @@ export function createChatStore({ } async function cancelJob(jobId: string): Promise { + if (runtimeSuspended) return; const generation = runtimeGeneration; const conversationId = state.jobConversationById[jobId]; if (!conversationId) { @@ -1286,6 +1326,7 @@ export function createChatStore({ conversationId: string, filePath: string, ): Promise { + if (runtimeSuspended) return; const generation = runtimeGeneration; const trimmed = filePath.trim(); if (!trimmed) { @@ -1635,6 +1676,7 @@ export function createChatStore({ } async function loadMoreHistory(conversationId: string): Promise { + if (runtimeSuspended) return; const generation = runtimeGeneration; const historyState = state.historyByConversation[conversationId]; if (!historyState?.hasMore || historyState.loading) { @@ -1673,6 +1715,7 @@ export function createChatStore({ return { bootstrap, + suspendRuntime, refreshCommandsIfStale, createConversation, deleteConversation, diff --git a/apps/undefined-chat/src/connection-recovery.test.tsx b/apps/undefined-chat/src/connection-recovery.test.tsx new file mode 100644 index 00000000..0f4cb882 --- /dev/null +++ b/apps/undefined-chat/src/connection-recovery.test.tsx @@ -0,0 +1,108 @@ +import { invoke } from "@tauri-apps/api/core"; +import { listen } from "@tauri-apps/api/event"; +import { fireEvent, render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { beforeEach, describe, expect, test, vi } from "vitest"; +import { App } from "./App"; +import { LanguageProvider, LOCALE_STORAGE_KEY } from "./i18n"; + +vi.mock("@tauri-apps/api/core", () => ({ invoke: vi.fn() })); +vi.mock("@tauri-apps/api/event", () => ({ listen: vi.fn() })); + +describe("connection recovery through the real Tauri client and store", () => { + beforeEach(() => { + vi.resetAllMocks(); + localStorage.clear(); + localStorage.setItem(LOCALE_STORAGE_KEY, "zh-CN"); + vi.mocked(listen).mockResolvedValue(() => undefined); + }); + + test.each([false, true])( + "key failure with rollback failure=%s keeps UI and Native target consistent and prevents Runtime requests", + async (rollbackFails) => { + let nativeUrl = "http://runtime-a.test"; + const runtimeCalls: Array<{ command: string; origin: string }> = []; + vi.mocked(invoke).mockImplementation(async (command, args) => { + switch (command) { + case "get_runtime_config": + return { runtimeUrl: nativeUrl }; + case "load_api_key_status": + return { available: true, storage: "system-keyring" }; + case "save_runtime_config": { + const { input } = args as { input: { runtimeUrl: string } }; + if (rollbackFails && input.runtimeUrl === "http://runtime-a.test") { + throw new Error("rollback denied"); + } + nativeUrl = input.runtimeUrl; + return { runtimeUrl: nativeUrl }; + } + case "save_api_key": + throw new Error("keyring denied"); + } + runtimeCalls.push({ command, origin: nativeUrl }); + switch (command) { + case "probe_runtime": + return { status: 200, ok: true, body: "ok" }; + case "list_conversations": + return { + conversations: [ + { id: "legacy-system-42", title: "A conversation" }, + ], + default_conversation_id: "legacy-system-42", + }; + case "get_history": + return { + conversation_id: "legacy-system-42", + items: [ + { message_id: "a-history", role: "user", content: "A history" }, + ], + }; + case "get_active_jobs": + return { jobs: [] }; + case "list_commands": + return { commands: [] }; + default: + throw new Error(`Unexpected Native command: ${command}`); + } + }); + render( + + + , + ); + await screen.findByText("A history"); + await userEvent.type( + screen.getByLabelText("消息输入"), + "A private draft", + ); + fireEvent.keyDown(window, { key: ",", ctrlKey: true }); + await userEvent.clear(screen.getByLabelText("Runtime URL")); + await userEvent.type( + screen.getByLabelText("Runtime URL"), + "http://runtime-b.test", + ); + await userEvent.type(screen.getByLabelText("API Key"), "B-key"); + await userEvent.click(screen.getByRole("button", { name: "保存并连接" })); + await screen.findByText(/keyring denied/); + expect(nativeUrl).toBe( + rollbackFails ? "http://runtime-b.test" : "http://runtime-a.test", + ); + expect(screen.getByLabelText("Runtime URL")).toHaveValue(nativeUrl); + expect(screen.getByLabelText("消息输入")).toBeDisabled(); + expect(screen.getByLabelText("消息输入")).toHaveValue( + rollbackFails ? "" : "A private draft", + ); + fireEvent.keyDown(window, { key: "Escape" }); + expect(screen.getByLabelText("Runtime URL")).toBeInTheDocument(); + fireEvent.keyDown(window, { key: "Enter", ctrlKey: true }); + expect(runtimeCalls.map(({ origin }) => origin)).toEqual( + Array(5).fill("http://runtime-a.test"), + ); + expect( + vi + .mocked(invoke) + .mock.calls.filter(([command]) => command === "send_message"), + ).toEqual([]); + }, + ); +}); diff --git a/apps/undefined-chat/src/i18n/index.ts b/apps/undefined-chat/src/i18n/index.ts index 014b5a87..a7a8a2cb 100644 --- a/apps/undefined-chat/src/i18n/index.ts +++ b/apps/undefined-chat/src/i18n/index.ts @@ -53,6 +53,8 @@ export type TranslationKey = | "setup.error.needUrl" | "setup.error.needApiKey" | "setup.error.invalidUrl" + | "setup.error.reconfigure" + | "setup.error.rollbackFailed" // 附件操作(App 级 alert / 选择器) | "attachment.saveFailed" | "attachment.saved" @@ -224,6 +226,10 @@ export const dictionaries: Record = { "setup.error.needUrl": "请输入 Runtime URL", "setup.error.needApiKey": "请输入 API Key", "setup.error.invalidUrl": "URL 格式不正确", + "setup.error.reconfigure": + "连接配置保存失败,请重新输入 API Key 并连接。{error}", + "setup.error.rollbackFailed": + "{error};恢复旧 Runtime URL 失败:{rollbackError}", "attachment.saveFailed": "附件保存失败", "attachment.saved": "已保存 {name}", "attachment.previewFailed": "附件无法预览", @@ -377,6 +383,10 @@ export const dictionaries: Record = { "setup.error.needUrl": "Please enter the Runtime URL", "setup.error.needApiKey": "Please enter the API Key", "setup.error.invalidUrl": "Invalid URL format", + "setup.error.reconfigure": + "Connection settings were not saved. Enter the API key and reconnect. {error}", + "setup.error.rollbackFailed": + "{error}; restoring the previous Runtime URL failed: {rollbackError}", "attachment.saveFailed": "Failed to save attachment", "attachment.saved": "Saved {name}", "attachment.previewFailed": "Attachment cannot be previewed", diff --git a/apps/undefined-chat/src/runtime-client/tauri.test.ts b/apps/undefined-chat/src/runtime-client/tauri.test.ts index b74a56af..74bb9470 100644 --- a/apps/undefined-chat/src/runtime-client/tauri.test.ts +++ b/apps/undefined-chat/src/runtime-client/tauri.test.ts @@ -17,6 +17,23 @@ describe("createTauriRuntimeClient", () => { vi.resetAllMocks(); }); + test("clears the persisted target through the registered Native command", async () => { + vi.mocked(invoke).mockResolvedValueOnce(undefined); + await createTauriRuntimeClient().clearRuntimeConfig(); + expect(invoke).toHaveBeenCalledWith("clear_runtime_config"); + }); + + test("does not persist a URL if the API key status request rejects", async () => { + vi.mocked(invoke).mockRejectedValueOnce( + new Error("key status unavailable"), + ); + await expect( + createTauriRuntimeClient().saveRuntimeConfig("http://runtime-b.test"), + ).rejects.toThrow("key status unavailable"); + expect(invoke).toHaveBeenCalledTimes(1); + expect(invoke).toHaveBeenCalledWith("load_api_key_status"); + }); + test("normalizes conversation and active job responses from Tauri commands", async () => { vi.mocked(invoke).mockResolvedValueOnce({ status: 200, diff --git a/apps/undefined-chat/src/runtime-client/tauri.ts b/apps/undefined-chat/src/runtime-client/tauri.ts index 0bdedde8..cf772dc9 100644 --- a/apps/undefined-chat/src/runtime-client/tauri.ts +++ b/apps/undefined-chat/src/runtime-client/tauri.ts @@ -464,12 +464,11 @@ export function createTauriRuntimeClient(): RuntimeClient { }, async saveRuntimeConfig(runtimeUrl: string): Promise { - const [config, apiKeyStatus] = await Promise.all([ - invoke("save_runtime_config", { - input: { runtimeUrl }, - }), - this.loadApiKeyStatus(), - ]); + // 先确认密钥状态可读取,避免并行读取失败时 URL 已保存却丢失回执。 + const apiKeyStatus = await this.loadApiKeyStatus(); + const config = await invoke("save_runtime_config", { + input: { runtimeUrl }, + }); const normalized = normalizeRuntimeConfig(config, apiKeyStatus); if (!normalized) { throw new Error("Runtime config response did not include runtimeUrl"); @@ -477,6 +476,10 @@ export function createTauriRuntimeClient(): RuntimeClient { return normalized; }, + async clearRuntimeConfig(): Promise { + await invoke("clear_runtime_config"); + }, + async saveApiKey(apiKey: string): Promise { return normalizeApiKeyStatus(await invoke("save_api_key", { apiKey })); }, diff --git a/apps/undefined-chat/src/runtime-client/types.ts b/apps/undefined-chat/src/runtime-client/types.ts index 949c19ac..59021848 100644 --- a/apps/undefined-chat/src/runtime-client/types.ts +++ b/apps/undefined-chat/src/runtime-client/types.ts @@ -293,6 +293,7 @@ export type EventStreamSubscription = { export type RuntimeClient = { getRuntimeConfig: () => Promise; saveRuntimeConfig: (runtimeUrl: string) => Promise; + clearRuntimeConfig: () => Promise; saveApiKey: (apiKey: string) => Promise; confirmInsecureStorageFallback: () => Promise; loadApiKeyStatus: () => Promise; diff --git a/apps/undefined-chat/src/test-fixtures.ts b/apps/undefined-chat/src/test-fixtures.ts index e10b778e..08ac82b8 100644 --- a/apps/undefined-chat/src/test-fixtures.ts +++ b/apps/undefined-chat/src/test-fixtures.ts @@ -153,6 +153,7 @@ export function runtimeClientStub( runtimeUrl, hasApiKey: true, })), + clearRuntimeConfig: vi.fn(async () => undefined), saveApiKey: vi.fn(async () => apiKeyStatus), confirmInsecureStorageFallback: vi.fn(async () => ({ ...apiKeyStatus, diff --git a/apps/undefined-chat/tests/e2e/connection-setup.test.tsx b/apps/undefined-chat/tests/e2e/connection-setup.test.tsx index 28cfefaa..0f3c4ffa 100644 --- a/apps/undefined-chat/tests/e2e/connection-setup.test.tsx +++ b/apps/undefined-chat/tests/e2e/connection-setup.test.tsx @@ -3,6 +3,7 @@ import userEvent from "@testing-library/user-event"; import { beforeEach, describe, expect, test, vi } from "vitest"; import { App } from "../../src/App"; import { createTauriRuntimeClient } from "../../src/runtime-client/tauri"; +import type { RuntimeConfig } from "../../src/runtime-client/types"; import { runtimeClientStub } from "../../src/test-fixtures"; import { renderWithProviders } from "../../src/test-utils"; @@ -50,12 +51,12 @@ describe("E2E: Connection Setup Flow", () => { test("保存配置并在连接后引导应用", async () => { const client = runtimeClientStub({ getRuntimeConfig: vi - .fn() - .mockResolvedValueOnce(null) - .mockResolvedValueOnce({ + .fn(async (): Promise => ({ runtimeUrl: "http://localhost:8788", hasApiKey: true, - }), + })) + .mockResolvedValueOnce(null) + .mockResolvedValueOnce(null), }); vi.mocked(createTauriRuntimeClient).mockReturnValue(client); @@ -118,12 +119,12 @@ describe("E2E: Connection Setup Flow", () => { test("支持不安全存储降级选项", async () => { const client = runtimeClientStub({ getRuntimeConfig: vi - .fn() - .mockResolvedValueOnce(null) - .mockResolvedValueOnce({ + .fn(async (): Promise => ({ runtimeUrl: "http://localhost:8788", hasApiKey: true, - }), + })) + .mockResolvedValueOnce(null) + .mockResolvedValueOnce(null), }); vi.mocked(createTauriRuntimeClient).mockReturnValue(client); @@ -168,7 +169,9 @@ describe("E2E: Connection Setup Flow", () => { await userEvent.type(keyInput, "sk-test"); await userEvent.click(connectBtn); - expect(await screen.findByText("网络连接失败")).toBeInTheDocument(); + expect(await screen.findByText(/网络连接失败/)).toBeInTheDocument(); + expect(client.clearRuntimeConfig).toHaveBeenCalledOnce(); + expect(screen.getByLabelText("消息输入")).toBeDisabled(); }); test("已有配置时可以通过设置按钮重新打开配置面板", async () => { diff --git a/docs/undefined-chat.md b/docs/undefined-chat.md index d6b4de95..251070cf 100644 --- a/docs/undefined-chat.md +++ b/docs/undefined-chat.md @@ -56,6 +56,8 @@ Undefined Chat 不持久化权威聊天数据。以下内容都以 Runtime 返 Runtime 请求必须由 Tauri command 拼接到已配置 origin 下,避免 React 传入任意 URL 后绕过目标主机限制。React 不直接持有 API Key;受保护请求由 Rust 侧注入 `X-Undefined-API-Key`。 +保存连接配置时会暂停 Runtime 请求及任务订阅。URL 保存后,密钥或降级确认失败会恢复先前 URL(首次配置则清除新 URL);客户端显示原生端实际保存的目标,并保持不可发送的配置界面,要求重新输入密钥后连接。恢复 URL 失败时也保持此限制;原生配置无法读取时保留最后确认的目标并继续禁用请求。不会用旧会话界面操作新目标,或假定失败的密钥写入一定没有副作用。 + 事件策略: - 默认使用 SSE 订阅 job events。 From 09665d16956c9e0124b33e92ff1b61263b0404ff Mon Sep 17 00:00:00 2001 From: Null <1708213363@qq.com> Date: Tue, 6 Oct 2026 13:21:52 +0800 Subject: [PATCH 044/121] fix(webui): ignore stale chat job poll completions --- docs/webui-guide.md | 1 + src/Undefined/webui/static/js/runtime.js | 30 ++--- tests/frontend/runtime_chat_harness.js | 139 ++++++++++++++++++++++ tests/test_webui_runtime_chat_behavior.py | 17 +++ 4 files changed, 170 insertions(+), 17 deletions(-) diff --git a/docs/webui-guide.md b/docs/webui-guide.md index d9a54fb4..3509894e 100644 --- a/docs/webui-guide.md +++ b/docs/webui-guide.md @@ -161,6 +161,7 @@ WebUI 内置的对话界面,直接与 Bot 的 AI 进行交互: - AI 消息支持引用:可引用整条 AI 消息、选中的某段文字,或 HTML 预览中点选的元素。引用会显示在输入框右侧的待发送引用条中,可在发送前移除;实际发送时不会新增接口或附件,而是自动在用户消息前拼接为 Markdown `>` 引用块,例如 `> 引用 AI:`,随后和文本、图片、文件一起进入同一条 WebChat job 消息。发送后的引用块会像代码块一样折叠显示,展开后内部固定高度并可滚动。发送失败时待引用内容会保留,发送成功后清空。 - 默认加载最新 50 条消息并滚动到底部;向上滚动到顶部会按后端返回的 `next_before` 游标懒加载更早历史,并保持原视口偏移,避免一次性恢复大量工具块造成卡顿。“自动滚动到底部”开关默认开启并保存在浏览器本地,关闭后 AI 回复、工具 / Agent 状态和 AI 阶段刷新不会打断当前位置;首次加载和主动发送新消息仍会定位到底部。系统开启减少动态效果时,聊天滚动会改为即时跳转。 - 对话由 WebChat job 执行。刷新页面、关闭页面、网络短暂中断或换另一个客户端重新访问 WebUI 时,后端任务继续运行,前端会从后端查询会话列表、当前运行 job,并用 `conversation_id + job_id + seq` 每 0.5 秒轮询增量事件、当前阶段快照和运行中工具 / Agent 快照自动续接;前端不会把关键任务状态只存放在浏览器本地,历史、会话、运行中 job 和事件游标都以 Runtime API 返回值为准。如果刷新后首次查询运行中 job 失败,WebUI 会退避重试并在网络恢复时再次尝试;`done` / `error` / `cancelled` 终态先到时仍保留 job 和发送锁,并继续轮询等待 `history_finalized=true`;确认后端收尾完成后刷新权威历史,刷新成功才解除发送锁;历史慢响应或刷新失败期间继续保留锁,并通过原轮询重试恢复。SSE 仅作为兼容方式保留,WebUI 不依赖长连接。 +- 切换对话后再发起新任务时,迟到的旧任务轮询响应不会覆盖新回复、改变新任务的发送锁和取消按钮,或打断新任务续接;旧请求的失败提示也不会影响当前任务。 - 运行中的 WebChat job 可从正在生成的 AI 气泡中取消。取消会调用 Runtime 的 job cancel 接口,等待历史收尾完成后解除当前会话的发送锁;如果取消发生在 AI 尚未产生任何回复内容前,前端会移除空 AI 气泡,并在最后一条用户消息上显示“重试”。点击重试会带 `reuse_previous_user_message=true` 用同一条纯文本重新发起 WebChat job,前端不再追加一条相同用户气泡,后端也会校验并复用最后一条可见用户历史,避免把同一条输入重复写入会话。 - 运行中的 AI 气泡会在 `AI` 标签后实时显示当前阶段和后端计算的总已用时,例如构建上下文、查长期记忆、查认知记忆、等待模型、等待工具、发送消息;任务完成后该位置显示本轮回复总用时。工具 / Agent 摘要行会在名称旁显示调用耗时,运行中先显示后端快照时间,轮询间隙用本地时间临时递增,下一次查询后自动校准;结束后固定显示后端结束事件的总耗时。轮询刷新只更新已有状态和计时节点,结构未变化时不会重建工具 / Agent 块,避免运行中闪烁。状态条区分运行中、成功和失败;整轮回复总耗时会写入 WebChat 历史 metadata。 - WebChat job 事件会更新同一个 AI 气泡;AI 正文和主对话工具 / Agent 调用按事件时序显示,例如工具调用、正文回复、结束工具会依次出现在同一气泡内。 diff --git a/src/Undefined/webui/static/js/runtime.js b/src/Undefined/webui/static/js/runtime.js index a6102a27..0ef38567 100644 --- a/src/Undefined/webui/static/js/runtime.js +++ b/src/Undefined/webui/static/js/runtime.js @@ -5387,6 +5387,7 @@ } function applyChatEventsPayload(data, jobId) { + if (runtimeState.activeJobId !== jobId) return; const events = data && Array.isArray(data.events) ? data.events : []; const job = data && data.job ? data.job : null; events @@ -5480,27 +5481,22 @@ format: "json", }), ]); + if (runtimeState.activeJobId !== jobId) return; runtimeState.chatPollBackoffMs = CHAT_POLL_INTERVAL_MS; applyChatEventsPayload(data, jobId); } catch (error) { - if (runtimeState.activeJobId === jobId) { - showToast(t("runtime.chat_reconnecting"), "warning", 1800); - runtimeState.chatPollBackoffMs = Math.min( - 8000, - Math.max( - CHAT_POLL_INTERVAL_MS, - runtimeState.chatPollBackoffMs * 1.6, - ), - ); - } else { - showToast( - `${t("runtime.failed")}: ${appendRuntimeApiHint(error.message || error)}`, - "error", - 5000, - ); - } + if (runtimeState.activeJobId !== jobId) return; + showToast(t("runtime.chat_reconnecting"), "warning", 1800); + runtimeState.chatPollBackoffMs = Math.min( + 8000, + Math.max( + CHAT_POLL_INTERVAL_MS, + runtimeState.chatPollBackoffMs * 1.6, + ), + ); } - if (runtimeState.activeJobId !== jobId || !runtimeState.chatBusy) { + if (runtimeState.activeJobId !== jobId) return; + if (!runtimeState.chatBusy) { stopChatPolling(); return; } diff --git a/tests/frontend/runtime_chat_harness.js b/tests/frontend/runtime_chat_harness.js index 4d930e27..233d816f 100644 --- a/tests/frontend/runtime_chat_harness.js +++ b/tests/frontend/runtime_chat_harness.js @@ -343,6 +343,7 @@ function createEnv(root, options = {}) { requests, requestDetails, setRoutes, + response, scrollActivity, scrollTopWrites, eventSourceConstructions, @@ -2875,6 +2876,144 @@ for (const status of ["done", "error", "cancelled"]) { SCENARIOS[`history_finalization_${status}`] = (env) => jobWaitsForHistoryFinalization(env, status); } +async function staleChatJobPoll(env, outcome) { + const { window, setRoutes, response } = env; + let jobPosts = 0; + let oldFinished = false; + let newPolls = 0; + const oldJob = { + job_id: "old", + conversation_id: "A", + status: "running", + last_seq: 2, + history_finalized: false, + }; + const newJob = { ...oldJob, job_id: "new" }; + setRoutes([ + { + match: "/chat/conversations", + reply: { body: { + conversations: [{ id: "A", title: "A" }, { id: "B", title: "B" }], + active_job: null, + } }, + }, + { + match: "/chat/history", + reply: (url) => ({ body: { + items: oldFinished && url.includes("conversation_id=A") + ? [{ role: "bot", content: "old-persisted" }] : [], + has_more: false, + next_before: null, + } }), + }, + { + match: "/chat/jobs/active", + reply: (url) => ({ body: { + job: url.includes("conversation_id=A") && jobPosts > 0 && !oldFinished + ? oldJob : null, + } }), + }, + { + match: "/jobs/old/events", + reply: { body: { job: oldJob, events: [ + { seq: 1, event: "meta", payload: { job_id: "old", conversation_id: "A" } }, + { seq: 2, event: "message", payload: { job_id: "old", conversation_id: "A", content: "old-live" } }, + ] } }, + }, + { + match: "/jobs/new/events", + reply: (url) => { + newPolls += 1; + const after = Number(new URL(url, window.location.href).searchParams.get("after")); + const events = [ + { seq: 1, event: "meta", payload: { job_id: "new", conversation_id: "A" } }, + { seq: 2, event: "message", payload: { job_id: "new", conversation_id: "A", content: "new-live" } }, + ...(newPolls > 1 ? [{ seq: 3, event: "message", payload: { job_id: "new", conversation_id: "A", content: "new-followup" } }] : []), + ].filter((event) => event.seq > after); + return { body: { job: newJob, events } }; + }, + }, + { + match: "/chat/jobs", + reply: () => ({ body: { + job_id: ++jobPosts === 1 ? "old" : "new", conversation_id: "A", + } }), + }, + ]); + const fetch = window.fetch; + const pending = []; + let deferOld = false; + window.fetch = (url, options) => { + if (deferOld && String(url).includes("/jobs/old/events")) { + return new Promise((resolve, reject) => pending.push({ url: String(url), resolve, reject })); + } + return fetch(url, options); + }; + window.RuntimeController.init(); + await window.RuntimeController.loadChatHistory(true); + const input = window.document.getElementById("runtimeChatInput"); + const button = window.document.getElementById("btnRuntimeChatSend"); + input.value = "first"; + button.click(); + await tick(); + deferOld = true; + window.document.querySelector('[data-conversation-select="B"]').click(); + await tick(); + window.document.querySelector('[data-conversation-select="A"]').click(); + await tick(); + if (pending.length !== 2) { + throw new Error(`expected two overlapping old job polls, got ${pending.length}`); + } + oldFinished = true; + pending[0].resolve(response(200, { + job: { ...oldJob, status: "done", last_seq: 3, history_finalized: true }, + events: [{ seq: 3, event: "done", payload: { job_id: "old", conversation_id: "A", reply: "old-live" } }], + })); + await tick(); + const oldUnlocked = !button.disabled; + input.value = "second"; + button.click(); + await tick(); + const snapshot = () => ({ + disabled: button.disabled, + nodes: chatNodes(window).map((node) => ({ text: node.contentText, job: node.jobId })), + cancelIds: Array.from(window.document.querySelectorAll("[data-cancel-job]")) + .filter((element) => !element.hidden).map((element) => element.dataset.cancelJob), + }); + const beforeLate = snapshot(); + if (outcome === "failure") { + pending[1].reject(new Error("obsolete poll failure")); + } else { + const terminal = outcome === "done"; + pending[1].resolve(response(200, { + job: terminal ? { ...oldJob, status: "done", history_finalized: true } : oldJob, + events: terminal ? [ + { seq: 99, event: "done", payload: { job_id: "old", conversation_id: "A", reply: "INTRUDER" } }, + ] : [ + { seq: 98, event: "meta", payload: { job_id: "old", conversation_id: "A" } }, + { seq: 99, event: "message", payload: { job_id: "old", conversation_id: "A", content: "INTRUDER" } }, + ], + })); + } + await tick(); + const afterLate = snapshot(); + await settle(650); + return { + oldUnlocked, + jobPosts, + beforeLate, + afterLate, + newPolls, + allNodes: chatNodes(window), + nextPollUrls: env.requests.filter((url) => url.includes("/jobs/new/events")), + toastText: textOf(window, "toast-container"), + }; +} + +for (const outcome of ["events", "done", "failure"]) { + SCENARIOS[`stale_job_poll_${outcome}`] = (env) => staleChatJobPoll(env, outcome); +} + async function main() { const args = parseArgs(process.argv.slice(2)); if (!args.scenario) throw new Error("缺少 --scenario"); diff --git a/tests/test_webui_runtime_chat_behavior.py b/tests/test_webui_runtime_chat_behavior.py index c1777bad..c1b44834 100644 --- a/tests/test_webui_runtime_chat_behavior.py +++ b/tests/test_webui_runtime_chat_behavior.py @@ -656,6 +656,23 @@ def test_auto_scroll_toggle_persists_preference() -> None: # --------------------------------------------------------------------------- # +@pytest.mark.parametrize("outcome", ["events", "done", "failure"]) +def test_late_old_job_poll_cannot_replace_or_stop_new_job(outcome: str) -> None: + result = run_scenario(f"stale_job_poll_{outcome}") + + assert result["oldUnlocked"] is True, result + assert result["jobPosts"] == 2, result + assert result["beforeLate"]["disabled"] is True, result + assert result["beforeLate"]["cancelIds"] == ["new"], result + assert result["afterLate"] == result["beforeLate"], result + assert result["newPolls"] >= 2, result + assert any("after=2&" in url for url in result["nextPollUrls"]), result + assert "obsolete poll failure" not in result["toastText"], result + bots = _bot_nodes(result) + assert [node["jobId"] for node in bots] == ["", "new"], bots + assert bots[-1]["contentTexts"] == ["new-live", "new-followup"], bots + + def test_foreign_conversation_events_are_ignored() -> None: """别的会话的事件不得写进当前会话的聊天区。 From 55483708540c7e4a13fc527e047f802a5c41a87d Mon Sep 17 00:00:00 2001 From: Null <1708213363@qq.com> Date: Tue, 6 Oct 2026 13:26:07 +0800 Subject: [PATCH 045/121] fix(chat): preserve later edits when message sending fails --- .../src/chat-store/send-recovery.test.ts | 239 ++++++++++++++++++ apps/undefined-chat/src/chat-store/store.ts | 36 ++- docs/undefined-chat.md | 2 + 3 files changed, 272 insertions(+), 5 deletions(-) create mode 100644 apps/undefined-chat/src/chat-store/send-recovery.test.ts diff --git a/apps/undefined-chat/src/chat-store/send-recovery.test.ts b/apps/undefined-chat/src/chat-store/send-recovery.test.ts new file mode 100644 index 00000000..3dc8679f --- /dev/null +++ b/apps/undefined-chat/src/chat-store/send-recovery.test.ts @@ -0,0 +1,239 @@ +import { describe, expect, test, vi } from "vitest"; +import type { Attachment, RuntimeClient } from "../runtime-client/types"; +import { conversation, job, runtimeClientStub } from "../test-fixtures"; +import { createChatStore } from "./store"; + +function deferred() { + let resolve!: (value: T) => void; + let reject!: (reason: Error) => void; + const promise = new Promise((done, fail) => { + resolve = done; + reject = fail; + }); + return { promise, resolve, reject }; +} + +function attachment(id: string): Attachment { + return { + id, + name: `${id}.txt`, + size: 4, + mediaType: "text/plain", + kind: "file", + downloadUrl: null, + previewUrl: null, + discarded: false, + }; +} + +describe("send failure recovery", () => { + test.each(["new draft", ""])( + "preserves later text edits including an intentional empty draft (%j)", + async (laterDraft) => { + const sent = deferred(); + const client = runtimeClientStub({ + sendMessage: vi + .fn() + .mockImplementationOnce(() => sent.promise) + .mockResolvedValueOnce(job()), + uploadAttachment: vi.fn(async () => attachment("file_a")), + }); + const store = createChatStore({ client }); + await store.bootstrap(); + store.updateDraft("default", " first message "); + store.addReference("default", { messageId: "a", quote: "first quote" }); + await store.addAttachmentPath("default", "/mock/a.txt"); + const original = store.getSnapshot(); + const sending = store.sendSelectedMessage(); + expect(client.sendMessage).toHaveBeenCalledWith({ + conversationId: "default", + message: { + text: "first message", + attachmentIds: ["file_a"], + references: [{ messageId: "a", quote: "first quote" }], + }, + }); + store.updateDraft("default", "typed after sending"); + store.updateDraft("default", laterDraft); + sent.reject(new Error("POST failed")); + await sending; + const state = store.getSnapshot(); + expect(state.draftsByConversation.default).toBe(laterDraft); + expect(state.attachmentsByConversation.default).toEqual( + original.attachmentsByConversation.default, + ); + expect(state.referencesByConversation.default).toEqual( + original.referencesByConversation.default, + ); + expect( + state.historyByConversation.default.items.some( + (item) => item.content === "first message", + ), + ).toBe(false); + expect(state.sendError).toBe("POST failed"); + await store.sendSelectedMessage(); + expect(client.sendMessage).toHaveBeenLastCalledWith({ + conversationId: "default", + message: { + text: laterDraft, + attachmentIds: ["file_a"], + references: [{ messageId: "a", quote: "first quote" }], + }, + }); + }, + ); + + test("preserves a new pending attachment and a new reference while restoring unchanged text", async () => { + const sent = deferred(); + const uploaded = deferred(); + const client = runtimeClientStub({ + sendMessage: vi.fn(() => sent.promise), + uploadAttachment: vi + .fn() + .mockResolvedValueOnce(attachment("file_a")) + .mockImplementationOnce(() => uploaded.promise), + }); + const store = createChatStore({ client }); + await store.bootstrap(); + store.updateDraft("default", " unchanged text "); + store.addReference("default", { messageId: "a", quote: "old quote" }); + await store.addAttachmentPath("default", "/mock/a.txt"); + const sending = store.sendSelectedMessage(); + const uploading = store.addAttachmentPath("default", "/mock/b.txt"); + store.addReference("default", { messageId: "b", quote: "new quote" }); + const edited = store.getSnapshot(); + sent.reject(new Error("POST failed")); + await sending; + expect(store.getSnapshot().draftsByConversation.default).toBe( + " unchanged text ", + ); + expect(store.getSnapshot().attachmentsByConversation.default).toEqual( + edited.attachmentsByConversation.default, + ); + expect(store.getSnapshot().referencesByConversation.default).toEqual([ + { messageId: "b", quote: "new quote" }, + ]); + uploaded.resolve(attachment("file_b")); + await uploading; + expect(store.getSnapshot().attachmentsByConversation.default).toMatchObject( + [{ name: "file_b.txt", status: "ready", attachmentId: "file_b" }], + ); + }); + + test("does not restore attachments or references after they were edited then cleared", async () => { + const sent = deferred(); + const client = runtimeClientStub({ + sendMessage: vi.fn(() => sent.promise), + }); + const store = createChatStore({ client }); + await store.bootstrap(); + store.updateDraft("default", "keep original"); + store.addReference("default", { messageId: "a", quote: "old quote" }); + await store.addAttachmentPath("default", "/mock/a.txt"); + const sending = store.sendSelectedMessage(); + await store.addAttachmentPath("default", "/mock/b.txt"); + const newAttachment = + store.getSnapshot().attachmentsByConversation.default[0]; + store.clearAttachment("default", newAttachment.id); + store.addReference("default", { messageId: "b", quote: "new quote" }); + store.clearReference("default", "b"); + sent.reject(new Error("POST failed")); + await sending; + expect(store.getSnapshot().draftsByConversation.default).toBe( + "keep original", + ); + expect(store.getSnapshot().attachmentsByConversation.default).toEqual([]); + expect(store.getSnapshot().referencesByConversation.default).toEqual([]); + }); + + test("editing another conversation still restores this conversation's unchanged fields", async () => { + const sent = deferred(); + const client = runtimeClientStub({ + sendMessage: vi.fn(() => sent.promise), + }); + const store = createChatStore({ client }); + await store.bootstrap(); + store.updateDraft("default", " exact original "); + const sending = store.sendSelectedMessage(); + store.updateDraft("other", "other conversation draft"); + sent.reject(new Error("POST failed")); + await sending; + expect(store.getSnapshot().draftsByConversation.default).toBe( + " exact original ", + ); + expect(store.getSnapshot().draftsByConversation.other).toBe( + "other conversation draft", + ); + }); + + test.each(["attachments", "references"])( + "an edited %s field does not stop restoration of the other unchanged field", + async (editedField) => { + const sent = deferred(); + const client = runtimeClientStub({ + sendMessage: vi.fn(() => sent.promise), + }); + const store = createChatStore({ client }); + await store.bootstrap(); + store.updateDraft("default", "original draft"); + store.addReference("default", { messageId: "a", quote: "old quote" }); + await store.addAttachmentPath("default", "/mock/a.txt"); + const original = store.getSnapshot(); + const sending = store.sendSelectedMessage(); + if (editedField === "attachments") { + await store.addAttachmentPath("default", "/mock/b.txt"); + } else { + store.addReference("default", { messageId: "b", quote: "new quote" }); + } + const edited = store.getSnapshot(); + sent.reject(new Error("POST failed")); + await sending; + const recovered = store.getSnapshot(); + expect(recovered.attachmentsByConversation.default).toEqual( + editedField === "attachments" + ? edited.attachmentsByConversation.default + : original.attachmentsByConversation.default, + ); + expect(recovered.referencesByConversation.default).toEqual( + editedField === "references" + ? edited.referencesByConversation.default + : original.referencesByConversation.default, + ); + }, + ); + + test.each([false, true])( + "removing a conversation clears its revision and rejects stale recovery (reimport=%s)", + async (reimport) => { + const sent = deferred(); + const client = runtimeClientStub({ + sendMessage: vi.fn(() => sent.promise), + }); + const store = createChatStore({ client }); + await store.bootstrap(); + store.updateDraft("default", "deleted draft"); + const sending = store.sendSelectedMessage(); + await store.deleteConversation("default"); + if (reimport) { + // 同名会话重新导入并两次编辑,也不能复用已删除会话的旧修订。 + store.dispatch({ + type: "conversation/upsert", + conversation: conversation(), + }); + store.updateDraft("default", "new draft"); + store.updateDraft("default", ""); + } + sent.reject(new Error("POST failed")); + await sending; + expect(store.getSnapshot().draftsByConversation.default).toBe( + reimport ? "" : undefined, + ); + expect( + store.getSnapshot().attachmentsByConversation.default, + ).toBeUndefined(); + expect( + store.getSnapshot().referencesByConversation.default, + ).toBeUndefined(); + }, + ); +}); diff --git a/apps/undefined-chat/src/chat-store/store.ts b/apps/undefined-chat/src/chat-store/store.ts index b68de85c..c466191d 100644 --- a/apps/undefined-chat/src/chat-store/store.ts +++ b/apps/undefined-chat/src/chat-store/store.ts @@ -676,6 +676,8 @@ export function createChatStore({ let runtimeUrl: string | null = null; let runtimeGeneration = 0; let runtimeSuspended = false; + const draftRevisionsByConversation = new Map(); + let draftRevision = 0; const listeners = new Set<() => void>(); const subscriptionsByJob = new Map(); const fallbackTimersByJob = new Map>(); @@ -698,6 +700,11 @@ export function createChatStore({ } function dispatch(action: ChatAction): void { + if (action.type === "draft/set") { + draftRevisionsByConversation.set(action.conversationId, ++draftRevision); + } else if (action.type === "conversation/remove") { + draftRevisionsByConversation.delete(action.conversationId); + } state = chatReducer(state, action); // 自动滚动偏好在状态更新后持久化(reducer 保持纯函数,副作用集中在此) if (action.type === "autoScroll/set") { @@ -727,6 +734,7 @@ export function createChatStore({ clearNativeSubscriptions(); sendInFlight = false; lastCommandsFetchedAt = 0; + draftRevisionsByConversation.clear(); state = { ...createInitialChatState(), autoScrollEnabled: state.autoScrollEnabled, @@ -1201,7 +1209,8 @@ export function createChatStore({ dispatch({ type: "send/error", error: "error.conversationRunning" }); return; } - const text = (state.draftsByConversation[conversationId] ?? "").trim(); + const draft = state.draftsByConversation[conversationId] ?? ""; + const text = draft.trim(); const attachments = state.attachmentsByConversation[conversationId] ?? []; if ( attachments.some((item) => ["queued", "uploading"].includes(item.status)) @@ -1244,6 +1253,10 @@ export function createChatStore({ dispatch({ type: "draft/set", conversationId, draft: "" }); dispatch({ type: "attachments/set", conversationId, attachments: [] }); dispatch({ type: "references/set", conversationId, references: [] }); + const clearedDraftRevision = + draftRevisionsByConversation.get(conversationId); + const clearedAttachments = state.attachmentsByConversation[conversationId]; + const clearedReferences = state.referencesByConversation[conversationId]; const input: SendMessageInput = { conversationId, @@ -1262,7 +1275,7 @@ export function createChatStore({ await startStreamForJob(job); } catch (err) { if (generation !== runtimeGeneration) return; - // 发送失败:移除乐观用户消息并恢复草稿/附件/引用 + // 发送失败:移除乐观消息,仅恢复发送后未再次编辑的字段。 const currentHistory = state.historyByConversation[conversationId]; if (currentHistory) { dispatch({ @@ -1276,9 +1289,22 @@ export function createChatStore({ total: Math.max(0, currentHistory.total - 1), }); } - dispatch({ type: "draft/set", conversationId, draft: text }); - dispatch({ type: "attachments/set", conversationId, attachments }); - dispatch({ type: "references/set", conversationId, references }); + if ( + draftRevisionsByConversation.get(conversationId) === + clearedDraftRevision + ) { + dispatch({ type: "draft/set", conversationId, draft }); + } + if ( + state.attachmentsByConversation[conversationId] === clearedAttachments + ) { + dispatch({ type: "attachments/set", conversationId, attachments }); + } + if ( + state.referencesByConversation[conversationId] === clearedReferences + ) { + dispatch({ type: "references/set", conversationId, references }); + } dispatch({ type: "send/error", error: errorMessage(err) }); } finally { if (generation === runtimeGeneration) sendInFlight = false; diff --git a/docs/undefined-chat.md b/docs/undefined-chat.md index 251070cf..2e45e349 100644 --- a/docs/undefined-chat.md +++ b/docs/undefined-chat.md @@ -52,6 +52,8 @@ Undefined Chat 不持久化权威聊天数据。以下内容都以 Runtime 返 连接到不同 Runtime 时,会清理旧实例的草稿、附件 UID、消息引用、历史、任务和图片预览缓存,并丢弃旧实例迟到的请求或订阅通知,避免相同会话 ID 造成跨实例误发送。同一 Runtime 的重连及 Android 恢复保留编辑状态和 UI 偏好;自动滚动、平台与布局偏好在切换实例时也保留。 +发送请求会立即清空编辑器并显示乐观用户消息。发送失败时移除该乐观消息,仅恢复请求后未再编辑的原始草稿、附件或引用;新输入和主动清空保留,各字段独立判断,不覆盖用户准备的下一条消息。 + ## 连接与事件 Runtime 请求必须由 Tauri command 拼接到已配置 origin 下,避免 React 传入任意 URL 后绕过目标主机限制。React 不直接持有 API Key;受保护请求由 Rust 侧注入 `X-Undefined-API-Key`。 From 0270b50016ec43aeb82bb4897320d68b3f486f94 Mon Sep 17 00:00:00 2001 From: Null <1708213363@qq.com> Date: Tue, 6 Oct 2026 13:31:12 +0800 Subject: [PATCH 046/121] fix(webui): scope history pagination to its request --- docs/webui-guide.md | 1 + src/Undefined/webui/static/js/runtime.js | 50 +++++-- tests/frontend/runtime_chat_harness.js | 174 ++++++++++++++++++++++ tests/test_webui_runtime_chat_behavior.py | 37 +++++ 4 files changed, 251 insertions(+), 11 deletions(-) diff --git a/docs/webui-guide.md b/docs/webui-guide.md index 3509894e..f2e5cb84 100644 --- a/docs/webui-guide.md +++ b/docs/webui-guide.md @@ -160,6 +160,7 @@ WebUI 内置的对话界面,直接与 Bot 的 AI 进行交互: - WebUI 场景下,代码优先直接作为聊天回复输出,不要默认转成文件附件;只有用户明确要求文件交付、代码过长不适合聊天展示,或确需附件工作流时才使用文件。所有代码块都应显式标注语言,例如 ` ```python `、` ```javascript `、` ```html `、` ```bash `,不确定语言时用 ` ```text `。 - AI 消息支持引用:可引用整条 AI 消息、选中的某段文字,或 HTML 预览中点选的元素。引用会显示在输入框右侧的待发送引用条中,可在发送前移除;实际发送时不会新增接口或附件,而是自动在用户消息前拼接为 Markdown `>` 引用块,例如 `> 引用 AI:`,随后和文本、图片、文件一起进入同一条 WebChat job 消息。发送后的引用块会像代码块一样折叠显示,展开后内部固定高度并可滚动。发送失败时待引用内容会保留,发送成功后清空。 - 默认加载最新 50 条消息并滚动到底部;向上滚动到顶部会按后端返回的 `next_before` 游标懒加载更早历史,并保持原视口偏移,避免一次性恢复大量工具块造成卡顿。“自动滚动到底部”开关默认开启并保存在浏览器本地,关闭后 AI 回复、工具 / Agent 状态和 AI 阶段刷新不会打断当前位置;首次加载和主动发送新消息仍会定位到底部。系统开启减少动态效果时,聊天滚动会改为即时跳转。 +- 切换会话或离开后重新进入同一会话时,上一页面的迟到历史响应会被忽略,不会混入当前历史、覆盖分页游标或清除当前正在加载的状态;旧请求的网络失败也不会给当前会话显示错误或打断其加载。后续分页继续使用当前会话返回的游标。 - 对话由 WebChat job 执行。刷新页面、关闭页面、网络短暂中断或换另一个客户端重新访问 WebUI 时,后端任务继续运行,前端会从后端查询会话列表、当前运行 job,并用 `conversation_id + job_id + seq` 每 0.5 秒轮询增量事件、当前阶段快照和运行中工具 / Agent 快照自动续接;前端不会把关键任务状态只存放在浏览器本地,历史、会话、运行中 job 和事件游标都以 Runtime API 返回值为准。如果刷新后首次查询运行中 job 失败,WebUI 会退避重试并在网络恢复时再次尝试;`done` / `error` / `cancelled` 终态先到时仍保留 job 和发送锁,并继续轮询等待 `history_finalized=true`;确认后端收尾完成后刷新权威历史,刷新成功才解除发送锁;历史慢响应或刷新失败期间继续保留锁,并通过原轮询重试恢复。SSE 仅作为兼容方式保留,WebUI 不依赖长连接。 - 切换对话后再发起新任务时,迟到的旧任务轮询响应不会覆盖新回复、改变新任务的发送锁和取消按钮,或打断新任务续接;旧请求的失败提示也不会影响当前任务。 - 运行中的 WebChat job 可从正在生成的 AI 气泡中取消。取消会调用 Runtime 的 job cancel 接口,等待历史收尾完成后解除当前会话的发送锁;如果取消发生在 AI 尚未产生任何回复内容前,前端会移除空 AI 气泡,并在最后一条用户消息上显示“重试”。点击重试会带 `reuse_previous_user_message=true` 用同一条纯文本重新发起 WebChat job,前端不再追加一条相同用户气泡,后端也会校验并复用最后一条可见用户历史,避免把同一条输入重复写入会话。 diff --git a/src/Undefined/webui/static/js/runtime.js b/src/Undefined/webui/static/js/runtime.js index 0ef38567..271288ee 100644 --- a/src/Undefined/webui/static/js/runtime.js +++ b/src/Undefined/webui/static/js/runtime.js @@ -28,6 +28,7 @@ chatHistoryCursor: null, chatHistoryHasMore: false, chatHistoryLoading: false, + chatHistoryRequest: null, chatHistoryFinalization: null, chatTopLoadSuppressedUntil: 0, chatAutoScroll: true, @@ -2097,6 +2098,10 @@ function clearChatMessages() { const log = get("runtimeChatLog"); if (!log) return; + runtimeState.chatHistoryRequest = null; + runtimeState.chatHistoryLoading = false; + const loader = get("runtimeChatLoadMore"); + if (loader) loader.textContent = ""; clearToolCollapseTimers(); log.innerHTML = ""; runtimeState.streamingMessageId = null; @@ -5058,6 +5063,7 @@ runtimeState.chatHistoryCursor = null; runtimeState.chatHistoryHasMore = false; runtimeState.chatHistoryLoading = false; + runtimeState.chatHistoryRequest = null; runtimeState.chatTopLoadSuppressedUntil = 0; runtimeState.lastEventSeq = 0; runtimeState.streamingMessageId = null; @@ -5133,19 +5139,27 @@ return; } runtimeState.chatHistoryLoading = true; - const conversationId = currentChatConversationId(); - const res = await api( - chatUrl("/api/runtime/chat/history", { limit: 50 }), - ); - const data = await parseJsonSafe(res); + const request = { conversationId: currentChatConversationId() }; + runtimeState.chatHistoryRequest = request; + let res; + let data; + try { + res = await api( + chatUrl("/api/runtime/chat/history", { limit: 50 }), + ); + data = await parseJsonSafe(res); + } catch (error) { + if (!isCurrentChatHistoryRequest(request)) return; + runtimeState.chatHistoryLoading = false; + runtimeState.chatHistoryRequest = null; + throw error; + } + if (!isCurrentChatHistoryRequest(request)) return; if (!res.ok || (data && data.error)) { runtimeState.chatHistoryLoading = false; throw new Error(buildRequestError(res, data)); } - if ( - conversationId !== currentChatConversationId() || - (expectedJobId && runtimeState.activeJobId !== expectedJobId) - ) { + if (expectedJobId && runtimeState.activeJobId !== expectedJobId) { runtimeState.chatHistoryLoading = false; return; } @@ -5166,6 +5180,13 @@ } } + function isCurrentChatHistoryRequest(request) { + return ( + runtimeState.chatHistoryRequest === request && + request.conversationId === currentChatConversationId() + ); + } + async function loadOlderChatHistory() { const log = get("runtimeChatLog"); if ( @@ -5176,6 +5197,8 @@ ) return; runtimeState.chatHistoryLoading = true; + const request = { conversationId: currentChatConversationId() }; + runtimeState.chatHistoryRequest = request; const loader = get("runtimeChatLoadMore"); if (loader) loader.textContent = t("runtime.chat_loading_more"); const previousHeight = log.scrollHeight; @@ -5189,6 +5212,7 @@ }), ); const data = await parseJsonSafe(res); + if (!isCurrentChatHistoryRequest(request)) return; if (!res.ok || (data && data.error)) { throw new Error(buildRequestError(res, data)); } @@ -5206,14 +5230,18 @@ runtimeState.chatHistoryHasMore = !!(data && data.has_more); log.scrollTop = previousTop + (log.scrollHeight - previousHeight); } catch (error) { + if (!isCurrentChatHistoryRequest(request)) return; showToast( `${t("runtime.failed")}: ${appendRuntimeApiHint(error.message || error)}`, "error", 5000, ); } finally { - runtimeState.chatHistoryLoading = false; - if (loader) loader.textContent = ""; + if (isCurrentChatHistoryRequest(request)) { + runtimeState.chatHistoryLoading = false; + runtimeState.chatHistoryRequest = null; + if (loader) loader.textContent = ""; + } } } diff --git a/tests/frontend/runtime_chat_harness.js b/tests/frontend/runtime_chat_harness.js index 233d816f..dfc3a348 100644 --- a/tests/frontend/runtime_chat_harness.js +++ b/tests/frontend/runtime_chat_harness.js @@ -3014,6 +3014,180 @@ for (const outcome of ["events", "done", "failure"]) { SCENARIOS[`stale_job_poll_${outcome}`] = (env) => staleChatJobPoll(env, outcome); } +async function staleChatHistoryPage(env, returnToSameConversation, outcome) { + const { window, setRoutes, response } = env; + const historyReads = { A: 0, B: 0 }; + setRoutes([ + { + match: "/chat/conversations", + reply: { body: { + conversations: [{ id: "A", title: "A" }, { id: "B", title: "B" }], + active_job: null, + } }, + }, + { match: "/chat/jobs/active", reply: { body: { job: null } } }, + { + match: "/chat/history", + reply: (url) => { + const conversationId = new URL(url, window.location.href).searchParams.get("conversation_id"); + historyReads[conversationId] += 1; + return { body: { + conversation_id: conversationId, + items: [{ role: "bot", content: `${conversationId}-current-${historyReads[conversationId]}` }], + has_more: true, + next_before: conversationId === "A" ? 10 : 20, + } }; + }, + }, + ]); + const fetch = window.fetch; + const pending = []; + window.fetch = (url, options) => { + if (new URL(String(url), window.location.href).searchParams.has("before")) { + return new Promise((resolve, reject) => pending.push({ url: String(url), resolve, reject })); + } + return fetch(url, options); + }; + window.RuntimeController.init(); + await window.RuntimeController.loadChatHistory(true); + await settle(1800); + const log = chatLog(window); + const loadOlder = () => { + log.scrollTop = 0; + log.dispatchEvent(new window.Event("scroll")); + }; + const snapshot = () => ({ + texts: botNodes(window).map((node) => node.contentText), + title: textOf(window, "runtimeChatCurrentTitle"), + loader: textOf(window, "runtimeChatLoadMore"), + scrollTop: log.scrollTop, + }); + loadOlder(); + await tick(); + if (pending.length !== 1) throw new Error("first conversation did not request older history"); + window.document.querySelector('[data-conversation-select="B"]').click(); + await tick(); + if (returnToSameConversation) { + window.document.querySelector('[data-conversation-select="A"]').click(); + await tick(); + } + await settle(1800); + loadOlder(); + await tick(); + if (pending.length !== 2) throw new Error("current view did not request older history"); + const beforeLate = snapshot(); + if (outcome === "failure") { + pending[0].reject(new Error("obsolete history page failure")); + } else { + pending[0].resolve(response(200, { + conversation_id: "A", + items: [{ role: "bot", content: "A-obsolete-page" }], + has_more: true, + next_before: 7, + })); + } + await tick(); + const afterLate = snapshot(); + loadOlder(); + await tick(); + const requestsWhileCurrentPagePending = pending.length; + const target = returnToSameConversation ? "A" : "B"; + pending[1].resolve(response(200, { + conversation_id: target, + items: [{ role: "bot", content: `${target}-legitimate-older` }], + has_more: true, + next_before: 3, + })); + await tick(); + const afterCurrentPage = snapshot(); + loadOlder(); + await tick(); + return { + target, + beforeLate, + afterLate, + requestsWhileCurrentPagePending, + afterCurrentPage, + pageUrls: pending.map((request) => request.url), + toastText: textOf(window, "toast-container"), + }; +} + +for (const target of ["other", "same"]) { + for (const outcome of ["success", "failure"]) { + SCENARIOS[`stale_history_page_${target}_${outcome}`] = (env) => + staleChatHistoryPage(env, target === "same", outcome); + } +} + +async function staleLatestChatHistory(env, initialLoad, failure) { + const { window, setRoutes, response } = env; + setRoutes([ + { match: "/chat/conversations", reply: { body: { + conversations: [{ id: "A", title: "A" }, { id: "B", title: "B" }], + active_job: null, + } } }, + { match: "/chat/jobs/active", reply: { body: { job: null } } }, + { match: "/chat/history", reply: { body: { + items: [{ role: "bot", content: "initial history" }], + has_more: false, next_before: null, + } } }, + ]); + const fetch = window.fetch; + const pending = []; + let deferA = initialLoad; + window.fetch = (url, options) => { + if (deferA && String(url).includes("/chat/history") && String(url).includes("conversation_id=A")) { + if (failure === "parse") { + return Promise.resolve({ ...response(200, {}), + json: () => new Promise((resolve, reject) => pending.push({ resolve, reject })), + }); + } + return new Promise((resolve, reject) => pending.push({ resolve, reject })); + } + return fetch(url, options); + }; + window.RuntimeController.init(); + if (initialLoad) { + window.__harness.setAuthenticated(true); + window.RuntimeController.onTabActivated("chat"); + await tick(); + } else { + await window.RuntimeController.loadChatHistory(true); + window.document.querySelector('[data-conversation-select="B"]').click(); + await tick(); + deferA = true; + window.document.querySelector('[data-conversation-select="A"]').click(); + await tick(); + } + window.document.querySelector('[data-conversation-select="B"]').click(); + await tick(); + window.document.querySelector('[data-conversation-select="A"]').click(); + await tick(); + if (pending.length !== 2) throw new Error(`expected two pending A latest reads, got ${pending.length}`); + const snapshot = () => ({ + texts: botNodes(window).map((node) => node.contentText), + title: textOf(window, "runtimeChatCurrentTitle"), + toasts: window.document.querySelectorAll(".toast").length, + activeRequests: env.requests.filter((url) => url.includes("/chat/jobs/active")).length, + }); + const beforeLate = snapshot(); + pending[0].reject(new Error("obsolete latest history failure")); + await tick(); + const afterLate = snapshot(); + const body = { items: [{ role: "bot", content: "A-current-latest" }], has_more: false, next_before: null }; + pending[1].resolve(failure === "parse" ? body : response(200, body)); + await tick(); + return { beforeLate, afterLate, afterCurrent: snapshot() }; +} + +for (const loading of ["initial", "forced"]) { + for (const failure of ["network", "parse"]) { + SCENARIOS[`stale_latest_history_${loading}_${failure}`] = (env) => + staleLatestChatHistory(env, loading === "initial", failure); + } +} + async function main() { const args = parseArgs(process.argv.slice(2)); if (!args.scenario) throw new Error("缺少 --scenario"); diff --git a/tests/test_webui_runtime_chat_behavior.py b/tests/test_webui_runtime_chat_behavior.py index c1b44834..bfa06bb4 100644 --- a/tests/test_webui_runtime_chat_behavior.py +++ b/tests/test_webui_runtime_chat_behavior.py @@ -288,6 +288,43 @@ def test_lazy_loading_older_history_keeps_viewport_position() -> None: assert result["lazyLoadScrollTop"] == result["lazyLoadExpectedTop"], result +@pytest.mark.parametrize("target", ["other", "same"]) +@pytest.mark.parametrize("outcome", ["success", "failure"]) +def test_late_history_page_cannot_modify_current_view_or_its_loading_state( + target: str, outcome: str +) -> None: + result = run_scenario(f"stale_history_page_{target}_{outcome}") + + assert result["afterLate"] == result["beforeLate"], result + assert result["beforeLate"]["loader"], result + assert result["requestsWhileCurrentPagePending"] == 2, result + conversation_id = "A" if target == "same" else "B" + current_text = f"{conversation_id}-current-{2 if target == 'same' else 1}" + assert result["beforeLate"]["texts"] == [current_text], result + assert result["afterCurrentPage"]["texts"] == [ + f"{conversation_id}-legitimate-older", + current_text, + ], result + assert result["afterCurrentPage"]["loader"] == "", result + assert len(result["pageUrls"]) == 3, result + assert f"conversation_id={conversation_id}" in result["pageUrls"][-1], result + assert "before=3" in result["pageUrls"][-1], result + assert "obsolete history page failure" not in result["toastText"], result + + +@pytest.mark.parametrize("loading", ["initial", "forced"]) +@pytest.mark.parametrize("failure", ["network", "parse"]) +def test_late_latest_history_failure_cannot_trigger_current_view_recovery( + loading: str, failure: str +) -> None: + result = run_scenario(f"stale_latest_history_{loading}_{failure}") + + assert result["beforeLate"]["title"] == "A", result + assert result["afterLate"] == result["beforeLate"], result + assert result["afterCurrent"]["texts"] == ["A-current-latest"], result + assert result["afterCurrent"]["toasts"] == 0, result + + # --------------------------------------------------------------------------- # # 工具块:快照去重与自动折叠 # --------------------------------------------------------------------------- # From d785f34b72c27c721e7d29cdea8512328cb23e7a Mon Sep 17 00:00:00 2001 From: Null <1708213363@qq.com> Date: Tue, 6 Oct 2026 19:15:06 +0800 Subject: [PATCH 047/121] fix(jm): clean cancelled downloads after event loop shutdown --- docs/configuration.md | 2 +- src/Undefined/jm/downloader.py | 13 ++++++------ tests/test_jm_downloader.py | 36 ++++++++++++++++++++++++++++++++++ 3 files changed, 44 insertions(+), 7 deletions(-) diff --git a/docs/configuration.md b/docs/configuration.md index 146d5e6c..03155a73 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1158,7 +1158,7 @@ api_key = "replace-with-your-key" - 车号不存在、接口不可用时发送一行 `JM 提取失败:<原因>`,日志记录异常类型与堆栈;车号与链接混排时按原文顺序取第一个,分享卡片结构异常只跳过该段。 - 除了自动提取,`jm_book` 工具让 AI 也能按需处理车号:`output_mode=send`(默认,等价自动提取)、`uid`(只下载并**把所有章节合成一份未加密 PDF** 注册为附件 UID,交给 `file_analysis_agent` 用 `extract_pdf` / `describe_pdf_page` 解析,不发送消息、不打包 zip)、`info`(只返回本子信息,不下载)。 - `info_agent` 的 `jm_search` 工具查询本子列表,`mode` 三种:`search`(关键词检索,可用 `target` 限定站内/作品/作者/标签/角色,也可直接输入纯数字车号)、`browse`(按分类浏览,不需要关键词)与 `ranking`(日/周/月排行,用 `time_range` 选日榜/周榜/月榜,固定按观看数排序)。三种模式共用 `genre` 大分类、`page` 翻页与 `n`(默认 5、上限 20);`search` / `browse` 另支持 `sort`(最新/观看/图片/点赞/评分/评论)与 `time_range` 时间范围;`sub_genre` 副分类必须与 `genre` 同时给出,且只有网页端接口支持,网页端被反爬拦截时会直接返回失败原因。只返回车号、标题、作者、分类与更新时间,不下载也不发送。查询复用同一份 `[jm]` 配置(代理、`request_timeout`、`session_dir`),且**不受** `auto_extract_enabled` 影响——该开关只约束自动提取管线。 -- jmcpy 的 HTTP 请求是异步的,但图片解码与 PDF 合成是同步 CPU 工作;这些步骤在线程中执行,不阻塞事件循环。副作用是单次自动提取不可取消,由 `request_timeout` / `image_timeout` 与 jmcpy 的多端点重试兜底。 +- 图片下载、解码与 PDF 合成在专用线程池执行,不阻塞事件循环。取消提取会结束等待,但不会强停仍在写盘的同步线程;线程结束后清理本次任务目录,即使 Bot 的事件循环已经关闭也会清理,避免残留原图或未加密 PDF。下载本身由 `request_timeout` / `image_timeout` 与 jmcpy 的多端点重试收敛。 - 默认不登录即可使用;需要在 jmcpy 中登录(例如访问账号可见内容)时,用 jmcpy 自己的方式保存会话,再把 `session_dir` 指向同一配置目录,或在 `session_dir` 留空时设置 `JMCPY_HOME`。目录值会去除首尾空白,显式 `session_dir` 优先于环境变量。 - 请遵守所在地法律法规与目标站点服务条款,只处理你有权获取的内容。 diff --git a/src/Undefined/jm/downloader.py b/src/Undefined/jm/downloader.py index 487e53e2..f3fe3584 100644 --- a/src/Undefined/jm/downloader.py +++ b/src/Undefined/jm/downloader.py @@ -27,7 +27,7 @@ from __future__ import annotations from collections.abc import Callable, Iterator, Sequence -from concurrent.futures import ThreadPoolExecutor +from concurrent.futures import Future, ThreadPoolExecutor from dataclasses import dataclass, field import asyncio import functools @@ -596,17 +596,18 @@ async def download_book( download = functools.partial( _download_archive_sync, book_id, task_dir, password=password, **options ) - future: asyncio.Future[JmDownload] = loop.run_in_executor(_DOWNLOAD_POOL, download) + worker_future: Future[JmDownload] = _DOWNLOAD_POOL.submit(download) + future = asyncio.wrap_future(worker_future, loop=loop) try: # shield:协程被取消时不要把线程一起取消——同步下载停不下来,强行删目录 # 只会被后续写入重新创建,所以等它跑完再清理 result = await asyncio.shield(future) except BaseException: - if future.done(): + if worker_future.done(): # 异常路径下调用方拿不到 task_dir,必须在这里清掉已下载的原图 await cleanup_download_dir(task_dir) else: - _cleanup_when_finished(future, task_dir, loop=loop) + _cleanup_when_finished(worker_future, task_dir, loop=loop) raise return result, task_dir @@ -616,14 +617,14 @@ def _remove_dir_now(path: Path) -> None: def _cleanup_when_finished( - future: "asyncio.Future[JmDownload]", + future: Future[JmDownload], task_dir: Path, *, loop: asyncio.AbstractEventLoop, ) -> None: """线程仍在写盘时,等它结束后再删任务目录(取消/超时路径)。""" - def _on_done(_future: "asyncio.Future[JmDownload]") -> None: + def _on_done(_future: Future[JmDownload]) -> None: try: loop.run_in_executor(None, _remove_dir_now, task_dir) except RuntimeError: diff --git a/tests/test_jm_downloader.py b/tests/test_jm_downloader.py index 429de1f2..dff55fa3 100644 --- a/tests/test_jm_downloader.py +++ b/tests/test_jm_downloader.py @@ -1,7 +1,9 @@ from __future__ import annotations from collections.abc import Sequence +from concurrent.futures import ThreadPoolExecutor import re +import threading from types import SimpleNamespace from typing import Any, Literal, TypeAlias import zipfile @@ -956,3 +958,37 @@ def _slow(*_args: Any, **_kwargs: Any) -> Any: break await asyncio.sleep(0.05) assert list((tmp_path / "jm").glob("*")) == [], "取消后必须清掉任务目录" + + +def test_cancelled_download_cleans_up_after_event_loop_closes( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + started = threading.Event() + release = threading.Event() + pool = ThreadPoolExecutor(max_workers=1) + monkeypatch.setattr(jm_downloader, "_DOWNLOAD_POOL", pool) + + def download(book_id: str, task_dir: Path, **options: Any) -> Any: + started.set() + assert release.wait(5) + (task_dir / "raw-page.jpg").write_bytes(b"temporary original") + return SimpleNamespace() + + monkeypatch.setattr(jm_downloader, "_download_plain_sync", download) + + async def cancel_download() -> None: + task = asyncio.create_task(download_book("1114751", config=_config())) + assert await asyncio.to_thread(started.wait, 5) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert list((tmp_path / "jm").glob("*")) + + try: + asyncio.run(cancel_download()) + # 专用下载线程在 loop 关闭后才写入原图并完成。 + release.set() + finally: + release.set() + pool.shutdown(wait=True) + assert list((tmp_path / "jm").glob("*")) == [] From 6bc41fd42702b27f50db3b4f2de9c1a1601808a6 Mon Sep 17 00:00:00 2001 From: Null <1708213363@qq.com> Date: Tue, 6 Oct 2026 19:17:26 +0800 Subject: [PATCH 048/121] fix(chat): guard conversation history request ordering --- apps/undefined-chat/README.md | 2 +- .../src/chat-store/history-order.test.ts | 179 ++++++++++++++++++ apps/undefined-chat/src/chat-store/store.ts | 38 +++- 3 files changed, 214 insertions(+), 5 deletions(-) create mode 100644 apps/undefined-chat/src/chat-store/history-order.test.ts diff --git a/apps/undefined-chat/README.md b/apps/undefined-chat/README.md index 21a152c6..991aa5d1 100644 --- a/apps/undefined-chat/README.md +++ b/apps/undefined-chat/README.md @@ -7,7 +7,7 @@ Undefined Chat 是 `apps/undefined-chat/` 下的 Tauri v2 + React 19 原生聊 ## 当前能力 - 会话管理:创建、删除、重命名、切换会话。当前没有置顶字段/API。 -- 消息历史:初始加载、cursor-based 加载更早消息、滚动锚点保持和长历史窗口化渲染。 +- 消息历史:初始加载、cursor-based 加载更早消息、滚动锚点保持和长历史窗口化渲染。同一会话的重载与分页共用请求顺序保护;发送新消息、完成任务刷新或删除会话后,迟到的旧响应不会覆盖正文、游标或加载状态。 - 消息渲染:Markdown、表格、任务列表、引用块、代码高亮、代码块复制;正文 HTML 经 sanitize 内联渲染,HTML 代码块可在独立预览窗口中运行。 - 附件:系统文件选择器、上传状态队列、Tauri 流式上传、下载和预览。当前显示上传中/成功/失败状态,不显示百分比进度。 - 图片:正文 `` 内联图、附件缩略图、blob 缓存、全屏查看、缩放、旋转和可滚动查看区域。 diff --git a/apps/undefined-chat/src/chat-store/history-order.test.ts b/apps/undefined-chat/src/chat-store/history-order.test.ts new file mode 100644 index 00000000..cf698203 --- /dev/null +++ b/apps/undefined-chat/src/chat-store/history-order.test.ts @@ -0,0 +1,179 @@ +import { describe, expect, test, vi } from "vitest"; +import type { + HistoryPageResponse, + HistoryResponse, +} from "../runtime-client/types"; +import { historyItem, job, runtimeClientStub } from "../test-fixtures"; +import { createChatStore } from "./store"; + +function deferred() { + let resolve!: (value: T) => void; + let reject!: (reason: Error) => void; + const promise = new Promise((resolvePromise, rejectPromise) => { + resolve = resolvePromise; + reject = rejectPromise; + }); + return { promise, resolve, reject }; +} + +function history(start: number, end: number, total = end): HistoryResponse { + return { + conversationId: "default", + virtualUserId: "webchat", + permission: "superadmin", + count: end - start, + items: Array.from({ length: end - start }, (_, index) => + historyItem({ + messageId: `message-${start + index}`, + content: `正文 ${start + index}`, + }), + ), + limit: 50, + before: null, + hasMore: start > 0, + nextBefore: start > 0 ? start : null, + total, + }; +} + +describe("history request ordering within one Runtime", () => { + test("a late page cannot replace the final history after sending and cancelling a job", async () => { + const oldPage = deferred(); + const finalHistory = history(72, 122); + finalHistory.items.splice( + -2, + 2, + historyItem({ messageId: "new-user", content: "新问题" }), + historyItem({ + messageId: "new-bot", + role: "bot", + content: "新回复", + }), + ); + const client = runtimeClientStub({ + getHistory: vi + .fn() + .mockResolvedValueOnce(history(70, 120)) + .mockResolvedValueOnce(finalHistory), + getHistoryPage: vi.fn(() => oldPage.promise), + cancelJob: vi.fn(async () => + job({ status: "cancelled", historyFinalized: true }), + ), + }); + const store = createChatStore({ client }); + await store.bootstrap(); + const pendingPage = store.loadMoreHistory("default"); + store.updateDraft("default", "新问题"); + await store.sendSelectedMessage(); + await store.cancelJob("job-1"); + await vi.waitFor(() => { + expect(store.getSnapshot().historyByConversation.default.items).toEqual( + finalHistory.items, + ); + }); + oldPage.resolve({ ...history(20, 70, 120), before: 70, cursor: 20 }); + await pendingPage; + expect(store.getSnapshot().historyByConversation.default).toEqual({ + items: finalHistory.items, + hasMore: true, + nextBefore: 72, + total: 122, + loading: false, + error: null, + }); + }); + + test.each(["success", "error"])( + "a superseded reload %s cannot replace a newer reload", + async (outcome) => { + const oldReload = deferred(); + const newer = history(72, 122); + const client = runtimeClientStub(); + const store = createChatStore({ client }); + await store.bootstrap(); + vi.mocked(client.getHistory) + .mockImplementationOnce(() => oldReload.promise) + .mockResolvedValueOnce(newer); + const pending = store.reloadHistory("default"); + await store.reloadHistory("default"); + if (outcome === "success") oldReload.resolve(history(70, 120)); + else oldReload.reject(new Error("旧请求失败")); + await pending; + expect(store.getSnapshot().historyByConversation.default).toEqual({ + items: newer.items, + hasMore: true, + nextBefore: 72, + total: 122, + loading: false, + error: null, + }); + }, + ); + + test("an old page error cannot clear a newer reload's loading state", async () => { + const page = deferred(); + const reload = deferred(); + const client = runtimeClientStub({ + getHistory: vi.fn(async () => history(70, 120)), + getHistoryPage: vi.fn(() => page.promise), + }); + const store = createChatStore({ client }); + await store.bootstrap(); + const pendingPage = store.loadMoreHistory("default"); + vi.mocked(client.getHistory).mockImplementationOnce(() => reload.promise); + const pendingReload = store.reloadHistory("default"); + page.reject(new Error("迟到分页错误")); + await pendingPage; + expect(store.getSnapshot().historyByConversation.default.loading).toBe( + true, + ); + expect(store.getSnapshot().historyByConversation.default.error).toBeNull(); + reload.resolve(history(72, 122)); + await pendingReload; + expect(store.getSnapshot().historyByConversation.default.loading).toBe( + false, + ); + }); + + test.each(["page", "reload"])( + "an in-flight %s cannot remove a newly sent optimistic message", + async (kind) => { + const response = deferred(); + const client = runtimeClientStub({ + getHistory: vi.fn(async () => history(70, 120)), + getHistoryPage: vi.fn(() => response.promise), + }); + const store = createChatStore({ client }); + await store.bootstrap(); + vi.mocked(client.getHistory).mockImplementationOnce( + () => response.promise, + ); + const pending = + kind === "page" + ? store.loadMoreHistory("default") + : store.reloadHistory("default"); + store.updateDraft("default", "保留新问题"); + await store.sendSelectedMessage(); + response.resolve({ ...history(20, 70, 120), before: 70, cursor: 20 }); + await pending; + const saved = store.getSnapshot().historyByConversation.default; + expect(saved.items.at(-1)?.content).toBe("保留新问题"); + expect(saved.total).toBe(121); + expect(saved.nextBefore).toBe(70); + expect(saved.loading).toBe(false); + }, + ); + + test("a pending history response cannot recreate a deleted conversation's history", async () => { + const response = deferred(); + const client = runtimeClientStub(); + const store = createChatStore({ client }); + await store.bootstrap(); + vi.mocked(client.getHistory).mockImplementationOnce(() => response.promise); + const pending = store.reloadHistory("default"); + await store.deleteConversation("default"); + response.resolve(history(70, 120)); + await pending; + expect(store.getSnapshot().historyByConversation.default).toBeUndefined(); + }); +}); diff --git a/apps/undefined-chat/src/chat-store/store.ts b/apps/undefined-chat/src/chat-store/store.ts index c466191d..e768f89a 100644 --- a/apps/undefined-chat/src/chat-store/store.ts +++ b/apps/undefined-chat/src/chat-store/store.ts @@ -678,6 +678,8 @@ export function createChatStore({ let runtimeSuspended = false; const draftRevisionsByConversation = new Map(); let draftRevision = 0; + const historyRequestsByConversation = new Map(); + let historyRequestGeneration = 0; const listeners = new Set<() => void>(); const subscriptionsByJob = new Map(); const fallbackTimersByJob = new Map>(); @@ -700,6 +702,14 @@ export function createChatStore({ } function dispatch(action: ChatAction): void { + if ( + action.type === "history/set" || + action.type === "history/error" || + action.type === "message/optimisticUser" || + action.type === "conversation/remove" + ) { + historyRequestsByConversation.delete(action.conversationId); + } if (action.type === "draft/set") { draftRevisionsByConversation.set(action.conversationId, ++draftRevision); } else if (action.type === "conversation/remove") { @@ -726,6 +736,7 @@ export function createChatStore({ fallbackTimersByJob.clear(); latestFallbackRequestByJob.clear(); fallbackBackoffByJob.clear(); + historyRequestsByConversation.clear(); } function resetRuntimeScope(nextUrl: string): void { @@ -765,13 +776,14 @@ export function createChatStore({ async function loadHistory(conversationId: string): Promise { if (runtimeSuspended) return; const generation = runtimeGeneration; + const request = beginHistoryRequest(conversationId); dispatch({ type: "history/loading", conversationId }); try { const history = await client.getHistory({ conversationId, limit: HISTORY_LIMIT, }); - if (generation !== runtimeGeneration) return; + if (!isCurrentHistoryRequest(conversationId, generation, request)) return; dispatch({ type: "history/set", conversationId, @@ -781,7 +793,7 @@ export function createChatStore({ total: history.total, }); } catch (err) { - if (generation !== runtimeGeneration) return; + if (!isCurrentHistoryRequest(conversationId, generation, request)) return; // 单会话历史加载失败降级为会话级错误,不升级为全局致命错误 dispatch({ type: "history/error", @@ -791,6 +803,23 @@ export function createChatStore({ } } + function beginHistoryRequest(conversationId: string): number { + const request = ++historyRequestGeneration; + historyRequestsByConversation.set(conversationId, request); + return request; + } + + function isCurrentHistoryRequest( + conversationId: string, + generation: number, + request: number, + ): boolean { + return ( + generation === runtimeGeneration && + historyRequestsByConversation.get(conversationId) === request + ); + } + async function startStreamForJob(job: ChatJob): Promise { if (runtimeSuspended) return; const generation = runtimeGeneration; @@ -1709,6 +1738,7 @@ export function createChatStore({ return; } + const request = beginHistoryRequest(conversationId); dispatch({ type: "history/loading", conversationId }); try { @@ -1717,7 +1747,7 @@ export function createChatStore({ historyState.nextBefore ?? undefined, HISTORY_LIMIT, ); - if (generation !== runtimeGeneration) return; + if (!isCurrentHistoryRequest(conversationId, generation, request)) return; const newItems = [...response.items, ...historyState.items]; @@ -1730,7 +1760,7 @@ export function createChatStore({ total: response.total, }); } catch (error) { - if (generation !== runtimeGeneration) return; + if (!isCurrentHistoryRequest(conversationId, generation, request)) return; dispatch({ type: "history/error", conversationId, From d38ecc0d8b9b3a8de8477724581b3a5c76f552a2 Mon Sep 17 00:00:00 2001 From: Null <1708213363@qq.com> Date: Tue, 6 Oct 2026 19:20:34 +0800 Subject: [PATCH 049/121] fix(chat): render failed code highlighting as plain text --- apps/undefined-chat/README.md | 2 +- .../src/rendering/CodeBlock.test.tsx | 38 +++++++++++++++++++ .../src/rendering/CodeBlock.tsx | 15 ++++++-- .../src/rendering/MarkdownContent.test.tsx | 13 +++++++ 4 files changed, 63 insertions(+), 5 deletions(-) diff --git a/apps/undefined-chat/README.md b/apps/undefined-chat/README.md index 991aa5d1..d256f0da 100644 --- a/apps/undefined-chat/README.md +++ b/apps/undefined-chat/README.md @@ -8,7 +8,7 @@ Undefined Chat 是 `apps/undefined-chat/` 下的 Tauri v2 + React 19 原生聊 - 会话管理:创建、删除、重命名、切换会话。当前没有置顶字段/API。 - 消息历史:初始加载、cursor-based 加载更早消息、滚动锚点保持和长历史窗口化渲染。同一会话的重载与分页共用请求顺序保护;发送新消息、完成任务刷新或删除会话后,迟到的旧响应不会覆盖正文、游标或加载状态。 -- 消息渲染:Markdown、表格、任务列表、引用块、代码高亮、代码块复制;正文 HTML 经 sanitize 内联渲染,HTML 代码块可在独立预览窗口中运行。 +- 消息渲染:Markdown、表格、任务列表、引用块、代码高亮、代码块复制;未知语言或高亮失败时保留为纯文本代码,代码中的 HTML 不会插入正文。正文 HTML 经 sanitize 内联渲染,HTML 代码块可在独立预览窗口中运行。 - 附件:系统文件选择器、上传状态队列、Tauri 流式上传、下载和预览。当前显示上传中/成功/失败状态,不显示百分比进度。 - 图片:正文 `` 内联图、附件缩略图、blob 缓存、全屏查看、缩放、旋转和可滚动查看区域。 - 命令:斜杠命令、子命令提示、方向键导航、Tab/Enter 补全;空态反馈(加载中/无匹配/不可用);窗口聚焦按 TTL 刷新命令列表。 diff --git a/apps/undefined-chat/src/rendering/CodeBlock.test.tsx b/apps/undefined-chat/src/rendering/CodeBlock.test.tsx index fff133fb..25e98b14 100644 --- a/apps/undefined-chat/src/rendering/CodeBlock.test.tsx +++ b/apps/undefined-chat/src/rendering/CodeBlock.test.tsx @@ -1,5 +1,6 @@ import { fireEvent, render, screen, waitFor } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; +import hljs from "highlight.js"; import type { ReactElement } from "react"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { LanguageProvider, LOCALE_STORAGE_KEY } from "../i18n"; @@ -30,6 +31,7 @@ describe("CodeBlock", () => { afterEach(() => { vi.clearAllMocks(); + vi.restoreAllMocks(); }); it("应该渲染代码块", () => { @@ -190,6 +192,42 @@ describe("CodeBlock", () => { }).not.toThrow(); }); + it.each(["notalanguage", "vue"])( + "未知语言 %s 的 HTML 保持为可复制的代码文本", + async (language) => { + const code = + '\n
\n& '; + const { container } = renderCodeBlock( + , + ); + expect(container.querySelector("code")?.textContent).toBe(code); + expect(container.querySelector("style, form, input, script")).toBeNull(); + expect(screen.getByText("plaintext")).toBeInTheDocument(); + fireEvent.click(screen.getByText("复制")); + await waitFor(() => expect(mockWriteText).toHaveBeenCalledWith(code)); + }, + ); + + it.each(["highlight", "highlightAuto"] as const)( + "%s 抛出异常时 HTML 同样保持为文本", + (method) => { + vi.spyOn(hljs, method).mockImplementation(() => { + throw new Error("受控高亮异常"); + }); + vi.spyOn(console, "error").mockImplementation(() => undefined); + const code = + '
'; + const { container } = renderCodeBlock( + , + ); + expect(container.querySelector("code")?.textContent).toBe(code); + expect(container.querySelector("style, form")).toBeNull(); + }, + ); + it("应该正确渲染空代码", () => { const code = ""; renderCodeBlock(); diff --git a/apps/undefined-chat/src/rendering/CodeBlock.tsx b/apps/undefined-chat/src/rendering/CodeBlock.tsx index 53d66edf..5c1bb41c 100644 --- a/apps/undefined-chat/src/rendering/CodeBlock.tsx +++ b/apps/undefined-chat/src/rendering/CodeBlock.tsx @@ -39,6 +39,9 @@ export function CodeBlock({ // htm 别名映射到 html const normalizedLanguage = language.toLowerCase() === "htm" ? "html" : language; + if (!hljs.getLanguage(normalizedLanguage)) { + return { highlightedCode: null, detectedLanguage: "plaintext" }; + } // 指定了语言,尝试使用指定语言高亮 const result = hljs.highlight(trimmedCode, { language: normalizedLanguage, @@ -58,7 +61,7 @@ export function CodeBlock({ } catch (err) { console.error("Highlight error:", err); return { - highlightedCode: trimmedCode, + highlightedCode: null, detectedLanguage: "plaintext", }; } @@ -141,9 +144,13 @@ export function CodeBlock({
 					
+						// Only highlighter output is HTML; fallback code is rendered as text.
+						dangerouslySetInnerHTML={
+							highlightedCode === null ? undefined : { __html: highlightedCode }
+						}
+					>
+						{highlightedCode === null ? code.trim() : undefined}
+					
 				
diff --git a/apps/undefined-chat/src/rendering/MarkdownContent.test.tsx b/apps/undefined-chat/src/rendering/MarkdownContent.test.tsx index 94a56570..b1ab04da 100644 --- a/apps/undefined-chat/src/rendering/MarkdownContent.test.tsx +++ b/apps/undefined-chat/src/rendering/MarkdownContent.test.tsx @@ -198,6 +198,19 @@ describe("MarkdownContent", () => { expect(screen.getByText("复制")).toBeInTheDocument(); }); + it("未知 fenced-code 语言不会将代码里的 HTML 插入正文", () => { + const code = + '\n'; + const { container } = render( + , + ); + expect(container.querySelector("code")?.textContent).toBe(code); + expect(container.querySelector("style, form, input")).toBeNull(); + }); + it("渲染混合内容", () => { const mixedContent = `# 标题 From 0ebaac5753e059f082f4950a169b632f329e8357 Mon Sep 17 00:00:00 2001 From: Null <1708213363@qq.com> Date: Tue, 6 Oct 2026 19:20:52 +0800 Subject: [PATCH 050/121] fix(webui): ignore stale active job recovery responses --- docs/webui-guide.md | 1 + src/Undefined/webui/static/js/runtime.js | 30 +++++++ tests/frontend/runtime_chat_harness.js | 97 +++++++++++++++++++++++ tests/test_webui_runtime_chat_behavior.py | 20 +++++ 4 files changed, 148 insertions(+) diff --git a/docs/webui-guide.md b/docs/webui-guide.md index f2e5cb84..2049262e 100644 --- a/docs/webui-guide.md +++ b/docs/webui-guide.md @@ -163,6 +163,7 @@ WebUI 内置的对话界面,直接与 Bot 的 AI 进行交互: - 切换会话或离开后重新进入同一会话时,上一页面的迟到历史响应会被忽略,不会混入当前历史、覆盖分页游标或清除当前正在加载的状态;旧请求的网络失败也不会给当前会话显示错误或打断其加载。后续分页继续使用当前会话返回的游标。 - 对话由 WebChat job 执行。刷新页面、关闭页面、网络短暂中断或换另一个客户端重新访问 WebUI 时,后端任务继续运行,前端会从后端查询会话列表、当前运行 job,并用 `conversation_id + job_id + seq` 每 0.5 秒轮询增量事件、当前阶段快照和运行中工具 / Agent 快照自动续接;前端不会把关键任务状态只存放在浏览器本地,历史、会话、运行中 job 和事件游标都以 Runtime API 返回值为准。如果刷新后首次查询运行中 job 失败,WebUI 会退避重试并在网络恢复时再次尝试;`done` / `error` / `cancelled` 终态先到时仍保留 job 和发送锁,并继续轮询等待 `history_finalized=true`;确认后端收尾完成后刷新权威历史,刷新成功才解除发送锁;历史慢响应或刷新失败期间继续保留锁,并通过原轮询重试恢复。SSE 仅作为兼容方式保留,WebUI 不依赖长连接。 - 切换对话后再发起新任务时,迟到的旧任务轮询响应不会覆盖新回复、改变新任务的发送锁和取消按钮,或打断新任务续接;旧请求的失败提示也不会影响当前任务。 +- 查询运行中任务的恢复请求同样绑定发起时的会话和任务身份;切换会话、返回原会话或建立新任务后,旧恢复请求的迟到回包与失败重试不会改变当前会话正文、任务状态或发送锁。 - 运行中的 WebChat job 可从正在生成的 AI 气泡中取消。取消会调用 Runtime 的 job cancel 接口,等待历史收尾完成后解除当前会话的发送锁;如果取消发生在 AI 尚未产生任何回复内容前,前端会移除空 AI 气泡,并在最后一条用户消息上显示“重试”。点击重试会带 `reuse_previous_user_message=true` 用同一条纯文本重新发起 WebChat job,前端不再追加一条相同用户气泡,后端也会校验并复用最后一条可见用户历史,避免把同一条输入重复写入会话。 - 运行中的 AI 气泡会在 `AI` 标签后实时显示当前阶段和后端计算的总已用时,例如构建上下文、查长期记忆、查认知记忆、等待模型、等待工具、发送消息;任务完成后该位置显示本轮回复总用时。工具 / Agent 摘要行会在名称旁显示调用耗时,运行中先显示后端快照时间,轮询间隙用本地时间临时递增,下一次查询后自动校准;结束后固定显示后端结束事件的总耗时。轮询刷新只更新已有状态和计时节点,结构未变化时不会重建工具 / Agent 块,避免运行中闪烁。状态条区分运行中、成功和失败;整轮回复总耗时会写入 WebChat 历史 metadata。 - WebChat job 事件会更新同一个 AI 气泡;AI 正文和主对话工具 / Agent 调用按事件时序显示,例如工具调用、正文回复、结束工具会依次出现在同一气泡内。 diff --git a/src/Undefined/webui/static/js/runtime.js b/src/Undefined/webui/static/js/runtime.js index 271288ee..398ef492 100644 --- a/src/Undefined/webui/static/js/runtime.js +++ b/src/Undefined/webui/static/js/runtime.js @@ -43,6 +43,7 @@ chatClockTimer: null, activeJobResumeTimer: null, activeJobResumeAttempts: 0, + activeJobResumeRequest: null, toolBlocks: new Map(), toolCollapseTimers: new Map(), chatAttachments: [], @@ -5059,6 +5060,9 @@ } function resetChatConversationState() { + stopActiveJobResumeTimer(); + runtimeState.activeJobResumeAttempts = 0; + runtimeState.activeJobResumeRequest = null; runtimeState.chatHistoryLoaded = false; runtimeState.chatHistoryCursor = null; runtimeState.chatHistoryHasMore = false; @@ -5535,6 +5539,9 @@ } async function attachChatJob(jobId, after = 0) { + runtimeState.activeJobResumeRequest = null; + stopActiveJobResumeTimer(); + runtimeState.activeJobResumeAttempts = 0; stopChatPolling(); runtimeState.activeJobId = jobId; runtimeState.activeJobConversationId = @@ -5548,6 +5555,14 @@ pollChatJob(jobId).catch(() => {}); } + function isCurrentActiveJobResumeRequest(request) { + return ( + runtimeState.activeJobResumeRequest === request && + request.conversationId === currentChatConversationId() && + request.jobId === String(runtimeState.activeJobId || "") + ); + } + async function resumeActiveChatJob() { const localJobId = runtimeState.activeJobId ? String(runtimeState.activeJobId) @@ -5562,10 +5577,16 @@ ) { return; } + const request = { + conversationId: currentChatConversationId(), + jobId: localJobId, + }; + runtimeState.activeJobResumeRequest = request; try { const data = await fetchJsonOrThrow( chatUrl("/api/runtime/chat/jobs/active"), ); + if (!isCurrentActiveJobResumeRequest(request)) return; const job = data && data.job ? data.job : null; if (!job || !job.job_id) { stopActiveJobResumeTimer(); @@ -5574,6 +5595,7 @@ stopChatPolling(); stopChatClock(); runtimeState.activeJobId = null; + request.jobId = ""; runtimeState.activeJobConversationId = ""; runtimeState.chatBusy = false; setButtonLoading(get("btnRuntimeChatSend"), false); @@ -5585,6 +5607,12 @@ } return; } + if ( + job.conversation_id && + String(job.conversation_id) !== request.conversationId + ) { + return; + } stopActiveJobResumeTimer(); runtimeState.activeJobResumeAttempts = 0; const jobId = String(job.job_id); @@ -5608,6 +5636,7 @@ ensureStreamingMessage(jobId); attachChatJob(jobId, runtimeState.lastEventSeq).catch(() => {}); } catch (_error) { + if (!isCurrentActiveJobResumeRequest(request)) return; runtimeState.activeJobResumeAttempts += 1; if ( runtimeState.activeJobResumeAttempts > @@ -5622,6 +5651,7 @@ ); stopActiveJobResumeTimer(); runtimeState.activeJobResumeTimer = setTimeout(() => { + if (!isCurrentActiveJobResumeRequest(request)) return; resumeActiveChatJob().catch(() => {}); }, delay); } diff --git a/tests/frontend/runtime_chat_harness.js b/tests/frontend/runtime_chat_harness.js index dfc3a348..b5a2d507 100644 --- a/tests/frontend/runtime_chat_harness.js +++ b/tests/frontend/runtime_chat_harness.js @@ -3188,6 +3188,103 @@ for (const loading of ["initial", "forced"]) { } } +async function staleActiveJobResume(env, returnToSameConversation, outcome) { + const { window, setRoutes, response } = env; + const currentJob = { + job_id: "current-A", conversation_id: "A", status: "running", + }; + setRoutes([ + { + match: "/chat/conversations", + reply: { body: { + conversations: [{ id: "A", title: "A" }, { id: "B", title: "B" }], + active_job: null, + } }, + }, + { + match: "/chat/history", + reply: (url) => ({ body: { + items: [{ role: "bot", content: url.includes("conversation_id=B") ? "B-history" : "A-history" }], + has_more: false, next_before: null, + } }), + }, + { + match: "/chat/jobs/active", + reply: (url) => ({ body: { + job: returnToSameConversation && url.includes("conversation_id=A") ? currentJob : null, + } }), + }, + { + match: "/jobs/current-A/events", + reply: (url) => { + const after = Number(new URL(url, window.location.href).searchParams.get("after")); + return { body: { job: currentJob, events: [ + { seq: 1, event: "meta", payload: currentJob }, + { seq: 2, event: "message", payload: { ...currentJob, content: "current-live" } }, + ].filter((event) => event.seq > after) } }; + }, + }, + { + match: "/jobs/obsolete-A/events", + reply: { body: { job: { job_id: "obsolete-A", conversation_id: "A", status: "running" }, events: [] } }, + }, + ]); + const fetch = window.fetch; + let pending; + const resumeUrls = []; + window.fetch = (url, options) => { + if (String(url).includes("/chat/jobs/active")) { + resumeUrls.push(String(url)); + if (!pending && String(url).includes("conversation_id=A")) { + return new Promise((resolve, reject) => { pending = { resolve, reject }; }); + } + } + return fetch(url, options); + }; + window.RuntimeController.init(); + const initialLoad = window.RuntimeController.loadChatHistory(true); + await tick(); + if (!pending) throw new Error("initial A resume request was not dispatched"); + window.document.querySelector('[data-conversation-select="B"]').click(); + await tick(); + if (returnToSameConversation) { + window.document.querySelector('[data-conversation-select="A"]').click(); + await tick(); + } + const snapshot = () => ({ + title: textOf(window, "runtimeChatCurrentTitle"), + nodes: chatNodes(window).map((node) => ({ text: node.contentText, job: node.jobId })), + disabled: window.document.getElementById("btnRuntimeChatSend").disabled, + cancelIds: Array.from(window.document.querySelectorAll("[data-cancel-job]")) + .filter((element) => !element.hidden).map((element) => element.dataset.cancelJob), + }); + const beforeLate = snapshot(); + const resumeCountBeforeLate = resumeUrls.length; + if (outcome === "failure") { + pending.reject(new Error("obsolete resume failure")); + } else { + pending.resolve(response(200, { + job: outcome === "empty" ? null : { job_id: "obsolete-A", conversation_id: "A", status: "running" }, + })); + } + await initialLoad; + await tick(); + const afterLate = snapshot(); + if (outcome === "failure") await settle(1100); + return { + beforeLate, afterLate, afterRetryWindow: snapshot(), + resumeCountBeforeLate, resumeCountAfterLate: resumeUrls.length, + obsoletePolls: env.requests.filter((url) => url.includes("/jobs/obsolete-A/events")), + }; +} + +for (const target of ["other", "same"]) { + for (const outcome of ["running", "empty", "failure"]) { + SCENARIOS[`stale_active_job_resume_${target}_${outcome}`] = (env) => + staleActiveJobResume(env, target === "same", outcome); + } +} + async function main() { const args = parseArgs(process.argv.slice(2)); if (!args.scenario) throw new Error("缺少 --scenario"); diff --git a/tests/test_webui_runtime_chat_behavior.py b/tests/test_webui_runtime_chat_behavior.py index bfa06bb4..d9092e22 100644 --- a/tests/test_webui_runtime_chat_behavior.py +++ b/tests/test_webui_runtime_chat_behavior.py @@ -693,6 +693,26 @@ def test_auto_scroll_toggle_persists_preference() -> None: # --------------------------------------------------------------------------- # +@pytest.mark.parametrize("target", ["other", "same"]) +@pytest.mark.parametrize("outcome", ["running", "empty", "failure"]) +def test_late_active_job_resume_cannot_replace_current_conversation_or_job( + target: str, outcome: str +) -> None: + result = run_scenario(f"stale_active_job_resume_{target}_{outcome}") + + before = result["beforeLate"] + assert before["title"] == ("A" if target == "same" else "B"), result + assert before["nodes"][0]["text"] == ( + "A-history" if target == "same" else "B-history" + ), result + assert before["disabled"] is (target == "same"), result + assert before["cancelIds"] == (["current-A"] if target == "same" else []), result + assert result["afterLate"] == before, result + assert result["afterRetryWindow"] == before, result + assert result["resumeCountAfterLate"] == result["resumeCountBeforeLate"], result + assert result["obsoletePolls"] == [], result + + @pytest.mark.parametrize("outcome", ["events", "done", "failure"]) def test_late_old_job_poll_cannot_replace_or_stop_new_job(outcome: str) -> None: result = run_scenario(f"stale_job_poll_{outcome}") From 56a53b8261add658574910f7be853d2ed327faa4 Mon Sep 17 00:00:00 2001 From: Null <1708213363@qq.com> Date: Tue, 6 Oct 2026 19:22:02 +0800 Subject: [PATCH 051/121] fix(api): close chat admission and roll back failed job creation --- docs/openapi.md | 1 + src/Undefined/api/routes/chat.py | 71 ++++++---- src/Undefined/api/webchat_store.py | 14 +- tests/test_runtime_api_chat_jobs.py | 196 ++++++++++++++++++++++++++++ 4 files changed, 255 insertions(+), 27 deletions(-) diff --git a/docs/openapi.md b/docs/openapi.md index febb77dc..72f54437 100644 --- a/docs/openapi.md +++ b/docs/openapi.md @@ -523,6 +523,7 @@ Undefined Chat 使用同一组 Runtime Chat 端点作为权威合同。客户端 - `POST /api/v1/chat/jobs`:创建后台 job。兼容旧 Body `{"message":"...","conversation_id":"..."}`,也支持结构化 Body `{"message":{"text":"...","attachment_ids":["..."],"references":[{"message_id":"...","quote":"..."}]},"conversation_id":"..."}`;`conversation_id` 可选。取消后重试最后一条纯文本用户消息时,可传 `reuse_previous_user_message: true`,Runtime 会要求最后一条可见用户历史与本次文本完全一致,并跳过重复写入用户历史。 - WebChat job 在同一会话内 single-flight;如果目标会话已有 job 正在运行或收尾落盘,创建新 job 返回 `409`。不同会话可以并发运行。兼容的非流式 `POST /api/v1/chat` 也走同一套会话级 job 锁,只是等待完成后返回同步结果。 +- Runtime 开始停止后,新的 job 和同步/SSE 聊天请求返回 `503`;此前已进入创建阶段的历史写入会先收敛,再取消并收尾其任务。结构化消息的历史预写失败时会撤销未启动的 job,缓存与磁盘均不追加失败消息,存储恢复后可以正常重试。若取消发生在历史原子写入已提交之后,该条用户历史会保留,但不会遗留未启动的 active job。 - 附件先通过 `POST /api/v1/chat/attachments` 以 multipart 上传并由 Runtime 返回附件 metadata;发送消息时只传 Runtime 生成的 `attachment_ids`。客户端不应把本地文件内容、下载 URL 或临时缓存路径拼入 `message.text`。 - 引用内容通过结构化 `references` 提交。Runtime 负责把引用写入历史 metadata,并按需要生成 AI 可见上下文;客户端不把引用块拼接进最终历史作为真源。 - `GET /api/v1/chat/jobs/active?conversation_id=`:返回兼容字段 `job` 和 active `jobs[]`。不传时 `jobs[]` 包含所有运行中 WebChat job,`job` 为最新 active job;传入 `conversation_id` 时只返回目标会话的 active job。 diff --git a/src/Undefined/api/routes/chat.py b/src/Undefined/api/routes/chat.py index abc3efd6..382811b3 100644 --- a/src/Undefined/api/routes/chat.py +++ b/src/Undefined/api/routes/chat.py @@ -379,14 +379,23 @@ def current_tool_call_snapshots( return payloads +class ChatJobManagerClosingError(RuntimeError): + """Runtime 正在收尾,不能接受新的聊天任务。""" + + class ChatJobManager: def __init__(self, ctx: RuntimeAPIContext) -> None: self._ctx = ctx self._jobs: dict[str, ChatJob] = {} self._lock = asyncio.Lock() + self._closing = False self._title_schedule_lock = asyncio.Lock() self.conversation_store = WebChatConversationStore() + def begin_shutdown(self) -> None: + """立即关闭任务创建入口,已有任务由 stop 收尾。""" + self._closing = True + async def create_job( self, text: str, @@ -397,6 +406,8 @@ async def create_job( pre_record_user_history: bool = False, reuse_previous_user_history: bool = False, ) -> ChatJob: + if self._closing: + raise ChatJobManagerClosingError("Chat service is stopping") await self.conversation_store.ensure_ready(self._ctx.history_manager) requested_conversation_id = str(conversation_id or "").strip() resolved_conversation_id = requested_conversation_id or _DEFAULT_CONVERSATION_ID @@ -421,6 +432,8 @@ async def create_job( or reuse_previous_user_history, ) async with self._lock: + if self._closing: + raise ChatJobManagerClosingError("Chat service is stopping") if any( self._job_blocks_history_mutation(existing) for existing in self._jobs.values() @@ -428,37 +441,45 @@ async def create_job( ): raise RuntimeError("Chat job is still running") self._jobs[job.job_id] = job - if pre_record_user_history: - await self.conversation_store.append_message( - resolved_conversation_id, - role="user", - text_content=text, - display_name=_VIRTUAL_USER_NAME, - user_name=_VIRTUAL_USER_NAME, - attachments=job.user_history_attachments or None, - references=job.user_history_references or None, - ) + try: + # stop 必须等创建阶段收敛后再快照,避免漏掉稍后才启动的 task。 + if pre_record_user_history: + await self.conversation_store.append_message( + resolved_conversation_id, + role="user", + text_content=text, + display_name=_VIRTUAL_USER_NAME, + user_name=_VIRTUAL_USER_NAME, + attachments=job.user_history_attachments or None, + references=job.user_history_references or None, + ) + await self._append_event( + job, + "meta", + { + "job_id": job.job_id, + "conversation_id": job.conversation_id, + "virtual_user_id": _VIRTUAL_USER_ID, + "permission": "superadmin", + }, + ) + await self._append_stage(job, "received") + job.task = asyncio.create_task( + self._run_job(job), name=f"webchat:{job.job_id}" + ) + except BaseException: + self._jobs.pop(job.job_id, None) + raise logger.info( "[RuntimeAPI][WebChat] 创建 job: job_id=%s conversation_id=%s text_len=%s", job.job_id, job.conversation_id, len(text), ) - await self._append_event( - job, - "meta", - { - "job_id": job.job_id, - "conversation_id": job.conversation_id, - "virtual_user_id": _VIRTUAL_USER_ID, - "permission": "superadmin", - }, - ) - await self._append_stage(job, "received") - job.task = asyncio.create_task(self._run_job(job), name=f"webchat:{job.job_id}") return job async def stop(self) -> None: + self.begin_shutdown() async with self._lock: jobs = list(self._jobs.values()) for job in jobs: @@ -2956,6 +2977,8 @@ async def chat_handler( job = await job_manager.create_job(text, conversation_id) except KeyError: return _json_error("Conversation not found", status=404) + except ChatJobManagerClosingError as exc: + return _json_error(str(exc), status=503) except RuntimeError: return _json_error("Chat job is still running", status=409) try: @@ -2989,6 +3012,8 @@ async def chat_handler( job = await job_manager.create_job(text, conversation_id) except KeyError: return _json_error("Conversation not found", status=404) + except ChatJobManagerClosingError as exc: + return _json_error(str(exc), status=503) except RuntimeError: return _json_error("Chat job is still running", status=409) logger.info( @@ -3106,6 +3131,8 @@ async def chat_job_create_handler( return _json_error("Attachment not found", status=404) except ValueError as exc: return _json_error(str(exc), status=400) + except ChatJobManagerClosingError as exc: + return _json_error(str(exc), status=503) except RuntimeError: return _json_error("Chat job is still running", status=409) logger.info( diff --git a/src/Undefined/api/webchat_store.py b/src/Undefined/api/webchat_store.py index 366313ca..b1a44381 100644 --- a/src/Undefined/api/webchat_store.py +++ b/src/Undefined/api/webchat_store.py @@ -257,7 +257,7 @@ async def append_message( ) -> dict[str, Any]: conv_id = _normalize_conversation_id(conversation_id) async with self._get_lock(conv_id): - conv = self._require_conversation_locked(conv_id) + conv = _copy_json(self._require_conversation_locked(conv_id)) timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") normalized_role = "bot" if role == "bot" else "user" record: dict[str, Any] = { @@ -554,10 +554,14 @@ def _require_conversation_locked(self, conversation_id: str) -> dict[str, Any]: async def _save_conversation_locked(self, conv: dict[str, Any]) -> None: normalized = _normalize_conversation(conv, str(conv.get("id") or uuid4().hex)) - self._cache[str(normalized["id"])] = normalized - await io.write_json( - self._path_for(str(normalized["id"])), normalized, use_lock=True - ) + + async def commit() -> None: + await io.write_json( + self._path_for(str(normalized["id"])), normalized, use_lock=True + ) + self._cache[str(normalized["id"])] = normalized + + await io.run_cancellation_safe(commit()) def _conversation_summary(self, conv: dict[str, Any]) -> dict[str, Any]: messages = _messages(conv) diff --git a/tests/test_runtime_api_chat_jobs.py b/tests/test_runtime_api_chat_jobs.py index 5ebac482..5ffaf36b 100644 --- a/tests/test_runtime_api_chat_jobs.py +++ b/tests/test_runtime_api_chat_jobs.py @@ -3,6 +3,7 @@ import asyncio import hashlib import json +import threading from pathlib import Path from types import SimpleNamespace from typing import Any, cast @@ -977,6 +978,201 @@ async def test_chat_job_cancelled_error_event_is_appended_once() -> None: assert len(cancelled_events) == 1 +@pytest.mark.asyncio +async def test_chat_job_history_write_failure_rolls_back_creation_and_cache( + monkeypatch: pytest.MonkeyPatch, +) -> None: + server = RuntimeAPIServer(_context(), host="127.0.0.1", port=8788) + manager = server._chat_job_manager + await manager.conversation_store.ensure_ready() + conversation = await manager.conversation_store.ensure_default_conversation() + conversation_id = str(conversation["id"]) + history_path = manager.conversation_store._path_for(conversation_id) + original_disk = history_path.read_bytes() + original_write = async_io.write_json_sync + fail_once = True + + def write_json(file_path: Path | str, data: Any, use_lock: bool = True) -> None: + nonlocal fail_once + if Path(file_path) == history_path and data.get("messages") and fail_once: + fail_once = False + raise OSError("history disk unavailable") + original_write(file_path, data, use_lock) + + run_chat = AsyncMock(return_value="chat") + monkeypatch.setattr(async_io, "write_json_sync", write_json) + monkeypatch.setattr(runtime_api_chat, "run_webui_chat", run_chat) + + def request() -> web.Request: + return cast( + web.Request, + cast( + Any, + _DummyRequest( + _json={ + "conversation_id": conversation_id, + "message": {"text": "hello"}, + } + ), + ), + ) + + with pytest.raises(OSError, match="history disk unavailable"): + await server._chat_job_create_handler(request()) + + assert not manager._jobs + assert await manager.get_active_job(conversation_id) is None + page = await manager.conversation_store.get_history_page( + conversation_id, limit=50, before=None + ) + assert page.total == 0 + assert history_path.read_bytes() == original_disk + run_chat.assert_not_awaited() + + retry = await server._chat_job_create_handler(request()) + assert retry.status == 202 + job = await manager.get_job(str(json.loads(retry.text or "{}")["job_id"])) + assert job is not None and job.task is not None + await asyncio.wait_for(job.task, timeout=2) + page = await manager.conversation_store.get_history_page( + conversation_id, limit=50, before=None + ) + assert [record["message"] for record in page.records] == ["hello"] + assert len(json.loads(history_path.read_text(encoding="utf-8"))["messages"]) == 1 + assert await manager.clear_history_when_idle(conversation_id) == 1 + await server.stop() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("endpoint", ["job", "sync", "sse"]) +async def test_chat_stop_rejects_new_jobs_while_existing_job_finishes( + endpoint: str, monkeypatch: pytest.MonkeyPatch +) -> None: + started = asyncio.Event() + cancelling = asyncio.Event() + release = asyncio.Event() + calls: list[str] = [] + + async def run_chat(_ctx: Any, *, text: str, **_kwargs: Any) -> str: + calls.append(text) + started.set() + try: + await asyncio.Event().wait() + finally: + cancelling.set() + await release.wait() + return "chat" + + monkeypatch.setattr(runtime_api_chat, "run_webui_chat", run_chat) + server = RuntimeAPIServer(_context(), host="127.0.0.1", port=8788) + manager = server._chat_job_manager + job = await manager.create_job("old") + other = await manager.conversation_store.create_conversation() + await asyncio.wait_for(started.wait(), timeout=2) + stopping = asyncio.create_task(server.stop()) + try: + await asyncio.wait_for(cancelling.wait(), timeout=2) + body: dict[str, Any] = { + "conversation_id": str(other["id"]), + "message": {"text": "new"} if endpoint == "job" else "new", + "stream": endpoint == "sse", + } + request = cast(web.Request, cast(Any, _DummyRequest(_json=body))) + response: web.StreamResponse + if endpoint == "job": + response = await server._chat_job_create_handler(request) + else: + response = await server._chat_handler(request) + assert response.status == 503 + assert len(manager._jobs) == 1 + assert calls == ["old"] + release.set() + await asyncio.wait_for(stopping, timeout=2) + assert job.status == "cancelled" + assert job.done.is_set() + assert not await manager.has_running_job() + with pytest.raises(runtime_api_chat.ChatJobManagerClosingError): + await manager.create_job("after stop") + finally: + release.set() + await stopping + + +@pytest.mark.asyncio +@pytest.mark.parametrize("cancel_creation", [False, True]) +async def test_chat_stop_waits_for_in_progress_history_write( + cancel_creation: bool, monkeypatch: pytest.MonkeyPatch +) -> None: + started = threading.Event() + release = threading.Event() + server = RuntimeAPIServer(_context(), host="127.0.0.1", port=8788) + manager = server._chat_job_manager + await manager.conversation_store.ensure_ready() + conversation = await manager.conversation_store.ensure_default_conversation() + conversation_id = str(conversation["id"]) + history_path = manager.conversation_store._path_for(conversation_id) + original_write = async_io.write_json_sync + + def controlled_write( + file_path: Path | str, data: Any, use_lock: bool = True + ) -> None: + if Path(file_path) == history_path and data.get("messages"): + started.set() + if not release.wait(timeout=3): + raise TimeoutError("history write not released") + original_write(file_path, data, use_lock) + + async def run_chat(_ctx: Any, **_kwargs: Any) -> str: + await asyncio.Event().wait() + return "chat" + + monkeypatch.setattr(async_io, "write_json_sync", controlled_write) + run_mock = AsyncMock(side_effect=run_chat) + monkeypatch.setattr(runtime_api_chat, "run_webui_chat", run_mock) + creation = asyncio.create_task( + manager.create_job("hello", conversation_id, pre_record_user_history=True) + ) + stopping: asyncio.Task[None] | None = None + try: + assert await asyncio.wait_for(asyncio.to_thread(started.wait, 1), timeout=2) + stopping = asyncio.create_task(server.stop()) + await asyncio.sleep(0) + assert manager._closing + with pytest.raises(runtime_api_chat.ChatJobManagerClosingError): + await manager.create_job("new") + if cancel_creation: + creation.cancel() + await asyncio.sleep(0) + assert not creation.done() + assert not stopping.done() + release.set() + if cancel_creation: + with pytest.raises(asyncio.CancelledError): + await creation + run_mock.assert_not_awaited() + assert not manager._jobs + else: + job = await creation + await asyncio.wait_for(stopping, timeout=2) + assert job.status == "cancelled" + assert job.done.is_set() + assert job.task is not None and job.task.done() + await asyncio.wait_for(stopping, timeout=2) + assert not await manager.has_running_job() + page = await manager.conversation_store.get_history_page( + conversation_id, limit=50, before=None + ) + assert [record["message"] for record in page.records] == ["hello"] + assert ( + len(json.loads(history_path.read_text(encoding="utf-8"))["messages"]) == 1 + ) + finally: + release.set() + await asyncio.gather(creation, return_exceptions=True) + if stopping is not None: + await stopping + + @pytest.mark.asyncio async def test_runtime_api_stop_cancels_running_webchat_job( monkeypatch: pytest.MonkeyPatch, From 19b84f0f14f9072ced2bb8bea97d3b077f30a672 Mon Sep 17 00:00:00 2001 From: Null <1708213363@qq.com> Date: Tue, 6 Oct 2026 19:25:19 +0800 Subject: [PATCH 052/121] fix(runtime): drain accepted work before closing transports --- docs/deployment.md | 2 +- src/Undefined/api/app.py | 30 ++++++++- src/Undefined/main.py | 88 ++++++++++++++++++------- src/Undefined/onebot/client.py | 13 ++++ src/Undefined/weixin/service.py | 27 ++++++++ tests/test_main_shutdown.py | 113 +++++++++++++++++++++++++++++++- tests/test_onebot_file_urls.py | 56 ++++++++++++++++ tests/test_weixin_service.py | 57 ++++++++++++++++ 8 files changed, 356 insertions(+), 30 deletions(-) diff --git a/docs/deployment.md b/docs/deployment.md index ead48817..769b998f 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -156,7 +156,7 @@ uv run Undefined-webui #### 优雅停机 -`SIGINT`(Ctrl+C)与 `SIGTERM` 都会触发同一套优雅停机流程:停止 OneBot 连接、关闭 Runtime API 与微信服务、停止史官 / memes worker,再由消息处理器排空并关闭 AI 队列、落盘历史,最后断开连接、关闭 AI 客户端与检索运行时、停止配置热重载并释放渲染资源。等待 worker 收尾期间模型队列保持可用,史官在向量写入后继续提交侧写模型请求也能完成;消息处理器收尾时新落盘的认知任务留待下次启动处理。容器、systemd、supervisor 等默认以 `SIGTERM` 停止进程,无需额外配置;请避免使用 `SIGKILL`(`docker kill -s KILL`),否则会跳过全部落盘清理。 +`SIGINT`(Ctrl+C)与 `SIGTERM` 都会触发同一套优雅停机流程:先停止 QQ、微信与普通 Runtime API 的新接入,等待已接受的入站处理和 API 请求收尾,停止史官 / memes worker,再由消息处理器排空 AI 队列并落盘历史。排空期间保留 OneBot 响应接收、微信发送和 Runtime 文件下载能力,回复完成后才关闭传输、AI 客户端、检索运行时与渲染资源。等待 worker 收尾期间模型队列保持可用,史官在向量写入后继续提交侧写模型请求也能完成;消息处理器收尾时新落盘的认知任务留待下次启动处理。容器、systemd、supervisor 等默认以 `SIGTERM` 停止进程;其强制终止等待时间应覆盖实际模型和下载收尾耗时。请避免使用 `SIGKILL`(`docker kill -s KILL`),否则会跳过全部落盘清理。 #### 自动启动选项 diff --git a/src/Undefined/api/app.py b/src/Undefined/api/app.py index 15bbb0a5..d913c1c2 100644 --- a/src/Undefined/api/app.py +++ b/src/Undefined/api/app.py @@ -62,6 +62,8 @@ def __init__( self._runner: web.AppRunner | None = None self._sites: list[web.TCPSite] = [] self._background_tasks: set[asyncio.Task[Any]] = set() + self._request_tasks: set[asyncio.Task[Any]] = set() + self._stopping = False self._naga_state = NagaState() self._chat_job_manager = chat.ChatJobManager(context) transport = getattr(context.onebot, "file_transport", None) @@ -96,10 +98,22 @@ async def start(self) -> None: cfg = self._context.config_getter() logger.info("[RuntimeAPI] 已启动: %s", cfg.api.display_url) + def begin_shutdown(self) -> None: + """关闭普通 API 接入,但保留待发 OneBot 文件的 HTTP 服务。""" + self._stopping = True + self._chat_job_manager.begin_shutdown() + + async def drain_requests(self) -> None: + await self._chat_job_manager.stop() + tasks = self._request_tasks - {asyncio.current_task()} + if tasks: + await asyncio.gather(*tasks, return_exceptions=True) + async def stop(self) -> None: + self.begin_shutdown() + await self.drain_requests() if self._file_store is not None: self._file_store.unavailable() - await self._chat_job_manager.stop() for task in self._background_tasks: task.cancel() @@ -142,7 +156,19 @@ async def _auth_middleware( response = _json_error("Unauthorized", status=401) _apply_cors_headers(request, response) return response - response = await handler(request) + is_file_request = onebot_files.is_file_request(request) + if self._stopping and not is_file_request: + response = _json_error("Runtime is shutting down", status=503) + _apply_cors_headers(request, response) + return response + task = asyncio.current_task() + if task is not None and not is_file_request: + self._request_tasks.add(task) + try: + response = await handler(request) + finally: + if task is not None: + self._request_tasks.discard(task) _apply_cors_headers(request, response) return response diff --git a/src/Undefined/main.py b/src/Undefined/main.py index ec269b34..43e813d4 100644 --- a/src/Undefined/main.py +++ b/src/Undefined/main.py @@ -514,8 +514,11 @@ def _apply_config_updates( ) shutdown_guard = install_shutdown_signal_handlers(logger) + onebot_run_task: asyncio.Task[None] | None = None try: - await _run_until_shutdown(onebot, shutdown_guard.event, logger) + onebot_run_task = await _run_until_shutdown( + onebot, shutdown_guard.event, logger + ) except KeyboardInterrupt: # 仅在信号处理器注册全部失败(如非主线程运行)时才会走到这里; # 正常安装后 SIGINT 会转为停机事件,不再抛 KeyboardInterrupt @@ -524,6 +527,59 @@ def _apply_config_updates( logger.exception("[异常] 运行期间发生未捕获的错误: %s", exc) finally: logger.info("[清理] 正在关闭机器人并释放资源...") + try: + await _close_message_runtime( + onebot, + handler, + runtime_api_server=runtime_api_server, + weixin_service=weixin_service, + meme_worker=meme_worker, + historian_worker=historian_worker, + logger=logger, + ) + finally: + if onebot_run_task is not None: + onebot_run_task.cancel() + await asyncio.gather(onebot_run_task, return_exceptions=True) + await ai.close() + if retrieval_registry is not None: + await retrieval_registry.stop() + await config_manager.stop_hot_reload() + await close_render_browser() + await close_render_cache() + shutdown_guard.restore() + logger.info("[退出] 机器人已停止运行") + + +async def _close_message_runtime( + onebot: OneBotClient, + handler: MessageHandler, + *, + runtime_api_server: RuntimeAPIServer | None, + weixin_service: WeixinService | None, + meme_worker: MemeWorker | None, + historian_worker: HistorianWorker | None, + logger: logging.Logger, +) -> None: + """先关闭新接入并排空已接受任务,最后释放回复所依赖的传输资源。""" + onebot.stop_accepting_messages() + if weixin_service is not None: + weixin_service.stop_accepting_messages() + if runtime_api_server is not None: + runtime_api_server.begin_shutdown() + try: + if runtime_api_server is not None: + await runtime_api_server.drain_requests() + await onebot.drain_message_tasks() + if weixin_service is not None: + await weixin_service.drain_message_tasks() + await _close_message_workers( + handler, + meme_worker=meme_worker, + historian_worker=historian_worker, + logger=logger, + ) + finally: if runtime_api_server is not None: try: await runtime_api_server.stop() @@ -534,21 +590,7 @@ def _apply_config_updates( await weixin_service.stop() except Exception: logger.exception("[清理] WeixinService stop 失败") - await _close_message_workers( - handler, - meme_worker=meme_worker, - historian_worker=historian_worker, - logger=logger, - ) await onebot.disconnect() - await ai.close() - if retrieval_registry is not None: - await retrieval_registry.stop() - await config_manager.stop_hot_reload() - await close_render_browser() - await close_render_cache() - shutdown_guard.restore() - logger.info("[退出] 机器人已停止运行") async def _close_message_workers( @@ -659,8 +701,8 @@ async def _run_until_shutdown( onebot: OneBotClient, shutdown_event: asyncio.Event, logger: logging.Logger, -) -> None: - """运行 OneBot 连接,收到停止信号或连接任务结束时返回。""" +) -> asyncio.Task[None]: + """等待停机条件;返回仍可收发的连接任务,由调用方在排空后关闭。""" run_task = asyncio.create_task(onebot.run_with_reconnect(), name="onebot-run") stop_task = asyncio.create_task(shutdown_event.wait(), name="shutdown-wait") try: @@ -671,15 +713,13 @@ async def _run_until_shutdown( if run_task in done: # 连接任务自行结束:把异常抛给上层处理 run_task.result() - return + return run_task logger.info("[退出] 收到停止信号 (SIGTERM/SIGINT),正在优雅停机...") + return run_task + except BaseException: run_task.cancel() - try: - await run_task - except asyncio.CancelledError: - pass - except Exception as exc: - logger.warning("[退出] 停止 OneBot 连接时发生异常: %s", exc) + await asyncio.gather(run_task, return_exceptions=True) + raise finally: stop_task.cancel() try: diff --git a/src/Undefined/onebot/client.py b/src/Undefined/onebot/client.py index 3639b9e7..90a930e4 100644 --- a/src/Undefined/onebot/client.py +++ b/src/Undefined/onebot/client.py @@ -126,6 +126,7 @@ def __init__( Callable[[dict[str, Any]], Coroutine[Any, Any, None]] | None ) = None self._running = False + self._accepting_messages = True self._tasks: set[asyncio.Task[None]] = set() self.file_transport = ( file_transport @@ -208,6 +209,15 @@ async def _cancel_message_tasks(self) -> None: if tasks: await asyncio.gather(*tasks, return_exceptions=True) + def stop_accepting_messages(self) -> None: + """停止接收新事件,保留 API 响应接收与发送能力供已接受任务收尾。""" + self._accepting_messages = False + + async def drain_message_tasks(self) -> None: + """等待已经接受的入站处理完成;调用前须关闭新消息接入。""" + if self._tasks: + await asyncio.gather(*tuple(self._tasks), return_exceptions=True) + async def _call_api( self, action: str, @@ -987,6 +997,9 @@ async def _dispatch_message(self, data: dict[str, Any]) -> None: ) return + if not self._accepting_messages: + return + # 事件类型的消息异步处理,不阻塞接收循环 post_type = data.get("post_type") if post_type == "message": diff --git a/src/Undefined/weixin/service.py b/src/Undefined/weixin/service.py index 332eac06..c757a090 100644 --- a/src/Undefined/weixin/service.py +++ b/src/Undefined/weixin/service.py @@ -260,6 +260,8 @@ def __init__( self._lifecycle_lock = asyncio.Lock() self._running = False self._closed = False + self._accepting_messages = True + self._inbound_completions: set[asyncio.Future[None]] = set() def _build_client( self, @@ -316,6 +318,14 @@ async def stop(self) -> None: self._running = False self._closed = True + def stop_accepting_messages(self) -> None: + """停止新入站消息,保持已连接帐号可发送排空中的回复。""" + self._accepting_messages = False + + async def drain_message_tasks(self) -> None: + if self._inbound_completions: + await asyncio.gather(*tuple(self._inbound_completions)) + async def status(self) -> dict[str, Any]: accounts = await self.store.list_accounts() return { @@ -857,6 +867,23 @@ async def _handle_inbound( alias: str, client: WeixinClientProtocol, message: InboundMessage, + ) -> None: + if not self._accepting_messages: + return + completion: asyncio.Future[None] = asyncio.get_running_loop().create_future() + self._inbound_completions.add(completion) + try: + await self._process_inbound(alias, client, message) + finally: + self._inbound_completions.discard(completion) + if not completion.done(): + completion.set_result(None) + + async def _process_inbound( + self, + alias: str, + client: WeixinClientProtocol, + message: InboundMessage, ) -> None: account = await self.store.get_account(alias) if account is None: diff --git a/tests/test_main_shutdown.py b/tests/test_main_shutdown.py index e1015c0e..3ef881b8 100644 --- a/tests/test_main_shutdown.py +++ b/tests/test_main_shutdown.py @@ -1,6 +1,7 @@ from __future__ import annotations import asyncio +import json import logging import os import signal @@ -14,10 +15,12 @@ from Undefined.handlers import MessageHandler from Undefined.main import ( _close_message_workers, + _close_message_runtime, _run_until_shutdown, install_shutdown_signal_handlers, ) from Undefined.services.queue_manager import QueueManager +from Undefined.onebot.client import OneBotClient logger = logging.getLogger("test-main-shutdown") @@ -123,7 +126,7 @@ async def run_with_reconnect(self) -> None: @pytest.mark.asyncio -async def test_run_until_shutdown_cancels_onebot_on_signal() -> None: +async def test_run_until_shutdown_keeps_onebot_alive_for_drain() -> None: onebot = _SlowOneBot() shutdown_event = asyncio.Event() @@ -132,11 +135,115 @@ async def test_run_until_shutdown_cancels_onebot_on_signal() -> None: ) await asyncio.sleep(0.05) shutdown_event.set() - await asyncio.wait_for(task, timeout=2) - + connection = await asyncio.wait_for(task, timeout=2) + assert onebot.cancelled is False + assert not connection.done() + connection.cancel() + await asyncio.gather(connection, return_exceptions=True) assert onebot.cancelled is True +class _ReplyWebSocket: + close_code: int | None = None + + def __init__(self) -> None: + self.frames: asyncio.Queue[str] = asyncio.Queue() + self.receiving = asyncio.Event() + self.sent: list[dict[str, Any]] = [] + + async def recv(self) -> str: + self.receiving.set() + return await self.frames.get() + + async def send(self, payload: str) -> None: + request = json.loads(payload) + self.sent.append(request) + await self.frames.put(json.dumps({"echo": request["echo"], "status": "ok"})) + + async def close(self) -> None: + self.close_code = 1000 + await self.frames.put("{}") + + +@pytest.mark.asyncio +async def test_shutdown_finishes_accepted_reply_before_closing_transport( + monkeypatch: pytest.MonkeyPatch, +) -> None: + onebot = OneBotClient("ws://isolated.invalid") + socket = _ReplyWebSocket() + onebot.ws = cast(Any, socket) + monkeypatch.setattr(onebot, "connect", AsyncMock()) + shutdown = asyncio.Event() + wait_task = asyncio.create_task(_run_until_shutdown(onebot, shutdown, logger)) + await asyncio.wait_for(socket.receiving.wait(), 1) + queue = QueueManager(ai_request_interval=0, max_retries=0) + reply_started = asyncio.Event() + release_reply = asyncio.Event() + order: list[str] = [] + + async def reply(request: dict[str, Any]) -> None: + reply_started.set() + await release_reply.wait() + response = await onebot.send_group_message(1, "accepted reply") + assert response["status"] == "ok" + order.append("delivered") + + calls: list[str] = [] + + async def inbound(data: dict[str, Any]) -> None: + calls.append("accepted") + await queue.add_private_request({"type": "private_reply"}) + + queue.start(reply) + onebot.set_message_handler(inbound) + await onebot._dispatch_message({"post_type": "message"}) + await asyncio.wait_for(reply_started.wait(), 1) + handler = MessageHandler.__new__(MessageHandler) + handler._background_tasks = set() + cast(Any, handler).pipeline_registry = SimpleNamespace(stop_hot_reload=AsyncMock()) + cast(Any, handler).ai_coordinator = SimpleNamespace(queue_manager=queue) + cast(Any, handler).message_batcher = SimpleNamespace(flush_all=AsyncMock()) + + async def flush() -> None: + order.append("history") + + cast(Any, handler).history_manager = SimpleNamespace(flush_pending_saves=flush) + shutdown.set() + connection = await asyncio.wait_for(wait_task, 1) + close_task = asyncio.create_task( + _close_message_runtime( + onebot, + handler, + runtime_api_server=None, + weixin_service=None, + meme_worker=None, + historian_worker=None, + logger=logger, + ) + ) + try: + await asyncio.sleep(0) + assert onebot._running + assert not close_task.done() + await onebot._dispatch_message({"post_type": "message"}) + await onebot._dispatch_message( + {"post_type": "notice", "notice_type": "notify", "sub_type": "poke"} + ) + assert calls == ["accepted"] + release_reply.set() + await asyncio.wait_for(close_task, 2) + await asyncio.wait_for(connection, 1) + assert order == ["delivered", "history"] + assert len(socket.sent) == 1 + assert socket.frames.empty() + finally: + release_reply.set() + close_task.cancel() + connection.cancel() + await asyncio.gather(close_task, connection, return_exceptions=True) + await queue.stop() + + @pytest.mark.asyncio async def test_run_until_shutdown_propagates_connection_error() -> None: with pytest.raises(RuntimeError, match="connection boom"): diff --git a/tests/test_onebot_file_urls.py b/tests/test_onebot_file_urls.py index a4f45817..d26b7cf9 100644 --- a/tests/test_onebot_file_urls.py +++ b/tests/test_onebot_file_urls.py @@ -116,6 +116,62 @@ async def test_get_head_range_tokens_source_cleanup_mode_and_actual_port( assert token not in caplog.text +async def test_shutdown_blocks_api_but_keeps_url_delivery_available( + tmp_path: Path, +) -> None: + source = tmp_path / "pending.txt" + source.write_bytes(b"accepted media") + async with runtime(tmp_path) as (server, store, _, _, _): + server.begin_shutdown() + await server.drain_requests() + url = await store.publish(source, "127.0.0.1") + async with ClientSession() as http: + async with http.get(f"http://127.0.0.1:{store.port}/health") as response: + assert response.status == 503 + async with http.get(url) as response: + assert response.status == 200 + assert await response.read() == b"accepted media" + + +async def test_shutdown_waits_for_accepted_runtime_request( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + entered = asyncio.Event() + release = asyncio.Event() + + async def health(self: RuntimeAPIServer, request: web.Request) -> web.Response: + entered.set() + await release.wait() + return web.json_response({"ok": True}) + + monkeypatch.setattr(RuntimeAPIServer, "_health_handler", health) + async with runtime(tmp_path) as (server, store, _, _, _): + async with ClientSession() as http: + + async def fetch_health() -> int: + async with http.get( + f"http://127.0.0.1:{store.port}/health" + ) as response: + await response.read() + return response.status + + request = asyncio.create_task(fetch_health()) + await asyncio.wait_for(entered.wait(), 1) + server.begin_shutdown() + drain = asyncio.create_task(server.drain_requests()) + try: + await asyncio.sleep(0) + assert not drain.done() + release.set() + assert await asyncio.wait_for(request, 1) == 200 + await asyncio.wait_for(drain, 1) + assert not server._request_tasks + finally: + release.set() + await asyncio.gather(request, drain, return_exceptions=True) + + async def test_tokens_are_per_file_and_ipv6_format(tmp_path: Path) -> None: source = tmp_path / "file" source.write_bytes(b"abc") diff --git a/tests/test_weixin_service.py b/tests/test_weixin_service.py index a9eb1a46..26999d12 100644 --- a/tests/test_weixin_service.py +++ b/tests/test_weixin_service.py @@ -238,6 +238,63 @@ def _account() -> WeixinAccount: ) +@pytest.mark.asyncio +async def test_quiesce_keeps_sending_available_until_accepted_inbound_finishes( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + config = _config(tmp_path) + store = WeixinStore(config.weixin) + await store.save_account(_account()) + client = FakeClient() + + def factory( + account: WeixinAccount, + state: UndefinedIlinkStateStore, + options: ClientOptions, + ) -> WeixinClientProtocol: + return client + + service = WeixinService( + config, + store=store, + login_manager=cast(QrLoginManager, FakeLoginManager()), + client_factory=factory, + ) + entered = asyncio.Event() + release = asyncio.Event() + + async def process( + alias: str, current_client: WeixinClientProtocol, message: InboundMessage + ) -> None: + entered.set() + await release.wait() + await service.send_text(10001, "accepted reply") + + monkeypatch.setattr(service, "_process_inbound", process) + message = InboundMessage.from_mapping("bot-account", {"from_user_id": "peer-1"}) + await service.start() + await asyncio.wait_for(client.run_started.wait(), 1) + task = asyncio.create_task(service._handle_inbound("primary", client, message)) + await asyncio.wait_for(entered.wait(), 1) + service.stop_accepting_messages() + drain = asyncio.create_task(service.drain_message_tasks()) + try: + await service._handle_inbound("primary", client, message) + await asyncio.sleep(0) + assert not drain.done() + assert not client.closed + release.set() + await asyncio.wait_for(drain, 1) + await task + assert client.sent_text == [("peer-1", "accepted reply")] + finally: + release.set() + await asyncio.gather(task, drain, return_exceptions=True) + await service.stop() + assert client.closed + + @pytest.mark.asyncio async def test_privileged_binding_requires_confirmation_before_network( tmp_path: Path, From 217deba0e6562390f9c7e9cf5e8eed411197dab2 Mon Sep 17 00:00:00 2001 From: Null <1708213363@qq.com> Date: Tue, 6 Oct 2026 19:26:03 +0800 Subject: [PATCH 053/121] fix(chat): retain blobs while images are displayed --- apps/undefined-chat/README.md | 2 +- .../src/image-viewer/ImageViewerModal.tsx | 2 + .../src/rendering/AttachmentImage.tsx | 6 +- .../src/rendering/AttachmentImageContext.tsx | 71 ++++++-- .../src/rendering/blob-lifecycle.test.tsx | 154 ++++++++++++++++++ 5 files changed, 218 insertions(+), 17 deletions(-) create mode 100644 apps/undefined-chat/src/rendering/blob-lifecycle.test.tsx diff --git a/apps/undefined-chat/README.md b/apps/undefined-chat/README.md index d256f0da..16813556 100644 --- a/apps/undefined-chat/README.md +++ b/apps/undefined-chat/README.md @@ -10,7 +10,7 @@ Undefined Chat 是 `apps/undefined-chat/` 下的 Tauri v2 + React 19 原生聊 - 消息历史:初始加载、cursor-based 加载更早消息、滚动锚点保持和长历史窗口化渲染。同一会话的重载与分页共用请求顺序保护;发送新消息、完成任务刷新或删除会话后,迟到的旧响应不会覆盖正文、游标或加载状态。 - 消息渲染:Markdown、表格、任务列表、引用块、代码高亮、代码块复制;未知语言或高亮失败时保留为纯文本代码,代码中的 HTML 不会插入正文。正文 HTML 经 sanitize 内联渲染,HTML 代码块可在独立预览窗口中运行。 - 附件:系统文件选择器、上传状态队列、Tauri 流式上传、下载和预览。当前显示上传中/成功/失败状态,不显示百分比进度。 -- 图片:正文 `` 内联图、附件缩略图、blob 缓存、全屏查看、缩放、旋转和可滚动查看区域。 +- 图片:正文 `` 内联图、附件缩略图、blob 缓存、全屏查看、缩放、旋转和可滚动查看区域。LRU 目标缓存 60 张;挂载中的图片和已打开的大图持有 blob 引用,超过目标时只淘汰无人使用的项,关闭或卸载消费者后再释放多余缓存。 - 命令:斜杠命令、子命令提示、方向键导航、Tab/Enter 补全;空态反馈(加载中/无匹配/不可用);窗口聚焦按 TTL 刷新命令列表。 - 引用:引用 bot 消息、划词引用选中文本、发送引用、引用芯片跳转当前已加载源消息;源消息不在当前页时会自动加载一页更早历史,仍未命中需手动继续加载更早历史。 - 工具调用块:层级展示、状态、运行中实时计时、阶段明细(stage detail)、与 WebUI 一致的输入/输出预览(JSON/Python 风格结构化、Markdown 输出、附件图片预览);历史回放支持 timeline/calls/events 多级回退。 diff --git a/apps/undefined-chat/src/image-viewer/ImageViewerModal.tsx b/apps/undefined-chat/src/image-viewer/ImageViewerModal.tsx index 60018c72..5e096574 100644 --- a/apps/undefined-chat/src/image-viewer/ImageViewerModal.tsx +++ b/apps/undefined-chat/src/image-viewer/ImageViewerModal.tsx @@ -1,6 +1,7 @@ import { useEffect, useRef, useState } from "react"; import type { ImageViewerState } from "../chat-store/types"; import { useTranslation } from "../i18n"; +import { useRetainedAttachmentBlobUrl } from "../rendering/AttachmentImageContext"; export type ImageViewerModalProps = { imageViewer: ImageViewerState | null; @@ -15,6 +16,7 @@ export function ImageViewerModal({ const previousActiveElementRef = useRef(null); const [zoom, setZoom] = useState(1); const [rotation, setRotation] = useState(0); + useRetainedAttachmentBlobUrl(imageViewer?.open ? imageViewer.src : null); useEffect(() => { if (imageViewer?.open) { diff --git a/apps/undefined-chat/src/rendering/AttachmentImage.tsx b/apps/undefined-chat/src/rendering/AttachmentImage.tsx index 2169fc9a..5b3ed682 100644 --- a/apps/undefined-chat/src/rendering/AttachmentImage.tsx +++ b/apps/undefined-chat/src/rendering/AttachmentImage.tsx @@ -33,12 +33,13 @@ export function AttachmentImage({ style, onOpenImage, }: AttachmentImageProps) { - const { loadAttachmentBlob } = useAttachmentImage(); + const { loadAttachmentBlob, retainAttachmentBlob } = useAttachmentImage(); const { t } = useTranslation(); const [state, setState] = useState({ status: "loading" }); useEffect(() => { let cancelled = false; + const release = retainAttachmentBlob(uid); setState({ status: "loading" }); loadAttachmentBlob(uid) .then((loaded) => { @@ -52,8 +53,9 @@ export function AttachmentImage({ }); return () => { cancelled = true; + release(); }; - }, [uid, loadAttachmentBlob]); + }, [uid, loadAttachmentBlob, retainAttachmentBlob]); if (state.status === "loading") { return ( diff --git a/apps/undefined-chat/src/rendering/AttachmentImageContext.tsx b/apps/undefined-chat/src/rendering/AttachmentImageContext.tsx index c8484434..6a594680 100644 --- a/apps/undefined-chat/src/rendering/AttachmentImageContext.tsx +++ b/apps/undefined-chat/src/rendering/AttachmentImageContext.tsx @@ -4,6 +4,7 @@ import { useCallback, useContext, useEffect, + useLayoutEffect, useMemo, } from "react"; import type { RuntimeClient } from "../runtime-client/types"; @@ -15,6 +16,8 @@ export type LoadedAttachmentBlob = { type AttachmentImageContextValue = { loadAttachmentBlob: (uid: string) => Promise; + retainAttachmentBlob: (uid: string) => () => void; + retainBlobUrl: (url: string) => (() => void) | undefined; }; const AttachmentImageContext = @@ -27,7 +30,7 @@ export type AttachmentImageProviderProps = { client: Pick; /** 按 Runtime URL 隔离 UID 缓存,保持消费组件与 DOM 的生命周期。 */ runtimeUrl?: string | null; - /** blob 缓存上限,超出按 LRU 淘汰并 revoke。默认 60。 */ + /** blob 缓存目标上限;超出时仅淘汰无消费者的 LRU 项。默认 60。 */ maxCacheSize?: number; children: ReactNode; }; @@ -42,8 +45,8 @@ export type AttachmentImageProviderProps = { * - LRU 缓存 uid→blobUrl:缩略图/正文图/全屏大图复用同一 blob,避免重复拉取。 * - inflight 去重:同一 UID 并发请求只发一次。 * - blob URL 归本 Provider 所有:消费组件**不**自行 revoke(缩略图与大图共享同一 - * URL,组件卸载即 revoke 会让仍打开的大图破图),仅在 LRU 淘汰、切换 Runtime - * 或 Provider 卸载时 revoke。 + * URL,组件卸载即 revoke 会让仍打开的大图破图)。消费者持有引用期间跳过 + * LRU 淘汰;切换 Runtime 或 Provider 卸载时统一 revoke。 */ export function AttachmentImageProvider({ client, @@ -56,11 +59,48 @@ export function AttachmentImageProvider({ runtimeUrl, cache: new Map(), inflight: new Map>(), + consumers: new Map(), generation: 0, }), [runtimeUrl], ); + const evictUnused = useCallback(() => { + for (const [uid, loaded] of scope.cache) { + if (scope.cache.size <= maxCacheSize) break; + if (scope.consumers.has(uid)) continue; + scope.cache.delete(uid); + URL.revokeObjectURL(loaded.url); + } + }, [maxCacheSize, scope]); + + const retainAttachmentBlob = useCallback( + (uid: string): (() => void) => { + const generation = scope.generation; + scope.consumers.set(uid, (scope.consumers.get(uid) ?? 0) + 1); + let released = false; + return () => { + if (released || generation !== scope.generation) return; + released = true; + const remaining = (scope.consumers.get(uid) ?? 1) - 1; + if (remaining > 0) scope.consumers.set(uid, remaining); + else scope.consumers.delete(uid); + evictUnused(); + }; + }, + [scope, evictUnused], + ); + + const retainBlobUrl = useCallback( + (url: string): (() => void) | undefined => { + for (const [uid, loaded] of scope.cache) { + if (loaded.url === url) return retainAttachmentBlob(uid); + } + return undefined; + }, + [scope, retainAttachmentBlob], + ); + const loadAttachmentBlob = useCallback( (uid: string): Promise => { const { cache, inflight } = scope; @@ -93,14 +133,7 @@ export function AttachmentImageProvider({ mediaType, }; cache.set(uid, loaded); - // 超上限淘汰最旧(队首)并 revoke - while (cache.size > maxCacheSize) { - const oldestKey = cache.keys().next().value; - if (oldestKey === undefined) break; - const victim = cache.get(oldestKey); - cache.delete(oldestKey); - if (victim) URL.revokeObjectURL(victim.url); - } + evictUnused(); return loaded; } catch { return null; @@ -112,7 +145,7 @@ export function AttachmentImageProvider({ inflight.set(uid, promise); return promise; }, - [client, maxCacheSize, scope], + [client, evictUnused, scope], ); // 切换 Runtime 或卸载时释放全部 blob URL,拒绝旧作用域迟到的响应。 @@ -121,6 +154,7 @@ export function AttachmentImageProvider({ return () => { scope.generation += 1; inflight.clear(); + scope.consumers.clear(); for (const { url } of cache.values()) { URL.revokeObjectURL(url); } @@ -129,8 +163,8 @@ export function AttachmentImageProvider({ }, [scope]); const value = useMemo( - () => ({ loadAttachmentBlob }), - [loadAttachmentBlob], + () => ({ loadAttachmentBlob, retainAttachmentBlob, retainBlobUrl }), + [loadAttachmentBlob, retainAttachmentBlob, retainBlobUrl], ); return ( @@ -152,3 +186,12 @@ export function useAttachmentImage(): AttachmentImageContextValue { } return value; } + +/** 大图独立持有 Provider 的 URL;外部图片和无 Provider 的查看器无需持有。 */ +export function useRetainedAttachmentBlobUrl(url: string | null): void { + const value = useContext(AttachmentImageContext); + // 在被动 effect 释放缩略图前接管引用,支持同次渲染中移除源缩略图。 + useLayoutEffect(() => { + if (url) return value?.retainBlobUrl(url); + }, [value, url]); +} diff --git a/apps/undefined-chat/src/rendering/blob-lifecycle.test.tsx b/apps/undefined-chat/src/rendering/blob-lifecycle.test.tsx new file mode 100644 index 00000000..5e878248 --- /dev/null +++ b/apps/undefined-chat/src/rendering/blob-lifecycle.test.tsx @@ -0,0 +1,154 @@ +import { fireEvent, render, screen, waitFor } from "@testing-library/react"; +import { StrictMode, useState } from "react"; +import { afterEach, describe, expect, test, vi } from "vitest"; +import type { ImageViewerState } from "../chat-store/types"; +import { LanguageProvider } from "../i18n"; +import { ImageViewerModal } from "../image-viewer/ImageViewerModal"; +import { runtimeClientStub } from "../test-fixtures"; +import { AttachmentImage } from "./AttachmentImage"; +import { AttachmentImageProvider } from "./AttachmentImageContext"; + +function Gallery({ + images, + removeOnOpen = false, +}: { + images: { id: string; uid: string }[]; + removeOnOpen?: boolean; +}) { + const [viewer, setViewer] = useState(null); + const [hideThumbnails, setHideThumbnails] = useState(false); + return ( + <> + {!hideThumbnails && + images.map(({ id, uid }) => ( + { + setViewer({ open: true, src, alt }); + if (removeOnOpen) setHideThumbnails(true); + }} + /> + ))} + setViewer(null)} /> + + ); +} + +describe("active image blob lifetimes", () => { + afterEach(() => vi.restoreAllMocks()); + + test("61 mounted images keep their URLs, and an open viewer retains a removed thumbnail", async () => { + const revoke = vi.spyOn(URL, "revokeObjectURL").mockClear(); + const client = runtimeClientStub(); + const uids = Array.from({ length: 61 }, (_, index) => `pic_${index}`); + const tree = (shown: string[]) => ( + + + ({ id: uid, uid }))} /> + + + ); + const { rerender, unmount } = render(tree(uids.slice(0, 1))); + const first = (await screen.findByAltText("pic_0")) as HTMLImageElement; + const firstUrl = first.src; + rerender(tree(uids)); + await waitFor(() => + expect(document.querySelectorAll("img")).toHaveLength(61), + ); + expect(screen.getByAltText("pic_0")).toBe(first); + expect(first).toHaveAttribute("src", firstUrl); + expect(revoke).not.toHaveBeenCalled(); + expect(firstUrl).toMatch(/^blob:/); + expect((await (await fetch(firstUrl)).arrayBuffer()).byteLength).toBe(4); + fireEvent.click(first); + expect(screen.getByRole("dialog").querySelector("img")).toHaveAttribute( + "src", + firstUrl, + ); + rerender(tree(uids.slice(1))); + expect(first.isConnected).toBe(false); + expect(revoke).not.toHaveBeenCalledWith(firstUrl); + expect((await (await fetch(firstUrl)).arrayBuffer()).byteLength).toBe(4); + fireEvent.click(screen.getByRole("button", { name: "关闭" })); + expect(revoke).toHaveBeenCalledWith(firstUrl); + expect(revoke).toHaveBeenCalledTimes(1); + await expect(fetch(firstUrl)).rejects.toThrow(); + unmount(); + expect(revoke).toHaveBeenCalledTimes(61); + }); + + test("two consumers sharing a UID retain it until both unmount", async () => { + const revoke = vi.spyOn(URL, "revokeObjectURL").mockClear(); + const client = runtimeClientStub(); + const tree = (count: number) => ( + + + ({ + id, + uid: "pic_shared", + }))} + /> + + + ); + const { rerender } = render(tree(2)); + const first = await screen.findByAltText("first"); + const second = await screen.findByAltText("second"); + const url = first.getAttribute("src"); + expect(second).toHaveAttribute("src", url); + expect(client.previewAttachment).toHaveBeenCalledTimes(1); + rerender(tree(1)); + expect(revoke).not.toHaveBeenCalledWith(url); + rerender(tree(0)); + expect(revoke).toHaveBeenCalledWith(url); + }); + + test("opening a viewer and removing its source in the same render transfers the reference", async () => { + const revoke = vi.spyOn(URL, "revokeObjectURL").mockClear(); + render( + + + + + , + ); + const thumbnail = await screen.findByAltText("handoff"); + const url = thumbnail.getAttribute("src"); + fireEvent.click(thumbnail); + expect(thumbnail.isConnected).toBe(false); + expect(screen.getByRole("dialog").querySelector("img")).toHaveAttribute( + "src", + url, + ); + expect(revoke).not.toHaveBeenCalledWith(url); + fireEvent.click(screen.getByRole("button", { name: "关闭" })); + expect(revoke).toHaveBeenCalledWith(url); + }); + + test("StrictMode cleanup does not discard the current consumer's blob", async () => { + const revoke = vi.spyOn(URL, "revokeObjectURL").mockClear(); + const { unmount } = render( + + + + + + + , + ); + const thumbnail = await screen.findByAltText("strict"); + const url = thumbnail.getAttribute("src"); + expect(revoke).not.toHaveBeenCalledWith(url); + unmount(); + expect(revoke).toHaveBeenCalledWith(url); + }); +}); From 590790e60a67c0b4d0cdc75f23a36beea98aefb0 Mon Sep 17 00:00:00 2001 From: Null <1708213363@qq.com> Date: Tue, 6 Oct 2026 19:34:50 +0800 Subject: [PATCH 054/121] fix(console): preserve schemes and IPv6 in profile migration --- apps/undefined-console/README.md | 10 +- apps/undefined-console/biome.json | 1 + apps/undefined-console/package.json | 3 +- .../src/connection-profiles.test.cjs | 252 ++++++++++++++++++ apps/undefined-console/src/main.ts | 85 +++++- 5 files changed, 337 insertions(+), 14 deletions(-) create mode 100644 apps/undefined-console/src/connection-profiles.test.cjs diff --git a/apps/undefined-console/README.md b/apps/undefined-console/README.md index c89902d8..67ee0771 100644 --- a/apps/undefined-console/README.md +++ b/apps/undefined-console/README.md @@ -7,7 +7,7 @@ A `Tauri v2` management client for Undefined that targets the remote Management - Desktop builds for `Windows`, `macOS`, and `Linux` - Android build support from the same codebase - Mobile-friendly shell for connecting to remote instances -- Separate connection profiles for `Management` and `Runtime-only` modes +- Local connection profiles with separate Management and Runtime protocol/port settings ## Scripts @@ -39,8 +39,12 @@ npm run tauri:build:no-strip ## Connection model -- **Management mode**: connect with `management_url` and a password/token flow. -- **Runtime-only mode**: connect with `runtime_url` and `X-Undefined-API-Key` compatible credentials. +- **Management**: open the configured WebUI, optionally logging in with its management password before navigation. +- **Runtime**: probe the configured Runtime's `/health` and `/openapi.json`. Chat authentication is configured in Undefined Chat or the WebUI. + +The connection editor stores a host and separate protocol/port choices for Management and Runtime. New profiles use HTTP by default; select HTTPS for TLS endpoints. Existing profiles using `managementUrl` / `runtimeUrl` are migrated with their original HTTP/HTTPS protocols and ports. IPv6 hosts accept either `::1` or `[::1]` and produce a URL with a single pair of brackets. Saving and reloading a migrated profile preserves these choices. + +`npm test` runs the connection migration, save/reload, probe, and login regressions with in-memory storage and network/navigation stubs. It is included in `npm run check` and never contacts a real instance. This scaffold intentionally keeps the frontend shell lightweight. It is suitable for embedding into both browser-hosted WebUI and Tauri clients. diff --git a/apps/undefined-console/biome.json b/apps/undefined-console/biome.json index 98b69ab7..f60f58d2 100644 --- a/apps/undefined-console/biome.json +++ b/apps/undefined-console/biome.json @@ -5,6 +5,7 @@ "files": { "includes": [ "**/src/**/*.ts", + "**/src/**/*.cjs", "**/package.json", "**/tsconfig.json", "**/vite.config.ts", diff --git a/apps/undefined-console/package.json b/apps/undefined-console/package.json index 131b5758..d611a9fc 100644 --- a/apps/undefined-console/package.json +++ b/apps/undefined-console/package.json @@ -9,9 +9,10 @@ "lint": "biome check src package.json tsconfig.json vite.config.ts src-tauri/tauri.conf.json src-tauri/capabilities/default.json", "lint:webui": "biome check --config-path ../../biome.json ../../src/Undefined/webui/static/js", "typecheck": "tsc --noEmit", + "test": "node --test src/connection-profiles.test.cjs", "tauri:fmt:check": "cargo fmt --manifest-path src-tauri/Cargo.toml --all --check", "tauri:check": "cargo check --manifest-path src-tauri/Cargo.toml", - "check": "npm run lint && npm run lint:webui && npm run typecheck && npm run tauri:fmt:check && npm run tauri:check", + "check": "npm run lint && npm run lint:webui && npm run typecheck && npm run test && npm run tauri:fmt:check && npm run tauri:check", "build": "npm run typecheck && vite build", "preview": "vite preview --host 0.0.0.0 --port 4173", "tauri:dev": "tauri dev", diff --git a/apps/undefined-console/src/connection-profiles.test.cjs b/apps/undefined-console/src/connection-profiles.test.cjs new file mode 100644 index 00000000..8bb0c50e --- /dev/null +++ b/apps/undefined-console/src/connection-profiles.test.cjs @@ -0,0 +1,252 @@ +const assert = require("node:assert/strict"); +const { readFileSync } = require("node:fs"); +const { join } = require("node:path"); +const { test } = require("node:test"); +const ts = require("typescript"); + +const PROFILE_KEY = "undefined.console.profiles"; +const source = readFileSync(join(__dirname, "main.ts"), "utf8"); +const compiled = ts.transpileModule(source, { + compilerOptions: { + module: ts.ModuleKind.CommonJS, + target: ts.ScriptTarget.ES2022, + }, +}).outputText; + +// Execute the actual app and its form handlers. Storage, HTTP and navigation are +// isolated in memory; imports outside the two mocked Tauri modules are rejected. +function loadConsole(profiles, { native = false } = {}) { + const storage = new Map(); + if (profiles !== undefined) + storage.set(PROFILE_KEY, JSON.stringify(profiles)); + const calls = []; + const navigations = []; + const app = { innerHTML: "" }; + const form = { + values: {}, + addEventListener(event, handler) { + this[event] = handler; + }, + }; + const document = { + documentElement: { lang: "", dataset: {} }, + querySelector(selector) { + if (selector === "#app") return app; + if (selector === "#profile-form") return form; + return null; + }, + querySelectorAll: () => [], + }; + const request = async (url, init = {}) => { + // No branch invokes Node/global fetch or any real Runtime. + calls.push({ url, ...init }); + return new Response( + JSON.stringify({ + access_token: "FAKE-TOKEN", + refresh_token: "FAKE-REFRESH", + }), + { status: 200, headers: { "Content-Type": "application/json" } }, + ); + }; + const exports = {}; + const load = new Function( + "require", + "exports", + "localStorage", + "document", + "window", + "fetch", + "FormData", + `${compiled}\nreturn { state, openWebui, runProbes, buildManagementUrl, buildRuntimeUrl };`, + ); + const consoleApp = load( + (name) => { + if (name === "@tauri-apps/api/core") return { isTauri: () => native }; + if (name === "@tauri-apps/plugin-http") return { fetch: request }; + if (name === "./style.css") return {}; + throw new Error(`Unexpected import: ${name}`); + }, + exports, + { + getItem: (key) => storage.get(key) ?? null, + setItem: (key, value) => storage.set(key, value), + }, + document, + { + location: { + href: "tauri://localhost/", + assign: (url) => navigations.push(url), + }, + }, + request, + class FormData { + constructor(currentForm) { + this.values = currentForm.values; + } + get(key) { + return this.values[key] ?? null; + } + }, + ); + return { + ...consoleApp, + app, + storage, + calls, + navigations, + submit(values) { + form.values = values; + form.submit({ currentTarget: form, preventDefault() {} }); + }, + }; +} + +test("legacy HTTPS origins stay HTTPS through the actual password login and probes", async () => { + const app = loadConsole( + [ + { + id: "tls", + managementUrl: "https://secure.example:443", + runtimeUrl: "https://secure.example:9443", + password: "FAKE-PASSWORD", + }, + ], + { native: true }, + ); + const profile = app.state.profiles[0]; + assert.equal(profile.managementScheme, "https"); + assert.equal(profile.runtimeScheme, "https"); + assert.equal(profile.managementPort, "443"); + assert.equal(profile.runtimePort, "9443"); + await app.openWebui(); + assert.equal( + app.calls[0].url, + "https://secure.example:443/api/v1/management/auth/login", + ); + assert.equal(app.calls[0].method, "POST"); + assert.equal(app.calls[0].body, '{"password":"FAKE-PASSWORD"}'); + assert.equal(new URL(app.navigations[0]).origin, "https://secure.example"); + await app.runProbes(); + assert.ok(app.calls.every((call) => new URL(call.url).protocol === "https:")); + assert.ok( + app.calls.some((call) => call.url === "https://secure.example:9443/health"), + ); +}); + +test("legacy IPv6 produces one pair of brackets and reaches login/navigation", async () => { + const app = loadConsole([ + { + id: "ipv6", + managementUrl: "http://[::1]:8787", + runtimeUrl: "https://[::1]:9443", + password: "FAKE-PASSWORD", + }, + ]); + const profile = app.state.profiles[0]; + assert.equal(profile.host, "::1"); + assert.equal(app.buildManagementUrl(profile), "http://[::1]:8787"); + assert.equal(app.buildRuntimeUrl(profile), "https://[::1]:9443"); + await app.openWebui(); + assert.equal( + app.calls[0].url, + "http://[::1]:8787/api/v1/management/auth/login", + ); + assert.equal(new URL(app.navigations[0]).origin, "http://[::1]:8787"); +}); + +test("the editor saves migrated protocol choices and reloads them without legacy URL fields", async () => { + const app = loadConsole([ + { + id: "migration", + name: "TLS connection", + managementUrl: "https://[2001:db8::1]:8443", + runtimeUrl: "http://[2001:db8::1]:8788", + }, + ]); + assert.match( + app.app.innerHTML, + /name="managementScheme">