diff --git a/.dockerignore b/.dockerignore index 93f0bf8..3228fa8 100644 --- a/.dockerignore +++ b/.dockerignore @@ -14,3 +14,8 @@ htmlcov *.db browser_profile uv.lock +.superpowers +build +dist +output +node_modules diff --git a/.env.example b/.env.example index c25efc0..7366776 100644 --- a/.env.example +++ b/.env.example @@ -63,6 +63,8 @@ SECOND_EYE_IMAGE=second-eye:local # SQLite 数据库文件路径(Docker 部署时会被 compose.yml 的 environment 覆盖为 # sqlite:////app/data/goodprice.db,容器内改这里不生效) DATABASE_URL=sqlite:///data/goodprice.db +# 日志与后续媒体缓存目录;测试/独立部署可指定其他目录。 +DATA_ROOT=data # Docker noVNC:默认关闭。临时登录时设置 ENABLE_NOVNC=1,并在 data/.novnc-password # 写入独立密码;登录完成后恢复为 0 并重启容器。 @@ -70,3 +72,33 @@ ENABLE_NOVNC=0 NOVNC_PASSWORD_FILE=/app/data/.novnc-password # noVNC 对外端口(compose 端口映射与登录页链接共用;需与 BIND_ADDRESS 一起对外可达) NOVNC_PORT=16080 +# 媒体预算:提高数量会增加模型用量、延迟、内存和磁盘;不自动拆分付费请求。 +VIDEO_ANALYSIS_ENABLED=false +MEDIA_IMAGE_COUNT=8 +MEDIA_VIDEO_COUNT=1 +MEDIA_FRAMES_PER_VIDEO=6 +MEDIA_IMAGE_QUOTA=4 +MEDIA_FRAME_QUOTA=4 +MEDIA_TOTAL_LIMIT=8 +# 未知模型上限保守8;只有确认提供商支持后才提高,最高16。 +MEDIA_MODEL_LIMIT=8 +MEDIA_VIDEO_MAX_MIB=100 +MEDIA_VIDEO_MAX_SECONDS=180 +MEDIA_CACHE_MIB=1024 +MEDIA_RETENTION_DAYS=7 +MEDIA_DOWNLOAD_TIMEOUT=60 +MEDIA_DECODE_TIMEOUT=30 +MEDIA_REQUEST_MAX_MIB=16 +MEDIA_FFMPEG_THREADS=1 +# 分阶段执行预算(设备建议不自动提高访问频率或模型RPM) +EXECUTION_ACTIVE_RUNS=2 +EXECUTION_CRAWLER_WORKERS=1 +EXECUTION_MODEL_WORKERS=1 +EXECUTION_MEDIA_WORKERS=1 +EXECUTION_PLATFORM_INTERVAL_SECONDS=10 +EXECUTION_MODEL_RPM=10 +EXECUTION_GLOBAL_ITEMS=200 +EXECUTION_TASK_ITEMS=60 +EXECUTION_RUN_DETAILS=30 +EXECUTION_RUN_HTTP_ATTEMPTS=60 +EXECUTION_RUN_MINUTES=15 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 08f38a5..95521e7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,6 +5,7 @@ on: push: branches: - main + - release permissions: contents: read @@ -14,8 +15,12 @@ concurrency: jobs: test: - name: Python tests - runs-on: ubuntu-latest + name: Python and browser tests (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest] steps: - name: Check out repository uses: actions/checkout@v4 @@ -39,6 +44,9 @@ jobs: - name: Check whitespace errors run: git diff --check + - name: Install test browser + run: uv run --locked playwright install --with-deps chromium + - name: Lint (ruff) run: uv run --locked ruff check . diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 632ef4b..40f4970 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -2,8 +2,6 @@ name: Release image on: push: - branches: - - release tags: - 'v*.*.*' permissions: @@ -19,8 +17,12 @@ env: jobs: test: - name: Release tests - runs-on: ubuntu-latest + name: Release tests (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest] steps: - name: Check out repository uses: actions/checkout@v4 @@ -41,8 +43,17 @@ jobs: - name: Install test dependencies run: uv sync --locked --extra dev + - name: Install test browser + run: uv run --locked playwright install --with-deps chromium + + - name: Check whitespace errors + run: git diff --check + + - name: Lint (ruff) + run: uv run --locked ruff check . + - name: Run tests - run: uv run --locked pytest -q + run: uv run --locked pytest -q --cov --cov-fail-under=80 docker: name: Build and publish GHCR image @@ -71,7 +82,6 @@ jobs: tags: | type=semver,pattern={{version}} type=semver,pattern={{major}}.{{minor}} - type=raw,value=release,enable=${{ github.ref == 'refs/heads/release' }} type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/v') }} - name: Build image and publish version tag diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..65f0b1f --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,55 @@ +# 更新记录 + +本文件记录用户可感知的变化与升级注意事项。正式版本见 [GitHub Releases](https://github.com/Comui520/second-eye/releases)。 + +## [Unreleased] + +## [0.2.0] - 2026-10-27 + +### 新增 + +- 同一商品链接下的型号/容量等规格、价格证据与人工报价;区分明确售价、区间价、询价和冲突,避免用占位价格推荐。 +- 图片与视频采样帧的混合分析、证据关联和受限媒体缓存;可配置输入数量、超时、磁盘预算与FFmpeg线程。 +- 持久化分阶段执行、容量与请求预算、版本/租约校验、恢复游标、独立通知投递。 +- 可配置抓取/媒体/模型并发及设备建议;识别容器有效资源,在内存不足时临时降低并发或缓派。 +- 通知历史中核验后重试与取消待发送通知的操作。 + +### 调整 + +- 移除整任务结束后固定300秒等待;保留平台导航间隔和模型RPM,慢模型期间可推进其他任务的抓取。 +- 按商品输入复用独立评分缓存,避免批次成员变化引起重复请求;浏览器在所属线程复用并隔离上下文。 +- 设置页按账号、模型、通知、媒体、资源分类独立保存,凭据留空保持,清除需确认。 +- 静态样式和HTMX随应用提供;正常运行不需要Node.js/npm或连接CDN。 +- main/release及正式Tag使用Windows/Linux测试、页面测试和覆盖率门槛;正式Tag通过检查后发布GHCR镜像。 + +### 修复 + +- 筛选商品后进入详情再返回时筛选丢失、加载更多重复请求,以及自动刷新打断选择和滚动。 +- 商品标题/规格/报价变化或证据过期后沿用旧评分的问题;聚合前再次核验评分对应的价格。 +- 任务暂停或修改后的迟到结果、超额候选丢失、卖家风险与拉黑行为在新流水线中的兼容问题。 +- 通知响应中断后可能重复发送,以及删除商品后SQLite复用编号导致漏发或误发旧通知的问题。 +- 启动时重写历史评分、旧图片字段漏分析、缓存恢复未核验内容和DNS阻塞超过下载时限的问题。 + +### 升级注意 + +- 升级前停止服务,备份实际数据目录中的数据库及配置;迁移会新增表和字段。回退请同时恢复旧镜像与升级前数据库备份。 +- 同一数据目录只运行一个应用进程。Docker使用镜像 `ghcr.io/comui520/second-eye:0.2.0`;版本Tag不包含镜像标签中的前导`v`。 +- 自定义ASGI启动改用 `uvicorn goodprice.main:create_app --factory`;常规 `python -m goodprice` 启动方式不变。 +- 视频默认关闭;新安装默认总视觉输入上限8张(混合时优先4原图+4帧),旧库保持4张直至明确修改。 +- 提高输入数量或并发会增加模型费用消耗速度、延迟、内存和磁盘压力;设备建议不会自动提高媒体配额或访问频率。 +- 外部渠道接受情况无法确定时显示“结果未知”,需核验后人工重试,可能重复。 + +### 已知限制 + +- 真实闲鱼多规格/视频控件仍需更多样本验证;不能识别时保留待核验。共享样品不代表具体规格实物品相。 +- 仅处理直接HTTP(S)媒体;不支持blob、HLS/DASH及受保护流。视频帧不代表完整视频、音频或动态故障验证。 +- 本地uv/Conda的视频分析需要自行安装FFmpeg;Docker已内置。macOS/ARM未完成实机验收。 +- 持久化与恢复增加SQLite开销,小批及时评分可能增加冷轮请求数,不承诺所有工作负载都更快。详见 [验证记录](docs/validation/2026-10-27-phase3-milestone.md)。 + +## 历史版本 + +- [v0.1.1](https://github.com/Comui520/second-eye/releases/tag/v0.1.1) +- [v0.1.0](https://github.com/Comui520/second-eye/releases/tag/v0.1.0) + +[Unreleased]: https://github.com/Comui520/second-eye/compare/v0.2.0...HEAD +[0.2.0]: https://github.com/Comui520/second-eye/compare/v0.1.1...v0.2.0 diff --git a/Dockerfile b/Dockerfile index e53a13e..f163523 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,12 +15,18 @@ ARG HTTPS_PROXY # not invalidate the large Chromium download. COPY requirements.lock ./ +# Use authenticated HTTPS transport and bounded retries for Debian downloads. +# Keep repository signatures enabled; transient HTTP CDN 502s are not a reason +# to bypass apt authentication. +RUN sed -i 's|http://deb.debian.org|https://deb.debian.org|g' /etc/apt/sources.list.d/debian.sources \ + && printf 'Acquire::Retries "3";\n' > /etc/apt/apt.conf.d/80-second-eye-retries + RUN env HTTP_PROXY="${HTTP_PROXY}" HTTPS_PROXY="${HTTPS_PROXY}" \ http_proxy="${HTTP_PROXY}" https_proxy="${HTTPS_PROXY}" \ sh -c 'pip install --no-cache-dir --retries 10 --timeout 120 -r requirements.lock && playwright install --with-deps chromium' RUN apt-get update \ - && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends x11vnc novnc websockify \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends x11vnc novnc websockify ffmpeg \ && rm -rf /var/lib/apt/lists/* COPY pyproject.toml README.md ./ diff --git a/README.md b/README.md index 78c178f..867f25d 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,9 @@ [![License: MIT](https://img.shields.io/badge/license-MIT-green)](LICENSE) [![Python](https://img.shields.io/badge/Python-3.11-blue)](https://www.python.org/) [![CI](https://github.com/Comui520/second-eye/actions/workflows/ci.yml/badge.svg)](https://github.com/Comui520/second-eye/actions/workflows/ci.yml) +[![Release](https://img.shields.io/github/v/release/Comui520/second-eye)](https://github.com/Comui520/second-eye/releases) + +版本更新见 [CHANGELOG](CHANGELOG.md),正式版本与镜像说明见 [GitHub Releases](https://github.com/Comui520/second-eye/releases)。 一个面向中文二手市场的本地开源工具:按关键词和价格区间监控闲鱼商品,使用 LLM 进行需求匹配、品相分析和批量性价比判断,并通过多种通知渠道推送结果。 @@ -31,8 +34,6 @@ Docker 是推荐的通用部署方式:应用、Python 依赖、Chromium 和 noVNC 都封装在镜像中,Windows、macOS、Linux 和 NAS 使用同一套 Compose 配置。 -本仓库不再把 Windows 双击启动器作为公共安装接口。仓库中曾经使用的 `.cmd`/PowerShell 启动器只适合作者自己的 Windows 工作流,已从 Git 追踪中排除;它们不会影响 Docker Compose 和跨平台手动启动方式。 - ## 快速开始:Docker Compose ### 1. 获取代码并创建配置 @@ -210,6 +211,72 @@ Conda 不是运行本项目的必需条件;新用户优先选择 Docker 或 uv ## 配置 +### 规格与价格证据(实验性) + +监控任务可在详情页填写目标型号/容量(例如 `iPhone 15 / 256GB`)。 +商品页面标价保留为原始观察,不代表目标规格的实际售价;仅明确规格价格才用于预算与性价比判断。 +价格私聊、区间价、信息冲突或详情提取失败的商品保留为待确认,不以占位价推荐。 +“待询价提醒”需在任务中主动开启,默认不增加外部通知。 + +商品详情可单独确认/撤销人工报价,默认有效期 7 天;人工报价优先,但证据变化需复核, +过期证据不继续作为推荐依据。人工报价与页面价的来源切换不当作降价。 +设置最低品相分后,未核验的图片或多规格共享样品不会绕过门槛。 + +当前支持可明确识别的商品范围 DOM 数据与 Schema.org Product/Offer,尚未取得真实多规格页面样本来验证所有闲鱼控件, +不能识别时显示待核验而不是猜测。v0.2.0已接入分阶段调度,并完成合成数据的Windows/Linux容器回归;真实平台组件兼容性仍有上述限制。 + +### 图片与视频证据 + +设置页的“媒体证据与用量”可配置保留数量、每次原图/帧配额、总输入上限、超时、缓存和解码线程。 +视频默认关闭;新安装默认一次最多8张,混合媒体先分配4张原图+4帧,单一媒体可借用余量。 +旧数据库默认保持4张上限;提高预算需明确确认费用、延迟、内存、存储和提供商限制,硬件建议不会提高图片数。 +模型上限未知时按8张处理,只有确认提供商支持后才可提高(最高16);不自动拆成多个收费请求。 + +只处理详情主媒体的直接HTTP(S)资源。blob、HLS/DASH、受保护流及未知页面结构会明确跳过, +不保证所有闲鱼页面的视频组件都能识别。视频仅采样起/中/末画面,不验证音频、整段动作或动态故障。 +没有明确规格关联的图片/帧是共享证据,不能据此确认某一SKU的实物品相;任务设置最低品相分时仍需有效评分。 + +Docker镜像内置FFmpeg/ffprobe。本地uv/Conda运行须自行安装FFmpeg并加入PATH;缺工具时视频不可分析,应用和图片功能仍可用。 +视频默认最大100MiB、180秒,下载60秒、解码30秒,缓存1GiB/7天;处理中的视频和临时输出同样计入磁盘预算。 +缓存被清理后保留证据摘要与时间位置,页面显示文件不可用;媒体文件通过内部编号访问,不公开签名下载地址或本地路径。 +相同内容的签名URL变化不会单独触发重复视觉收费,内容/需求/模型/选图设置变化才使分析缓存失效。 + +### 导航与设置整理 + +列表的任务、状态、价格状态、排序、已加载范围会带入详情及操作返回链接。加载更多不再反复请求同一页, +页面返回与原生后退可恢复列表位置;选择商品或操作表单时不自动替换正在使用的列表。 +设置分为账号、模型、通知、媒体与资源,单独保存互不覆盖,凭据留空保持原值,清除需明确确认。 +页面使用随包提供的 HTMX 和预编译样式,保留现有简洁界面,普通启动不需要连接 CDN 或安装 Node.js。 +维护者修改样式类后可按 `goodprice/web/static/THIRD_PARTY.md` 重建 CSS。 + +设备建议只在本机检测 CPU、内存及容器限额,不上传机器标识或自动提高配置;低占用/均衡仅设置工作并发, +不会增加视觉输入、开启视频或提高访问频率。设置页显示保存配置与内存保护后的实际并发;内存不足时暂缓派发,恢复后继续。 + +### 分阶段执行与恢复 + +应用使用单进程协调器,把搜索、详情、解析、媒体、模型、聚合拆成持久化步骤。默认允许两个任务分阶段推进, +抓取、媒体、模型工作并发各1;慢模型期间可继续抓取另一任务。取消原有整任务结束后的固定300秒等待, +保留同账号单页面导航和默认10秒平台间隔。普通启动不需要Redis、Celery或Node.js。 + +默认模型上限10次HTTP请求/分钟,每轮最多30个详情、60次模型尝试和15分钟墙钟时间;重试也占配额, +429的Retry-After会延后请求。超出预算或工作容量的候选保留游标,下轮继续;暂停、删除或修改任务后拒绝旧结果。 +降低并发时先排空正在执行的步骤。浏览器在所属线程内复用,任务间使用独立上下文;同一数据目录只运行一个应用进程。 + +每个规格按自己的确价与证据独立评分,再在当前有效、已完成的候选中比较。内容、提示词、需求或模型配置变化使缓存失效; +批次成员变化不重新请求已缓存的独立评分。每次视觉输入仍遵守媒体设置,不通过增加图片或自动拆分请求来加速。 +本轮分析期间仅价格过期、其他已采集内容未变时,可刷新价格并复用已采集媒体;这不表示媒体被重新下载或核验。 + +通知按渠道独立持久化,发送前重新检查规格价格和任务条件。已成功渠道不自动重发;进程中断或超时可能显示“结果未知”, +需自行核验后确认重试,可能重复。删除尚未发送的通知会取消其投递,正在发送的通知无法撤回。 + +持久化和恢复会增加SQLite读写开销,纯本地无网络工作不一定比串行实现快。离线对照可运行 +`uv run --locked python scripts/benchmark_stages.py --rounds 3 --output output/benchmark`, +报告相同30件合成商品的覆盖、冷/热模型调用、SQL、耗时和Python资源;压缩逻辑时钟排除真实导航等待,不能用作真实平台提速比例。 +本次升级的验证范围和已知开销见 [升级验证记录](docs/validation/2026-10-27-phase3-milestone.md)。 + +应用现使用无导入副作用的工厂;自定义 ASGI 部署请使用 `uvicorn goodprice.main:create_app --factory`, +原有 `python -m goodprice` 启动方式不变。升级旧库前先停止服务并备份数据库,回退使用旧镜像与备份库成对恢复。 + 配置文件: ~~~text @@ -257,11 +324,13 @@ Docker 和本地运行都会在项目的 data/ 目录中保存 SQLite 数据库 ~~~bash git switch release git pull --ff-only origin release -git tag -a v0.1.1 -m "Release v0.1.1" -git push origin v0.1.1 +git tag -a v0.2.1 -m "Release v0.2.1" +git push origin v0.2.1 ~~~ 普通 PR 和 main/release 推送会执行测试和 Docker 构建校验,但不会覆盖正式镜像。 +上面的版本号为下一次发版示例;发版前同步 `pyproject.toml`/锁文件并更新 CHANGELOG,通过检查后推送新Tag。 +GitHub Release的版本说明使用CHANGELOG对应条目,待该Tag的镜像发布成功后再公开版本。 ## 开发与测试 diff --git a/docs/superpowers/plans/2026-10-26-phase1-offer-pricing-plan.md b/docs/superpowers/plans/2026-10-26-phase1-offer-pricing-plan.md new file mode 100644 index 0000000..08931ab --- /dev/null +++ b/docs/superpowers/plans/2026-10-26-phase1-offer-pricing-plan.md @@ -0,0 +1,97 @@ +# 阶段一:规格与可信价格 Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** 使预算、排名和提醒只使用匹配规格的可靠价格,支持待询价与独立人工报价,安全迁移旧库。 + +**Architecture:** 纯函数裁决 + 不可变 DTO + 短事务服务;保持原调度入口可用,后续用相同阶段函数接新协调器。 + +**Tech Stack:** Python >=3.11、Pydantic、SQLAlchemy/SQLite、Playwright、pytest。 + +**Spec:** `D:/good-price/docs/superpowers/specs/2026-10-26-variant-price-media-design.md` §3–4、§7–9。 + +## Global Constraints + +继承 `2026-10-26-upgrade-roadmap.md` 全部约束。金额用分整数与 Decimal 转换;人工报价默认 7 天;未知价不排名;最低品相分 >0 且未核验时进入待核验。 + +## Review Focus + +- 测试 import 副作用触碰生产库(P1.1);“256GB”匹配“128GB”或同名组合合并(P1.2/P1.3)。 +- ORM 迁移隐式丢数据、外键孤儿(P1.2);人工价/页面价切换伪降价(P1.3/P1.4);部分表单清空报价(P1.5)。 + +### P1.1:测试隔离与无副作用应用工厂 + +**Files:** 修改 `goodprice/main.py`、`goodprice/config.py`、`goodprice/__main__.py`、`tests/conftest.py`、`tests/test_main.py`;新增 `tests/test_import_isolation.py`、`scripts/benchmark_pipeline.py`。 + +**Interfaces:** `build_app(settings: Settings | None = None, session_factory=None, with_scheduler: bool = True) -> FastAPI` 保持;新增 `create_app() -> FastAPI` 作为部署工厂;uvicorn 改 factory 调用。日志路径从显式 settings/data_root 获取,import 不 mkdir/init_db/backfill。 + +- [ ] 写 `test_import_does_not_initialize_db_or_write_logs`:子进程在临时 cwd 导入 main/入口,不出现项目 data 写入、连接或调度启动;对 Path/engine 禁止生产路径访问。写 `test_explicit_app_uses_only_temporary_store`,断言 app 数据库、日志、通知与 scheduler 隔离。 +- [ ] 运行 `uv run --locked pytest -q tests/test_import_isolation.py`,应因当前 import-time app 初始化而失败;保存失败证据,不运行真实库。 +- [ ] 实现 lazy factory,fixture 在收集前指定临时资源目录,所有模型与平台测试使用 fake/MockTransport;新增基准脚本只构建显式测试工厂,拒绝非临时数据库与真实外部 URL。 +- [ ] 运行上述用例和 `tests/test_main.py`,应通过;运行基准合成 30 件商品,保存原架构耗时/SQL/调用次数到测试生成目录,不提交机器私有报告。 +- [ ] 独立审查无副作用与入口兼容后,提交 `test: isolate app startup and establish pipeline baseline`。 + +### P1.2:规范化规格与幂等迁移 + +**Files:** 新增 `goodprice/offers/__init__.py`、`goodprice/offers/types.py`、`goodprice/offers/matching.py`、`goodprice/migrations.py`、`tests/test_offer_migrations.py`、`tests/test_offer_identity.py`;修改 `models.py`、`db.py`、`services/task_service.py`、`web/routes.py` 的 TaskCreate/任务表单。 + +**Interfaces:** +- `stable_offer_key(platform_sku_id: str | None, attributes: dict[str, str]) -> str | None`;歧义不强制造 key。 +- `match_variant(target: str, attributes: dict[str, str], title: str) -> Literal['matched','mismatched','uncertain']`。 +- PriceClaim:`source: str`、`kind: str`、`text: str`、`amount_cents: int | None`、`min_cents/max_cents: int | None`、`scope_key: str | None`、`observed_at: datetime`、`evidence_id: str`。 +- OfferCandidate:`key: str`、`attributes: dict[str,str]`、`availability: str`、`claims: tuple[PriceClaim,...]`;提供默认单件 offer 而非区间 SKU。 +- 新增 ListingOffer、OfferObservation、ManualQuote、OfferEvaluation、SchemaVersion 表;WatchTask 新字段 target_variant/notify_pending_price/version,Notification 增可空 offer_id/event_key;旧字段保留。 + +- [ ] 写迁移测试:v0.1.1 合成库迁移两次,原任务/通知/快照逐项不变;旧最低/最高价不变成真实 offer,标记 legacy_unverified;默认新字段不触发重发;删除父记录无新孤儿。 +- [ ] 写身份/匹配测试:不同显示顺序 key 不变、相同属性但不同原站 SKU 不合并、256GB 不匹配 128GB、目标含否定约束且无证据时 uncertain。 +- [ ] 运行 `pytest tests/test_offer_migrations.py tests/test_offer_identity.py -q`,确认新接口缺失导致失败。 +- [ ] 实现模型/DTO/带版本事务迁移,识别旧安装须发生在初始化改变表结构之前;备份恢复只在临时库演练。任务 API/表单统一白名单,版本随业务条件改变递增。 +- [ ] 全量验证旧模型/DB/TaskService/API 测试及新测试,范围默认取 Spec;审查后提交 `feat: add versioned offer and price evidence storage`。 + +### P1.3:价格、来源、报价和新鲜度裁决 + +**Files:** 新增 `goodprice/offers/pricing.py`、`goodprice/offers/service.py`、`tests/test_offer_pricing.py`、`tests/test_manual_quotes.py`。 + +**Interfaces:** +- `parse_money_cents(raw: str) -> int`,有限非负人民币金额,精度超过分拒绝,不浮点猜价。 +- QuoteInput:`offer_key: str, amount_cents: int, confirmed_at: datetime, expires_at: datetime, evidence_fingerprint: str, revoked: bool=False`。 +- PriceDecision:`state: str, amount_cents: int | None, source: str | None, evidence_ids: tuple[str,...], reason: str, fresh_until: datetime | None`。 +- `decide_price(offer: OfferCandidate, quote: QuoteInput | None, now: datetime, ttl_seconds: int) -> PriceDecision`。 +- `is_comparable_drop(previous: PriceDecision, current: PriceDecision, same_offer: bool) -> bool`。 +- `OfferService(session_factory)` 的 `record_observation(listing_id: int, offer: OfferCandidate) -> int`、`confirm_quote(offer_id: int, quote: QuoteInput) -> int`、`revoke_quote(quote_id: int) -> None`,校验与保存同事务。 + +- [ ] 写最小断言:`parse_money_cents('1500.00') == 150000`;参数化拒绝 NaN/Infinity/负数/不支持币种/小数三位。 + 最小红绿测试: + ```python + def test_money_uses_integer_cents(): + assert parse_money_cents("1500.00") == 150000 + ``` +- [ ] 写确价、range、询价、冲突、未知、作用范围不明、SKU 售罄/未知、人工价优先/变化暂停/七天过期、一个调度间隔证据过期测试;页面1500/报价1400/预算1450须使用有效报价,不用页面价。 +- [ ] 写 `test_source_switch_is_not_price_drop`、`test_refresh_failure_does_not_extend_freshness`、`test_revoke_keeps_original_observation`,运行新用例确认红。 +- [ ] 实现优先级和新鲜度纯函数,服务在 commit 前校验合并值;指纹排除纯采集时间变化、包含价格/规格/作用范围。保存证据与人工来源分离。 +- [ ] 运行新用例与原 satisfaction/crawl tests;审查后提交 `feat: resolve trustworthy prices and manual quotes`。 + +### P1.4:真实 SKU 采集和价格正确性接入 + +**Files:** 新增 `goodprice/crawler/offers.py`、`goodprice/services/offer_pipeline.py`、`tests/test_offer_parser.py`、`tests/test_offer_pipeline.py`;修改 crawler/base.py、parser.py、xianyu.py、services/crawl_service.py、satisfaction.py;新增脱敏/合成 fixtures。 + +**Interfaces:** `parse_offer_candidates(html: str) -> list[OfferCandidate]`;`OfferPipeline(session_factory)` 的 `evaluate_listing(listing_id: int, task_id: int, candidates: list[OfferCandidate], now: datetime) -> list[int]` 返回已保存评估 ID;网络/模型通过注入客户端执行,不把 session 包在请求周围。 + +- [ ] fixture 包含配件99/128GB1200/256GB1500;测试目标256GB预算1400时不入选、1600时入选、搜索标价99低于任务下限仍抓详情。文本私聊价保留待确认且没有性价比调用。 +- [ ] 写任务 fetch_detail=False、20组合截断、30详情预算、空/异常详情不误判售罄、旧商品刷新、重新上架状态、最低品相分未核验不放行等失败用例。 +- [ ] 运行 `pytest tests/test_offer_parser.py tests/test_offer_pipeline.py -q` 确认红。 +- [ ] 实现页面范围限定解析和有界只读选项交互,确切来源才确价,移走详情前价格过滤;普通单件生成默认 offer。未适配页面保留 unknown,不将平台状态猜成支持格式。 +- [ ] 将已有串行运行接到相同 offer 阶段函数,保存与通知以 offer 事件引用;旧 listing 标价仅兼容展示,去重不按采集时间每轮生成新事件;有限搜索未见改为疑似未见。 +- [ ] 无样本时写明实验性边界,运行 parser/crawler/crawl/notification 全部测试及新回归;审查后提交 `feat: evaluate listings using matching offer prices`。 + +### P1.5:规格/价格交互、通知与阶段验收 + +**Files:** 新增 `goodprice/web/offer_routes.py`、`tests/test_offer_api.py`;修改 routes.py 注册 router,商品卡片/详情/任务表单/通知文案及原 API 测试。 + +**Interfaces:** GET `/api/listings/{id}/offers`,POST `/offers/{id}/quotes`(金额、到期、确认原文),POST `/quotes/{id}/revoke`;响应包含 price_state/effective_price/source/fresh_until/evidence,不复用旧 price 的“确认价”含义。 + +- [ ] 写缺失/错误报价返回422且无写入;不存在 offer 返回404;报价不会被抓取覆盖;待询价开关关闭不外发;同规格相同事件成功渠道不重发、来源切换不叫降价。 +- [ ] 明确兼容测试:旧API仍有原页面标价 `price`,新 effective_price/source 不隐式取旧值;父摘要基于当前任务已匹配offer确定生成,多个offer时显示范围/数量而不伪称统一确价。 +- [ ] 运行新增 API/通知测试确认红;实现模板与路由,状态 unknown/pending 不生成本批最优,不把空价渲染成0;未保存秘密不回显。 +- [ ] 执行 `uv lock --check`、`uv run --locked ruff check .`、`uv run --locked pytest -q --cov --cov-fail-under=80`;临时库验证迁移/恢复、api 与页面渲染。 +- [ ] 阶段独立审查,按指纹/金额/旧库/提示误导检查;红绿回归修复后提交 `feat: expose offer evidence and inquiry workflow`,勾选总计划阶段一记录。 diff --git a/docs/superpowers/plans/2026-10-26-phase2-media-evidence-plan.md b/docs/superpowers/plans/2026-10-26-phase2-media-evidence-plan.md new file mode 100644 index 0000000..8086f5b --- /dev/null +++ b/docs/superpowers/plans/2026-10-26-phase2-media-evidence-plan.md @@ -0,0 +1,98 @@ +# 阶段二:可配置媒体证据 Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** 安全识别商品图片/视频、受限抽帧,以用户确认的输入预算形成可追溯分析,而非虚构已看完整视频。 + +**Architecture:** 媒体模块从详情提取结果生成规范证据;受控下载器和本地 FFmpeg 处理,配额管理器统一计算临时/永久文件;以 data URL 复用现有 chat/responses 客户端。 + +**Tech Stack:** Python >=3.11、httpx、Pillow、FFmpeg/ffprobe、SQLAlchemy、pytest;Docker 显式安装 FFmpeg,本地缺工具优雅降级。 + +**Spec:** `D:/good-price/docs/superpowers/specs/2026-10-26-variant-price-media-design.md` §5、§6.2、§7–9;前置为阶段一计划完成。 + +## Global Constraints + +继承总计划。视频默认关闭;默认保留8原图/1视频/每视频6帧,请求原图配额4/帧配额4/总上限8,旧安装总上限保留4;不自动多批收费。视频100 MiB/180秒、下载60秒/解码30秒、缓存1 GiB/7天,范围严格按 Spec §5.2/§6.2。 + +## Review Focus + +- 签名 URL 刷新不等于媒体变化,下载失败不缓存成成功(P2.1/P2.3)。 +- DNS 重绑定/重定向内网、图片解码炸弹、本地路径穿越(P2.2/P2.3)。 +- 两个200 MiB任务突破256 MiB缓存、清理器删活跃输入(P2.2)。 +- 模型虚构 evidence ID 或拿样品评分代表具体 SKU(P2.4)。 +- 应用硬件 profile 无意提升旧安装4张上限、视频缺工具导致启动失败(P2.1/P2.5)。 + +### P2.1:媒体契约、数量配置与提取 + +**Files:** 新增 `goodprice/media/{__init__,types,selection}.py`、`tests/test_media_selection.py`、`tests/test_media_parser.py`;修改 config.py、models.py、migrations.py、crawler/base.py/parser.py/xianyu.py、.env.example、设置一致性测试。 + +**Interfaces:** +- MediaCandidate:`key: str, kind: Literal['image','video'], url: str, offer_key: str | None, source_locator: str, poster_url: str | None`。 +- MediaEvidence:`id: str, kind: Literal['image','frame'], content_hash: str, local_path: Path, offer_key: str | None, video_id: str | None, timestamp_ms: int | None`。 +- MediaSelection:`items: tuple[MediaEvidence,...], omitted_ids: tuple[str,...], limitations: tuple[str,...]`。 +- `select_media(images: list[MediaEvidence], frames: list[MediaEvidence], image_quota: int, frame_quota: int, total_limit: int) -> MediaSelection`:混合交替公平分配,单类型可借额,设0的媒体明确未分析。 +- `extract_media_candidates(html: str) -> list[MediaCandidate]`:仅商品主媒体,未适配 blob/HLS/DRM 标记 unsupported,不自行下载。 +- 新 MediaAsset ORM 记录来源、类型、父视频、处理状态、hash、时间位置;新设置名称统一用 `media_*`(数量/上限/超时)及 `video_analysis_enabled`。 + +- [ ] 写 `test_default_mixed_selection_is_four_images_four_frames`、`test_single_type_borrows_unused_quota`、`test_zero_quota_reports_omitted_type`、`test_invalid_or_over_model_limit_rejected`。 + 对含至少4张有效原图和4帧的 fixture,断言: + ```python + selected = select_media(images, frames, image_quota=4, frame_quota=4, total_limit=8) + assert len(selected.items) == 8 + assert sum(item.kind == "frame" for item in selected.items) == 4 + ``` +- [ ] 写纯图/纯视频/混合/推荐流排除/非实际SKU媒体共享与顺序稳定测试;迁移旧配置总上限4且视频关闭,新安装8,既有显式值保留。 +- [ ] 运行新测试确认红;实现类型、表迁移、配置字段和配额选择器,Settings 跨字段验证引用 Spec 精确范围,未知模型上限保守8,显式确认最高16。 +- [ ] 运行媒体/配置/迁移测试,审查后提交 `feat: configure media evidence and input budgets`。 + +### P2.2:统一磁盘配额与安全下载 + +**Files:** 新增 `goodprice/media/budget.py`、`download.py`、`cache.py`、`tests/test_media_budget.py`、`tests/test_media_download.py`。 + +**Interfaces:** +- `MediaBudget(root: Path, max_bytes: int, retention_seconds: int)`,`reserve(key: str, bytes_needed: int)` 返回上下文管理器/预留token;`commit(token: str, paths: list[Path]) -> None`、`release(token: str) -> None`。 +- `MediaDownloader(budget: MediaBudget, transport=None, resolver=None)` 的 `download(candidate: MediaCandidate, max_bytes: int, timeout_seconds: int) -> Path`;缓存命中由内容和处理参数决定,不只由 URL 决定。 +- 请求使用经过验证/绑定的目的地址与原始 hostname TLS 校验;若底层不能防止解析后重绑定则域不安全,不能只 resolve 一次后普通 client.get。 + +- [ ] 写响应无 Content-Length/伪长度/重定向metadata/IPv6内网/解析后重绑定/凭据跨域/慢流/断流测试;失败半成品不存在,异常不含URL令牌和Cookie。 +- [ ] 写256 MiB配额下两个200 MiB任务只有一个可派发,活动输入不被LRU删除,单任务预算不能容纳时明确报错,重启回收预留且不突破缓存根路径。 +- [ ] 执行 `pytest tests/test_media_budget.py tests/test_media_download.py -q` 确认红。 +- [ ] 实现受限流式下载与重定向逐跳校验,临时文件位于同一受控根目录;配额包含全部中间文件,预先保守估算输出,磁盘剩余不足则延后/拒绝,不偷偷删证据继续算成功。 +- [ ] 运行新用例和媒体选择回归,审查网络与路径边界后提交 `feat: bound media downloads and cache disk usage`。 + +### P2.3:受限解码、抽帧与可复现缓存 + +**Files:** 新增 `goodprice/media/frames.py`、`processing.py`、`tests/test_video_frames.py`、`tests/test_media_processing.py`;修改 Dockerfile、pyproject.toml、uv.lock、requirements.lock、.dockerignore。 + +**Interfaces:** `probe_video(path: Path, timeout_seconds: int) -> VideoInfo`(duration_seconds/dimensions);`extract_frames(path: Path, count: int, output_dir: Path, timeout_seconds: int, threads: int) -> list[FrameFile]`(path/timestamp_ms/hash);`prepare_image(path: Path, max_edge: int = 1280) -> Path`;`MediaProcessor.process(candidates: list[MediaCandidate], settings: Settings) -> list[MediaEvidence]`。 + +VideoInfo 与 FrameFile 定义在 media/types.py,字段分别为 `duration_seconds: float, width: int, height: int` 和 `path: Path, timestamp_ms: int, content_hash: str`。配额 `reserve` 返回 `BudgetReservation(token: str, bytes_reserved: int)` 上下文对象;未commit的退出总会release。 + +- [ ] 合成小视频在临时目录生成并记录清晰帧时间位置;假子进程测试缺FFmpeg、进程超时、超180秒、输出超预留配额和非法协议输入;图片测试超解码像素、伪MIME和坏文件。 +- [ ] 测试 content hash 相同只解码一次、签名URL变化内容相同不付费重算、重复/模糊帧不会补造达到数量;缓存被清理显示证据不可用,不返回坏路径。 +- [ ] 运行新测试确认红;实现参数数组启动FFmpeg,禁止shell/网络输入,默认线程1、受控超时、有限输出,Pillow处理前先检查像素上限。 +- [ ] Docker 安装FFmpeg;Pillow加入正式依赖,uv lock后以 `uv export --locked --no-dev --extra jev --no-emit-project --format requirements.txt --output-file requirements.lock` 同步镜像依赖,检查无无关升级。 +- [ ] Windows真实工具可用时跑抽帧;否则标明缺失并在Linux隔离镜像跑真实工具测试。本地缺工具页面可打开,视频状态标为不可用。审查后提交 `feat: extract bounded video frame evidence`。 + +### P2.4:联合视觉分析与证据校验 + +**Files:** 新增 `goodprice/analysis/media_analysis.py`、`tests/test_media_analysis.py`;修改 analysis/llm.py/prompts.py、services/offer_pipeline.py、services/crawl_service.py、评估序列化。 + +**Interfaces:** `analyze_media(offer: OfferCandidate, selection: MediaSelection, client: LLMClient, requirement: str, request_limit_bytes: int) -> dict`;`validate_evidence_references(verdict: dict, allowed_ids: set[str]) -> dict`;结果含 observations、seller_claims、conflicts、unverified、condition_score 可空、evidence引用。 + +- [ ] MockTransport 断言 chat与responses输入使用可解码 data URL、精确选中数量及证据 ID,不含宿主路径;16 MiB上限超出则不发送请求,不静默缩减输入。 +- [ ] 测试模型引用不存在ID被拒绝、共享样品不能产生SKU确定品相、视频失败仅图分析时标明覆盖限制、无输入不产生伪评分、最低分>0未知时待核验。 +- [ ] 测试缓存 key 包含内容/任务条件/选图参数/模型/提示词版本;只刷新采集时间不重复模型调用,调整设置不自动重跑历史。 +- [ ] 运行新用例确认红;实现证据提示词、输入构造、schema校验和精确缓存失效,与价格结果分开,不允许视觉模型反写实际售价。 +- [ ] 跑全部LLM/Jev/offer/media测试及审查后提交 `feat: analyze mixed media with traceable evidence`。 + +### P2.5:媒体设置、证据展示与阶段验收 + +**Files:** 新增 `goodprice/web/media_routes.py`、`tests/test_media_api.py`;修改设置/商品详情模板、router注册、README、.env.example。 + +**Interfaces:** GET `/media/{asset_id}` 仅内部ID映射受控路径;GET `/api/media/capabilities` 返回FFmpeg可用性和有效预算,不返回宿主路径;媒体配置待阶段三分类保存统一管理,字段校验复用Settings。 + +- [ ] 写路径穿越/未知ID/文件清理返回404,页面显示抽帧时间位置/未分析/工具缺失;数量增大显示费用/延迟/内存/存储/模型限制,确认前不扩大旧4张预算。 +- [ ] 运行新用例确认红;实现最小简洁表单和证据展示,生成HTML不泄露签名URL令牌,本地缺FFmpeg提供安装指引而不静默安装系统工具。 +- [ ] 全套Ruff/锁文件/pytest覆盖率>=80%,隔离Docker构建和FFmpeg/SDK检测;媒体下载全为fake或受控测试资源,不挂生产data。 +- [ ] 阶段独立审查安全与资源边界,修复回归并提交 `feat: expose media evidence and explain analysis costs`;记录真实平台未验证项,勾选总计划阶段二。 diff --git a/docs/superpowers/plans/2026-10-26-phase3-pipeline-interaction-plan.md b/docs/superpowers/plans/2026-10-26-phase3-pipeline-interaction-plan.md new file mode 100644 index 0000000..2c7e379 --- /dev/null +++ b/docs/superpowers/plans/2026-10-26-phase3-pipeline-interaction-plan.md @@ -0,0 +1,123 @@ +# 阶段三:有界流水线、资源推荐与交互 Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [x]`) syntax for tracking. + +**Goal:** 让抓取、媒体、模型不再互相串行阻塞,在可配置的资源边界内可靠恢复,并修复全站导航和设置交互。 + +**Architecture:** 持久化状态与短事务单写入协调器,有限线程池处理I/O,Playwright由所属线程独占,FFmpeg为受限子进程;模板保留简洁风格。 + +**Tech Stack:** Python >=3.11、APScheduler、线程池、SQLAlchemy/SQLite、Playwright、HTMX、psutil、pytest。 + +**Spec:** `D:/good-price/docs/superpowers/specs/2026-10-26-variant-price-media-design.md` §6–9;前置为阶段一、二。 + +## Global Constraints + +继承总计划;任务同一时刻一个run,最多合并一次后续到期信号。默认active runs=2,抓取/模型/媒体worker各1,FFmpeg线程1;可调上限为4/2/8/2/4。平台间隔默认10秒(5–300),每账号并行导航1;API RPM默认10(1–60)。配置只影响新派发工作,降低并发先排空,设备建议不自动提高配额。 + +## Review Focus + +- 修改/删除/暂停后的迟到结果和重启通知重复(P3.1–P3.3)。 +- 达到工作项上限时游标丢失、单任务饿死其他任务(P3.1/P3.2)。 +- 容器无quota/嵌套限额/低内存误报强硬件(P3.4)。 +- 查询丢失、加载更多重复、未提交设置分类被清空(P3.5/P3.6)。 +- 性能数据改善来自减少分析覆盖而非架构提速(P3.7对比同数据/同输入预算)。 + +### P3.1:持久化工作状态、租约与版本 + +**Files:** 新增 `goodprice/execution/{__init__,types,store}.py`、`tests/test_execution_store.py`;修改models.py/migrations.py、services/task_service.py。 + +**Interfaces:** Stage 枚举 search/detail/parse/media/analyze/aggregate/notify;RunToken:run_id/task_id/task_version/lease_id;WorkResult:token/stage/input_fingerprint/output 的不可变DTO;新增 TaskRun/WorkItem/NotificationEvent/NotificationDelivery ORM。 +`ExecutionStore(session_factory)` 提供 `submit(task_id: int, kind: str, target_id: int | None = None) -> int | None`、`claim(stage: Stage, now: datetime) -> WorkItemDTO | None`、`complete(result: WorkResult, now: datetime) -> bool`、`recover(now: datetime) -> int`、`invalidate_task(task_id: int) -> None`;只传ID/DTO,不向worker泄漏session。 + +所有DTO在execution/types.py:RunToken各ID为int、lease_id为str;WorkItemDTO含 `id: int, token: RunToken, stage: Stage, input_fingerprint: str, payload: dict, attempt: int, deadline_at: datetime`;WorkResult.output为dict。Run/Work状态与通知状态分开,禁止拿NotificationDelivery的sent冒充Run完成。 + +- [x] 写同任务去重、运行中多次触发只补一次、全局200/每任务60背压、恢复游标、暂停删除和版本变更拒绝迟到complete、租约只被一个worker领取等失败测试。 +- [x] 写跨轮观察过期先刷新,实质内容不变复用已有模型结果;观察变化使相关下游失效,不把未知当否定。 +- [x] 运行 `pytest tests/test_execution_store.py -q` 确认红;实现带版本迁移、条件更新和短事务状态机,notification独立租约而非依赖已结束run。 +- [x] 回归迁移/报价/任务服务;独立审查后提交 `feat: persist bounded task execution state`。 + +### P3.2:资源通道、调度与取消恢复 + +**Files:** 新增 `goodprice/execution/limits.py`、`workers.py`、`coordinator.py`、`tests/test_execution_limits.py`、`tests/test_coordinator.py`;修改scheduler.py、main.py、crawler/xianyu.py、services/crawl_service.py、旧queue测试。 + +**Interfaces:** +- `ResourceLimits.from_settings(settings: Settings) -> ResourceLimits`(全部并发、RPM、间隔、工作项/run预算),含服务端范围验证。 +- `RateGate.reserve(key: str, now: float) -> float` 返回最早允许时间;`WorkerPools.dispatch(stage: Stage, token: RunToken, payload: dict) -> Future[WorkResult]`。 +- `Coordinator(store: ExecutionStore, limits: ResourceLimits, workers: WorkerPools, clock)` 提供 `start()/stop(timeout: float)/submit(task_id: int)/submit_reanalysis(listing_id: int)/pause(task_id: int)/update_limits(limits: ResourceLimits)`。 +- `BrowserWorker` 所在线程负责 playwright/browser/context 创建/使用/关闭;平台节点执行包含SKU只读交互的账号级节流;先前offer/media阶段函数保持相同输入输出。 + +- [x] 用受控Future/假时钟写慢模型期间第二任务抓取可推进、active=1时明确串行、同账号导航峰值1、不同资源峰值<=设置、429 Retry-After/重试计RPM、等待不占其他槽、轮转公平测试。 +- [x] 写15分钟墙钟/详情30/HTTP60预算延后、关机恢复只补未完成或过期步骤、取消/改任务后模型迟到不提交、降低并发先排空、高压力暂停派发不丢工作。 +- [x] 写scheduler修改任务interval后触发器更新、禁用队列任务不再执行、manual重新分析走同一资源池的失败用例,运行确认红。 +- [x] 实现单进程协调器与按阶段线程所有权;去掉整任务完成后固定300秒等待,保留平台速率/风险退避;APScheduler仅提交信号与not-before。 +- [x] 确保业务网络调用不在写事务中;分阶段替换原TaskQueue而不双消费,app生命周期可靠关闭浏览器/线程/子进程,debug/reload不启两套scheduler。 +- [x] 并发/任务/队列/爬虫全部回归并独立审查后提交 `feat: run crawling and analysis through bounded stages`。 + +### P3.3:独立通知投递与可信批次聚合 + +**Files:** 新增 `goodprice/execution/delivery.py`、`tests/test_delivery_lifecycle.py`;修改offer_pipeline、crawl_service的聚合/通知路径、通知历史API/模型。 + +**Interfaces:** `DeliveryService(store, notifiers)` 的 `deliver(event_id: int, now: datetime) -> str`、`retry(delivery_id: int, now: datetime) -> str`;返回 pending/sent/failed/uncertain/awaiting_revalidation/superseded/cancelled。通知发送前核验task版本与价格新鲜度。 + +- [x] 写run完成后合法pending可发送、run超时后迟到计算不能生成事件、暂停/删除/改条件撤销待发送、证据过期延后、不重复成功渠道、崩溃unknown结果uncertain不得自动重发等失败用例。 +- [x] 写人工重试也重新核验;确价恢复但实质证据未变复用原业务事件;同一批次只比较当前就绪/确价/门槛通过offer,未完成不算低分,“本批最优”含比较范围。 +- [x] 新测试红后实现outbox与逐渠道状态,业务事件key排除纯观察时间;外部发送与记录间窗口如实uncertain,不宣称exactly-once。 +- [x] 全通知/offer/执行回归、审查后提交 `feat: deliver verified offer events independently`。 + +### P3.4:跨平台设备检测、资源profile与运行保护 + +**Files:** 新增 `goodprice/resources.py`、`tests/test_resource_profiles.py`;修改config.py、.env.example、锁文件、设置一致性测试;新增 `goodprice/web/resource_routes.py`。 + +**Interfaces:** `detect_resources() -> ResourceSnapshot`(effective_cpu:float|None、memory_limit_bytes:int|None、available_memory_bytes:int|None、source:str、warnings:tuple[str,...]、detected_at);`recommend_profile(snapshot: ResourceSnapshot) -> dict`;`resource_profile(name: Literal['low','balanced']) -> dict` 仅返回并发字段;POST `/settings/resources/apply-profile` 验证后确认应用,GET `/api/resources/recommendation` 只读。 + +- [x] mock Windows/macOS/Linux/cgroup v1/v2 CPU quota/cpuset/memory.max与current、无限制/文件缺失/权限异常/祖先更低quota;容器优先有效较小值,不把宿主资源当容器资源。 +- [x] 检测样本要覆盖cgroup从 `/proc/self/cgroup` 和mountinfo定位到当前进程而不是只读根目录;可用余量考虑宿主available与容器limit-current的较小值,不因cpuset字符串稀疏漏算。 +- [x] 断言 CPU>=4、limit>=6GiB、available>=3GiB 才建议balanced,否则low;未知来源low;建议GET不改配置/不跑benchmark/不上传机器ID。 +- [x] 断言profile低占用(2,1,1,1,1)、均衡(2,1,2,1,1),不修改媒体数量、视频开关、平台间隔和RPM;无确认POST不应用。 +- [x] 测试红后加入psutil正式依赖并同步两锁文件;实现能力可降级检测与运行时仅降档保护,保存值/当前有效值/原因分开显示。 +- [x] 设备/配置/执行回归及审查后提交 `feat: recommend explicit cross-platform resource budgets`。 + +### P3.5:列表上下文与设置分类保存 + +**Files:** 新增 `goodprice/web/list_context.py`、`settings_sections.py`、`tests/test_list_context.py`、`tests/test_settings_sections.py`;修改routes.py、settings_service.py、列表/详情/设置模板,新增轻量静态交互文件。 + +**Interfaces:** `ListContext.from_query(query) -> ListContext`、`ListContext.url(path: str = '/listings') -> str`、`safe_return_to(raw: str | None, fallback: str) -> str`;`SettingsSection` 白名单仅account/models/notifications/resources/media/deployment_readonly;`save_section(section: str, form: Mapping[str,str]) -> Settings` 在单事务校验合并后保存。 + +- [x] 写任务/状态/价格状态/排序/加载范围roundtrip;非法/重复/超长参数规范化;外域、协议相对、编码换行/双编码return_to回标准列表,不用任意Referer。 + URL边界的最小断言: + ```python + assert safe_return_to("//example.com/listings", "/listings") == "/listings" + assert safe_return_to("/listings?task_id=7&show=active", "/listings") == "/listings?task_id=7&show=active" + ``` +- [x] 写仅保存media不改变notification复选框;本分类未勾才关闭;秘密空值保持、显式清除有确认;数据库/容器运行模式等部署字段拒绝Web修改;无效值不提交半套配置。 +- [x] 新用例红后实现规范URL链接贯通卡片、详情返回及所有操作重定向;分类用锚点/轻量tab保持当前分类,服务端白名单控制写入,不再遍历全部Settings做公共表单。 +- [x] 分类支持unsaved提示、失败保留非秘密输入,清楚显示新增预算的代价和生效范围;沿用现有CSS/HTMX,无前端设计技能。 +- [x] API/设置/所有页面渲染回归、审查后提交 `fix: retain listing context and isolate settings sections`。 + +### P3.6:全站功能交互回归 + +**Files:** 新增 `tests/e2e/test_navigation.py`、`test_settings.py`、`test_task_feedback.py`、`tests/e2e/conftest.py`;修改列表/卡片/详情/任务/通知/仪表盘/base模板与必要API。UI功能测试沿用Playwright,不用视觉重设计技能。 + +**Interfaces:** E2E fixture 只启动显式临时Settings、禁用真实scheduler/通知/模型、使用动态未占用回环端口,浏览器退出后清理;sessionStorage按规范查询key保存加载范围与锚点,秘密禁止保存。 + +- [x] 先写失败流程:筛选→加载更多→详情→页面返回/原生后退→刷新,断言task/status/sort/已加载卡片范围保留;随后删除/拉黑/重评返回相同上下文,被删锚点恢复到相近项。 +- [x] 测试轮询不清选中项/正在编辑表单/当前滚动;加载更多稳定排序同分用ID,卡片无重复;交互时暂停刷新并给“有新结果,点击更新”。 +- [x] 测试任务重复运行去重/暂停反馈、通知pending/uncertain状态、失败HTTP不会伪成功、全选只针对已加载项、设置分类返回与unsaved提示。 +- [x] 各主要页面测试窄屏无横向溢出、键盘可达、禁JS核心GET筛选/POST表单可用;实现最小模板/JS修复,保留风格。 +- [x] 浏览器回归通过后审查并提交 `fix: preserve navigation and improve simple page interactions`。 + +### P3.7:全项目审计、性能对比与隔离部署验收 + +**Files:** 扩展 `scripts/benchmark_pipeline.py`、`tests/test_pipeline_regressions.py`、`.github/workflows/ci.yml`;修改README、Dockerfile/compose必要参数化、相关发现的具体模块。新增 `docs/validation/variant-media-upgrade.md` 保存脱敏方法/结果,不提交私人机器信息。 + +**Interfaces:** benchmark `--dataset synthetic-30 --rounds 5 --mode cold|warm --output `,输出JSON包含浏览器启动数、SQL、模型HTTP、阶段中位/P95、资源峰值、处理/待处理/失败数量;必须比较同输入预算和相同业务判定结果。 + +- [x] 每项新发现bug先最小失败测试;重点旧库/启动副作用/后台重评绕过队列/interval更新/下架误判/长事务/N+1/资源泄漏/配置commit前校验,不能仅重构不证明症状。 +- [x] 运行冷/热两轮对照基准,保证缓存轮不重复相同媒体模型调用、浏览器不逐商品重启;报告无明显回归或如实解释开销,不制造任意提速百分比。 +- [x] CI按Windows/Linux配置单测与浏览器功能验证;requirements.lock用jev extra重新导出与uv.lock比较,镜像SDK/FFmpeg可用;不在CI中推镜像,release的测试门槛与main保持一致。 +- [x] 运行 `uv lock --check`、`uv run --locked ruff check .`、`uv run --locked pytest -q --cov --cov-fail-under=80` 和E2E;迁移临时旧库两遍、还原备份演练通过。 +- [x] Docker以唯一测试镜像/容器名、独立临时卷和空通知配置构建启动;检查healthz、表单、SDK/FFmpeg、视频开关、noVNC开/关,绝不使用生产compose up替换正在运行的服务。 +- [x] 整分支独立代码审查、修复并完整复验,审查真实样本范围、缓存/预算、生命周期、API/旧库兼容。报告未验证macOS/ARM/平台样本限制,不冒充全部完成。 +- [x] 提交 `test: validate upgrade performance and deployment compatibility`,填写总计划完成记录;用户选择集成前保持独立分支,不自动push/发版。 + +验收:见 `docs/validation/2026-10-27-phase3-milestone.md`。按已批准的阶段末一次独立审查执行;四项Important已RED→GREEN并复验,未自动集成/发布。 diff --git a/docs/superpowers/plans/2026-10-26-upgrade-roadmap.md b/docs/superpowers/plans/2026-10-26-upgrade-roadmap.md new file mode 100644 index 0000000..859adf8 --- /dev/null +++ b/docs/superpowers/plans/2026-10-26-upgrade-roadmap.md @@ -0,0 +1,88 @@ +# 规格价格、媒体与流水线升级实施总计划 + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** 将单标价图片盯价工具升级为按规格、可信价格和媒体证据判断、可配置资源且保持简洁交互的跨平台应用。 + +**Architecture:** 保留单进程 FastAPI/SQLite/Playwright 与服务端模板。分三个可独立测试的子计划实施,先完成纯领域裁决与迁移,再加入媒体,最后整合有界流水线与全站交互;不引入 Redis/Celery/SPA。 + +**Tech Stack:** Python >=3.11、FastAPI、SQLAlchemy/SQLite、Playwright、Jinja2/HTMX、pytest/Ruff;阶段二新增 Pillow、FFmpeg,阶段三新增 psutil。 + +**Spec:** `D:/good-price/docs/superpowers/specs/2026-10-26-variant-price-media-design.md`(已获用户批准)。 + +## Global Constraints + +- 不使用前端设计类技能;保留简洁的颜色、布局基础与服务端模板,只改善信息与交互行为。 +- Windows/macOS/Linux/NAS/Docker 均不以操作系统专属 shell/fork 为前提;Windows 本地和 Linux Docker 实测,其余未测明示。 +- 不读取/提交真实 Cookie、密钥、私人文档,不挂载生产 data/ 进行测试,不发送真实通知,不自动触发付费模型调用。 +- 不更改/删除本地忽略的 `.cmd`、PowerShell 启动器,不自动 push main/release、创建 Tag 或发布镜像。 +- 价格未知不视为零,不按起售价排名,不自动联系卖家;必要证据过期暂停正常推荐。 +- 待询价外部通知默认关闭,人工报价默认 7 天;视频默认关闭;旧安装总视觉输入保留 4,新安装为 8(混合 4+4)。 +- 低占用首次默认:active runs=2、抓取/模型/媒体 worker 各 1、FFmpeg 线程=1;硬件建议需确认才能应用。 +- 不默默放宽访问和资源预算;导航间隔默认 10 秒(5–300),模型 RPM 默认 10(1–60),并发和大小有服务端硬上限。 +- 默认每轮详情 30、模型 HTTP 尝试 60、墙钟 15 分钟;默认工作项全局 200/每任务 60;取自设计 §6.1。 +- 在新架构完成前保留阶段一/二已通过的交付;独立阶段提交可审查,但不把部分功能误标为整轮完成或正式稳定版。 +- Ruff、锁文件一致性、全套测试与 >=80% 覆盖率、迁移、隔离 Docker 和独立审查均是交付门槛,不靠覆盖率取代行为用例。 + +## Review Focus + +1. import app/pytest 收集意外初始化真实数据库:P1.1 用子进程与拒绝路径访问验证。 +2. 价格来源切换伪造降价、任务目标变更沿用报价:P1.3/P1.4 以 observation/quote 与任务版本测试。 +3. 临时媒体文件绕过磁盘预算、模型输出引用不存在证据:P2.2/P2.4 分别覆盖。 +4. 暂停/到期之后迟到 worker 仍写入或发通知:P3.1/P3.2/P3.3 使用假时钟和受控 Future 验证。 +5. 设置分类遗漏复选框清掉其他分类、return_to 开放重定向:P3.5 做 API 和真实浏览器用例。 + +## 计划拆分与执行次序 + +| 顺序 | 子计划 | 交付 | +| --- | --- | --- | +| 1 | `2026-10-26-phase1-offer-pricing-plan.md` | 测试隔离、规格/可信价/人工报价、旧库迁移、现有流水线接入 | +| 2 | `2026-10-26-phase2-media-evidence-plan.md` | 可配置媒体预算、安全下载/抽帧、证据分析与缓存 | +| 3 | `2026-10-26-phase3-pipeline-interaction-plan.md` | 受控流水线、设备建议、全站交互、综合性能和缺陷修复 | + +每个子计划开始前先阅读本总计划和 Spec;后续计划通过下列契约衔接,不平行改同一大文件。 + +## 共享接口与文件边界 + +- `goodprice/offers/types.py`:不可变 PriceClaim/OfferCandidate/PriceDecision/QuoteInput/OfferEvaluation,不能携带 ORM session。 +- `goodprice/offers/pricing.py`:金额、价格状态、有效来源与新鲜度裁决;`matching.py`:目标组合匹配与稳定身份。 +- `goodprice/offers/service.py`:短事务保存报价、观察与评估;业务规则调用纯函数,模板不重复判断。 +- `goodprice/media/types.py`:MediaCandidate/MediaEvidence/MediaSelection;`budget.py`:输入与磁盘预算,`download.py`:远端边界,`frames.py`:受限本地解码。 +- `goodprice/execution/types.py`:Stage/RunToken/WorkResult;`store.py`:持久化状态;`limits.py`:并发/限流;`coordinator.py`:唯一派发与写入协调。 +- `goodprice/web/list_context.py`:规范化列表状态;`settings_sections.py`:可写分类白名单;拆分新增路由文件,不把所有逻辑继续堆到 routes.py。 +- `goodprice/resources.py`:只读跨平台设备检测、建议与可见有效限制;不代替平台/模型限流。 +- 继续使用原 `models.py`、`db.py` 注册新表与迁移入口。领域和执行逻辑放独立包,避免循环导入。 +- `CrawlService` 阶段一/二是迁移适配层;阶段三接到协调器,最终不允许旧队列与新协调器双消费。 + +## 安全执行与基线 + +- [ ] 实施前读取 using-git-worktrees 技能,使用原生 managed worktree 从已批准设计提交建隔离 checkout;不复制 .env 或 data/。 +- [ ] 检查远程 main 的新变化与本地设计基线差异,仅 fetch 不自动合并社区新代码;必要冲突先解释。 +- [ ] 先执行 P1.1,之后才运行导入 `goodprice.main` 的测试或启动服务。 +- [ ] 基准使用合成页面、媒体、假模型和假时钟,保留请求次数、SQL、浏览器启动次数、中位数/P95及资源峰值;不压测真实平台。 +- [ ] 每任务遵循红→绿→审查→修正→提交;每阶段独立审查并跑全量回归,发现超范围架构问题先更新设计。 + +## 交付记录 + +- [ ] 阶段一完成,记录 commit、命令、结果和真实 SKU 样本验证范围。 +- [ ] 阶段二完成,记录 Linux/Windows FFmpeg 测试、视频适配限制与全部预算行为。 +- [x] 阶段三完成,记录队列峰值/重启回归、浏览器交互结果、性能对比和剩余问题。 +- [ ] 整分支独立审查与最终复验完成;仅此时总结完成范围,未验证真实平台能力保持实验性。 +- [ ] 用户选择集成/发版方式前,不 push main/release,不升级正在运行的生产容器或数据库。 + +## 当前状态 + +设计与计划已批准,采用主线程持续实施、阶段末独立审查。阶段一的规格/价格核心和阶段二媒体核心已实施并记录验证, +真实闲鱼规格与媒体控件仍待脱敏样本验证。阶段三已实施导航/设置整理、查询批量化、设备检测及执行/通知集成, +开发分支应用已由协调器替换旧队列,预算/限速、浏览器所有权、恢复与投递已接入。 +冷/热性能对照、隔离Docker/Linux验证及整分支独立审查和修复已完成,见 +[阶段三验证记录](../../validation/2026-10-27-phase3-milestone.md)。Windows463通过/1缺FFmpeg跳过,Linux464通过。 +保留真实平台组件与macOS/ARM未验限制;此状态不代表原本机生产服务已更新,不自动推送、发版或替换生产服务。 + +## 自审覆盖记录 + +- Spec §4 的规格/金额/报价/来源/过期/迁移由 P1.2–P1.5 实现,§5 的媒体数量/代价/安全/缓存/证据由 P2.1–P2.5 实现。 +- Spec §6.1 的恢复/背压/限流/取消与通知由 P3.1–P3.3,§6.2 的设备/容器限额由 P3.4,§6.3 的全站交互由 P3.5–P3.6 实现。 +- Spec §7–8 的旧库/隔离部署/性能/跨平台验收由 P1.1/P1.2/P3.7 覆盖;私人文件与不发布限制全阶段适用。 +- 接口从 offers/types → media/types → execution/types 顺序定义;任何阶段需要改变前置契约,须同步消费者测试,而非另造平行实现。 +- 默认预算及用户确认边界均引用批准设计;未取得真实多规格/视频样本时,明确保留实验性限制,不视作适配验收通过。 diff --git a/docs/superpowers/specs/2026-10-26-variant-price-media-design.md b/docs/superpowers/specs/2026-10-26-variant-price-media-design.md new file mode 100644 index 0000000..ebc5f5f --- /dev/null +++ b/docs/superpowers/specs/2026-10-26-variant-price-media-design.md @@ -0,0 +1,421 @@ +# 规格价格、多媒体证据、受控流水线与交互升级设计 + +日期:2026-10-26 + +基线:`f728ff8` / v0.1.1 +版本:修订版 2(追加资源配置、流水线与全站交互要求) + +状态:用户已批准修订设计;正在编写实施计划,尚未开始实现。 + +## 1. 目标与已确认范围 + +用户确认采用方案 A:分阶段实施,完成规格与价格可信度、多媒体分析,再进行全项目检查、性能优化及缺陷修复。 +重点是同一商品链接中的型号/容量组合,以及“标价不代表实际售价、需要询价”的商品。 +继续沿用 Python、FastAPI、SQLAlchemy、SQLite、Playwright、Jinja2/HTMX,保留 uv/Conda 与 Docker 运行方式。 +用户进一步确认:媒体输入数量由用户设置且展示代价;采用有界分阶段流水线提速,各硬件敏感并发可配置; +提供保守推荐和设备检测建议;全站交互优化但沿用现有简洁风格,不使用前端设计类技能;所有变化保持跨平台。 + +核心原则:先确认选项、价格及证据,再判断预算与性价比。未知不是零元,提取失败不是没有瑕疵。 + +非目标:自动询价、自动聊天、下单或支付;绕过登录/验证码/访问控制;穷举商家全部 SKU; +音轨理解和完整动态故障诊断;宣称兼容未经验证的所有闲鱼页面;重写前端或引入分布式服务。 + +## 2. 方案比较与选择 + +- A(用户选择):分阶段建立规格/价格基础,再接入媒体证据,最后测量并优化。每阶段可测试、审查、回退。 +- B(不采用):同时替换全部抓取、数据和分析逻辑,末尾统一测试。集成风险大,难以定位回归。 +- 技术实现采用结构化页面证据优先、确定性规则裁决,LLM 只辅助抽取文本声明与总结,不生成实际售价或虚构 SKU。 + 全靠 LLM 推断型号和价格不能提供可核对证据;一开始绑定某一家原生视频 API 则限制现有模型兼容性。 +- 调度方案比较:仅修补串行队列改动小但模型仍阻塞抓取;全面异步/分布式重写成本高、增加部署负担; + 用户已选择推荐的有界分阶段流水线(§6.1),保留有界队列用于公平性、去重和背压,不引入 Celery/Redis 作为运行前提。 +- 页面继续使用服务端模板及小量 HTMX/JavaScript;改善状态保持与操作反馈,不引入新视觉系统或 SPA 框架。 + +## 3. 当前实现与必要改变 + +- `ListingData/ListingDetail/Listing` 只有图片列表,没有视频媒体契约;品相请求最多发送四张图片。 +- `variants` 仅由价格区间生成“最低价/最高价”,不是真实可购买组合。 +- 搜索标价在详情提取前就被预算过滤;它还用于价格走势、性价比、重推,可能错误比较不同规格。 +- `_fetch_detail` 通常有描述就不刷新;现有商品重评未必更新规格与媒体。 +- 网络、模型调用处于较长数据库会话内;需测量锁占用、浏览器启动、重复模型请求与关系查询次数。 +- “重新上架”判断发生在 status 被设为 active 之后,是需以回归测试验证的疑点;有限搜索结果未见不等于已下架。 +- 当前应用无用户认证。升级不扩大默认网络暴露范围,不借此次改造冒充解决多用户安全问题。 +- 当前队列只有一个消费者,整轮执行完再固定等待 300 秒;抓取前抖动也占着任务执行通道。 + 因此慢模型、媒体处理和纯等待会阻塞其他任务,需要拆分资源通道,而不是简单删除所有队列或无限开线程。 +- `listings_detail.html` 的返回链接固定 `/listings`,卡片未传入来源参数;这解释页面返回按钮丢失筛选。 + 浏览器原生后退、加载更多、HTMX 刷新恢复仍需浏览器回归验证,不能将静态检查等同于已复现全部症状。 + +## 4. 阶段一:规格与价格可信度 + +### 4.1 数据与职责 + +沿用按任务隔离的 Listing 父记录;保留 `Listing.price` 与旧 PriceSnapshot 为“页面标价/旧标价走势”, +不再把它们直接当作目标规格的成交价。新增规范化的选项和观察记录,不继续向无类型 variants 字典塞业务状态。 + +- WatchTask:新增独立的目标规格文本 `target_variant`(如“iPhone 15 / 256GB / 非扩容”)和 `notify_pending_price`。 +- ListingOffer:`listing_id`、稳定 `offer_key`、原站 SKU ID(如存在)、规范化属性、选项名称、可用状态、 + 当前价格状态/证据、匹配状态、评估结果。唯一约束 `(listing_id, offer_key)`。 +- OfferObservation:关联 offer、原始来源、价格状态、确定价或区间、采集时间、页面位置/原文片段、内容指纹。 + 新价格以人民币分整数保存;原始金额用 Decimal 校验后转换,拒绝负数、非有限值和不支持的币种。 +- ManualQuote:关联 offer、金额、用户说明、确认/失效时间、撤销状态;独立于抓取值,绝不被爬虫覆盖。 +- offer 维度的评估与通知记录:记录使用的 observation/quote、媒体证据和任务条件指纹;通知增加 offer 与事件类型引用。 + +原站 SKU ID 优先作为身份;无 ID 时按完整规范化属性组合生成稳定键,不依赖显示顺序或价格。 +多个不同选项出现相同属性键、或无法确认组合完整性时标记歧义,不把它们合并。 +普通单件无规格商品使用默认 offer;旧 variants 区间仍展示为历史信息,不能迁移为真实 SKU。 +父 Listing 的摘要兼容旧 API,但新增明确的有效价格/规格/状态字段,前端不得再将旧 price 标成确认价。 + +### 4.2 价格状态与裁决 + +| 状态 | 含义 | 允许预算判断与性价比排名 | +| --- | --- | --- | +| exact | 当前具体可用选项有明确价格,且无适用的冲突或非售价声明 | 是 | +| range_or_from | 仅起售价或区间,目标选项未确定 | 否 | +| inquiry | 明示需咨询、标价非售价或占位价 | 否 | +| conflict | 同一选项同时存在互斥的价格声明 | 否 | +| unknown | 抓取失败、信息缺失或映射不明 | 否 | + +另有“人工确认报价”来源,不将它伪装为原站 exact。exact 仅表示页面证据明确,不保证卖家最终履约价。 +stock 状态使用 available/unavailable/unknown;SKU 不可用时不参与正常推荐,未知库存明确展示。 +“起”本身不能推断高价规格售价;“优惠私聊”与“所有标价非实际售价”区分作用范围; +声明作用范围不清时保守待确认,不能让一个疑似数字覆盖其他明确证据。 +来自图片或 LLM 的识别仅为待核对声明:必须关联证据,不独立提升为 exact。 + +### 4.3 采集与目标匹配 + +首先从商品所属 DOM/结构化页面状态读取真实 SKU 组合;只读必要的选项交互,不点击购买/咨询/支付。 +支持的结构化状态格式需要真实样本验证,不预先假定平台存在某个 API。 +最多处理每链接 20 个候选组合,只针对目标匹配选项做后续昂贵分析;限制截断时显示“规格获取不完整”。 +规则负责型号、容量等可确定约束;LLM 可以提出映射候选,但不能把不确定候选标为已匹配。 +没有填写 target_variant 时,普通单件照常处理;多规格商品显示已核对的选项,多个匹配项分别评估,不能默认最低价代表整件。 +明确型号/容量不匹配不推送;目标匹配不确定则留待确认,不进正常排名。 + +搜索阶段只保留非价格的安全过滤与抓取数量上限。预算过滤在 offer 解析后进行,避免起售价提前漏掉目标规格。 +默认每任务每轮最多 30 个详情候选;超过预算进入后续轮次的有界待抓取队列(持久化或可重建),记录未处理数量,不能静默当作不匹配。 +明确要求规格核验而 fetch_detail 关闭时,保留待确认状态并提示该设置不能获得确价,不偷偷启用详情抓取。 +exact 的单价落在任务预算内才进入正常候选;待确认价不以标价判断超预算,不进入性价比模型/本批最优。 +已知品牌或商家身份不作为直接排除条件;包装、配件、整机作为不同选项匹配,避免交叉借用证据。 + +### 4.4 人工报价与通知(下列默认值为本设计提议) + +- 待询价商品默认站内保留,独立的待确认提醒开关默认关闭,避免升级后突然增加外部通知;用户可按任务开启。 +- 手动报价必须绑定明确规格;确认页面同时展示最新抓取的价格证据,默认有效期 7 天,可修改或撤销。 +- 人工报价有效且规格匹配时可用于预算和排名,所有结果注明“人工报价”和确认时间。 + 新抓取的同 SKU 价格/非售价声明有实质变化时,旧报价转为需复核;选项不可用或身份歧义也会使报价不可用于推荐。 +- 当人工报价与页面 exact 同时有效,优先采用用户对照最新页面证据确认的人工报价(人工操作页明确提示该优先级); + 该报价必须记录确认时的页面证据指纹。若页面证据实质变化,先暂停人工报价而不是偷偷切回页面价继续推送。 + 用户复核或撤销后才重新确定有效来源。页面价与人工报价的历史分开比较;两种来源切换仅称“价格来源更新”, + 不标为降价,同来源内的新价格与旧价格可比时才计算降幅。人工报价到期后可采用仍新鲜的页面 exact,注明来源切换。 +- 不同规格分别做同价历史比较;从 A 切换到 B 不能叫“降价”。历史证据保留,不覆盖。 +- 预算、规格或关键需求修改使评估失效;重评不沿用旧目标报价,不复用旧任务条件的评分。 +- 通知以 offer、事件类型、证据/评估版本、渠道记录状态;待确认转明确可触发一次正常提醒。 + 相同待确认状态与证据不重复推送;渠道失败单独记录,不将其他成功渠道重新发送。 + 不声称网络超时情况下能严格 exactly-once;无幂等接口的渠道需提示可能重复,避免无限重试。 +- 未知价格只展示需求和媒体分析的独立结果,不产生可与正常候选混排的综合性价比名次。 + +### 4.5 刷新与页面行为 + +任务表单、API、设置、列表、详情和通知统一使用价格解析服务输出;不在模板另写价格裁决。 +详情提供规格明细、页面标价与有效价格的区分、原文依据、人工报价的新增/撤销操作。 +普通列表仅将确价候选纳入默认性价比排序;待确认独立筛选,预算/价格排序不拿占位价填空。 +SKU 与有效价格最多缓存一个调度间隔,手动重新分析和刷新强制更新;人工报价独立保持。 +下次抓取失败不抹除旧证据,但将其标为旧观察,不把该轮故障视为库存变化或降价。 +价格、规格映射与可用状态的核验有效期为一个任务调度间隔,自最后成功核验的时间计算; +短暂失败不提前缩短这个有效期,但该失败本身不产生新的价格事件或提醒。 +任一必要证据到期后,该 offer 转为待复核并暂停正常排名与正常外部提醒,不删历史记录、不重新认定售罄。 +人工报价的七天有效期只描述报价本身,不能代替新鲜的规格/可用状态证据;暂停推荐不会撤销或缩短报价期限, +后续成功核验无实质变化时可在原有效期内恢复。恢复不凭状态变化重复发送已成功的事件。 + +## 5. 阶段二:视频及图文混合证据 + +### 5.1 媒体契约 + +MediaAsset 记录 listing/可选 offer、image/video/frame 类型、原始来源/页面位置、封面、时间位置、内容 hash、 +采集时间、父视频引用、处理状态与失败原因。没有确切 SKU 关联时标记 listing_shared,不假设属于任意一台机器。 +只采集商品详情主媒体区域,排除推荐流、头像、广告;不能只按整个网页的 video/img 标签抓取。 +视频 URL 可能过期、blob 或为分段流:首版支持可授权获取的直接 HTTP(S) 媒体;blob/DRM/未适配流清楚标注不支持, +不绕过权限。必要时通过正常页面刷新重新获取临时 URL。 + +### 5.2 抽帧与资源边界(设计默认值,可配置) + +使用 FFmpeg/ffprobe 子进程处理本地受限文件,再把帧转为现有视觉接口支持的图片输入;不要求特定厂商的原生视频 API。 +Docker 显式安装 FFmpeg;uv/Conda 检测外部工具,缺失时禁用视频处理并提示安装,不让应用无法启动。 +视频分析为明确开关,默认关闭且显示“未分析视频”;启用后与图片联合分析。以下均为可见、可配置、服务端校验的资源预算, +不是靠前端隐藏或硬编码数量决定: + +| 设置 | 默认 | 第一版允许范围 | 代价与语义 | +| --- | --- | --- | --- | +| 每商品原图保留数 | 8 | 1–32 | 下载/存储增加,不等于每张均分析 | +| 每商品视频数 | 1 | 1–3 | 每个视频分别占下载/解码预算 | +| 每视频保留采样帧数 | 6 | 1–24 | 解码耗时、磁盘与证据量增加 | +| 每请求原图配额 | 4 | 0–16 | 更多图片通常增加模型用量和延迟 | +| 每请求视频帧配额 | 4 | 0–16 | 同上,抽帧不代表完整理解视频 | +| 每请求总视觉输入上限 | 8 | 1–16 | 必须不大于用户所选模型已知支持限制 | +| 视频文件上限 | 100 MiB | 10–200 MiB | 流式按字节截止,不只信 Content-Length | +| 视频时长上限 | 180 秒 | 15–300 秒 | 超限明确跳过,不偷偷分析前段后冒充全部 | +| 缓存总量 | 1 GiB | 256 MiB–10 GiB | 占用部署机器/容器挂载卷磁盘 | + +混合媒体默认先给原图和视频帧各 4 个名额,按清晰度、去重和覆盖性确定顺序;单一媒体可借用另一类型未使用的名额, +但不能突破总上限。有混合媒体时,某类配额设 0 即用户主动不分析该类,结果必须显示该覆盖限制。 +每类配置配额之和可大于总上限;混合场景按原图/帧交替公平分配至总上限,固定顺序使结果可复现, +默认自然得到 4+4;额外素材保留作证据。无任何可选输入时不发起伪视觉请求。 +仅调整数量不自动拆成多个付费请求、不自动提升并发;执行一次品相评估仅使用一个选定媒体批次。 +模型图像上限未知时保守限定在 8,用户可在高级设置显式确认其模型上限(最高 16); +已知不支持该值时拒绝配置,不伪称所有 OpenAI 兼容服务支持相同图片数。 +提高数量前提示费用/响应延迟/内存/下载及模型超限风险,展示配置前后输入上限变化;无提供商账单信息时不伪造金额。 +默认将模型图片缩放至最长边 1280 像素、限制图片解码像素和请求体大小(默认 16 MiB,最高 32 MiB), +超限时停止该批评估并提示降低设置,不能静默删掉证据后仍标为完全分析。 +这些配置纳入 Settings、设置表单、`.env.example` 与评估缓存指纹,重启后保留。 + +流式下载默认最多 60 秒、解码默认最多 30 秒(均可在高级设置 10–120 秒内调整),超预算中止并删除半成品; +媒体处理默认并发 1,FFmpeg 默认每进程 1 个解码线程;可调范围见 §6.2。 +有限采样覆盖起、中、末,过滤模糊/重复帧;不宣称未采样片段已检查,也不对音频或动态故障下确定结论。 +下载器验证 URL、DNS 解析地址和每次重定向;禁止回环、内网、链路本地、云 metadata 等目标。 +媒体下载与可配置内网模型地址是不同信任边界;不用媒体请求携带任意域 Cookie。子进程不经 shell, +FFmpeg 仅读本地输入、限制网络协议和执行时间;文件输出在受控缓存根目录内。 + +### 5.3 缓存与证据分析 + +缓存位于 data/media-cache,默认上限 1 GiB、保留 7 天(可设 1–30 天);原视频成功抽帧后删除,失败半成品立即清理。 +该总上限包含原图、保留帧、下载半成品、处理中视频及其他磁盘中间文件,不另设不受控临时目录。 +媒体任务开始前按有界下载大小和输出文件预算预留空间,所有读写共用配额记账; +先清理无占用的过期/LRU 文件,仍不足则延后任务并显示磁盘受限,不突破用户上限。 +活跃任务的输入/输出保留占用引用,不被清理器删除;结束/失败/进程恢复核对并释放预留。 +小缓存与高并发不能同时满足时降低实际派发并发;若单任务最坏所需空间都无法容纳,拒绝该配置组合或要求降低文件/采样预算, +不进入永远无法启动的等待。设置页解释配置并发与磁盘允许的有效并发,避免“256 MiB 缓存却同时下载两份 200 MiB”。 +同时检查磁盘剩余空间并预留系统安全余量;外部进程占用导致空间不足时失败清理,不能承诺控制整块磁盘的使用量。 +按内容 hash/处理参数去重,媒体 URL 改变不直接认定内容改变;无法取得内容时不可凭 URL 缓存成成功。 +已被分析引用的帧若因容量被清理,保留摘要、hash、时间位置并显示“证据文件已清理”;可重新获取,不显示坏链接。 +API 不接受任意本地路径;媒体通过受控 ID 映射提供,不泄露签名 URL 的令牌、Cookie 和宿主文件路径。 +模型请求使用可验证的 data URL 或受支持上传,不把容器本地文件路径传给远程模型;现有 chat/responses 两接口都要覆盖测试。 +分析输出包括证据 ID、来源、视频时间位置、观察到的瑕疵、卖家声明、信息冲突和未验证项。 +结构化校验只允许引用本次输入的证据 ID;拒绝模型虚构的帧、价格、容量或来源。 +共享媒体不能自动生成某一 SKU 的确定品相结论;分析结果保留样品/不保证对应实物提示。 +抓帧或模型失败时保留可用图片分析;无有效视觉证据时不生成“品相达标”结论,正常提醒明确标识未核验。 +最低品相分是明确门槛:任务设置大于 0 时,必须有适用于该 offer 的有效视觉评分达到门槛, +未分析、分析失败、只有不确定关联的共享样品媒体均进入待核验,不参与正常排名或正常外部提醒。 +最低分为 0 时,确价且规格匹配的商品可在明确标注“品相未核验”的情况下按价格等已知信息排名/提醒, +但不能生成品相通过结论。媒体与文字有实质冲突时保留冲突状态,不能用缺失分数的默认半分掩盖。 +关闭视觉而任务最低分大于 0 时提示配置冲突,不能静默放行或修改用户门槛;这是相对于旧放行逻辑的有意行为改变。 +评估缓存包含证据内容、任务条件、模型/提示词版本;变更时失效。支持手动强制重评,避免陈旧结论。 + +## 6. 阶段三:全项目检查、性能与缺陷修复 + +先建立可重复的基线,再按实测瓶颈修复;不承诺“没有任何 bug”或给出未测量的提速比例。 + +- 正确性:重新上架、有效价格变化、规格售罄、部分解析失败、重评、暂停/删除任务、通知失败恢复。 +- 生命周期:有限搜索结果未见只能称“本轮未见/疑似不可用”;只有详情确证下架才标为确定下架。 +- 数据库:外部请求不占写事务;按商品/步骤短事务,迁移失败不留下半套新结构;验证并发读写与 N+1。 +- 浏览器:按 §6.1 的所有权规则复用浏览器/隔离上下文,取消/异常可靠关闭;平台访问限速独立于分析并发。 +- 模型:同证据不重复分析;有限超时、退避、重试与单任务调用预算;预算耗尽保留待处理状态,不误报否定。 +- 可观测性:每轮记录候选数、详情耗时、媒体耗时、LLM 次数、缓存命中、SQL 次数和失败状态;日志屏蔽秘密。 +- 安全回归:表单先校验后提交、敏感配置与运行时配置边界、媒体下载边界、noVNC 默认关闭、路径穿越。 + +用固定的合成/脱敏商品集和 mock 网络/模型做可重复基准,比较冷启动与第二轮热缓存: +热缓存不重复启动每商品浏览器、不重复相同媒体的模型分析;记录中位数与 P95、查询数、调用数和资源峰值。 +任何优化需证明结果一致、无重复通知、无明显回归;绝不为性能提高抓取频率或取消安全等待。 + +### 6.1 有界分阶段流水线 + +APScheduler 仅提交到期信号,不直接执行完整任务;人工重新分析也走同一资源管理器。 +单进程协调器管理持久化 Run/WorkItem(含任务版本、阶段、输入指纹、重试时间、运行租约和状态), +阶段为搜索/详情抓取 → 纯解析与规则裁决 → 可选媒体处理 → 模型分析 → 聚合与写入 → 通知投递。 +配置、凭据不复制到工作项;工作项仅保存必要引用和脱敏错误。线程池适用于网络 I/O,视频用受限子进程, +不为简单 HTML 解析创建大量 CPU 进程,也不使用 OS 专属 fork 实现。 + +- 抓取通道默认单 worker:该线程创建并拥有 Playwright 及浏览器,其他线程不访问这些对象。 + 可配置最多两个抓取 worker,各自拥有完整独立生命周期;每次抓取/任务使用隔离 context,完成后关闭。 + 浏览器按有限寿命与错误情况回收,不跨线程共享页面或持有宿主登录窗口。每账号同时平台导航最多 1, + 即使抓取 worker 增为 2 也不增加同账号并行导航;额外 worker 不是鼓励多账号绕过限制。 +- 平台请求间隔、429/验证码退避在账号级统一生效;默认最小导航间隔 10 秒,高级可在 5–300 秒内设置。 + 不再在整个任务结束后无条件等待 300 秒;等待下一次允许访问时不占用模型或媒体槽位。 + 抖动改为调度器上的 not-before 时间,不用 sleep 占着整个任务执行槽。 + 更低间隔可能增加风控和账号限制风险,需明确提示;平台风险反馈可延长间隔,不能自动缩短用户值。 +- 模型通道独立线程池与全局并发上限;可设 RPM 限额(默认 10,范围 1–60),重试也计入。 + 每请求 timeout、最多 2 次重试,指数退避不占 worker 槽;并发不会绕过 RPM 或提供商返回的 Retry-After。 + 未知厂商限额按保守值处理;已知更低限额优先,不能自动以设备强大为由提高 API 访问量。 +- 解析输出和模型输出为不可变 DTO,线程不传递 ORM 对象/数据库 session。协调器单写入路径, + 写事务短且无外部网络调用;读请求用独立 session。SQL/SQLite 忙等待有限,重试时重新核验任务版本。 +- 所有通道有界:默认全局最多 200 个未完成 WorkItem,每任务最多 60 个;达到上限暂停上游生产。 + 搜索候选和下一轮待处理项去重持久化,超过上限以未完成运行游标/受限剩余 ID 集恢复, + 不把未处理项当作“不匹配/下架”,不让内存无限积压。任务间轮转公平,单任务不会占满所有槽位。 +- 每任务同时仅一个 active run;运行中再次触发只合并为一次后续到期信号,不并行跑同任务。 + 默认同时 active run 数低占用和均衡均为 2(最高 4,可手动降到 1);active 仅表示允许跨任务分阶段推进, + 实际资源占用仍由各通道限制。进程启动按租约恢复已完成与未完成步骤,不整轮盲目重做。 + 等模型/退避的 run 保持 active 身份,但不占抓取槽;默认两个 run 足以使慢模型期间另一任务能抓取。 + 用户主动将 active runs 设为 1 时明确说明会退化为跨任务串行,跨任务重叠验收仅在 active runs 至少 2 时适用。 +- 预算默认每 run 最多 30 个详情、60 个模型 HTTP 尝试(含重试)、15 分钟墙钟期限; + 高级允许分别 1–100、1–200、5–60 分钟。到期后未完成步骤延后至下一次运行,不硬杀共享线程。 + 网络与子进程有独立硬超时;到期/取消后回来的迟到结果不得提交。 +- 暂停任务:停止新调度,取消该任务待执行工作,活跃请求协作结束;恢复后按正常调度重建未完成工作。 + 修改预算/规格等任务条件时递增任务版本:旧工作返回后只可保存为旧证据,不能覆盖新评估或发送通知。 + 删除任务撤销其工作;每次发布结果与写入前用 Run ID + 任务版本 + 租约进行条件检查。 +- 聚合不等待整批所有任务:仅针对当前 run 已就绪、确价、门槛通过的 offer,按批次生成局部比较。 + 每 offer 的正常通知须等该 offer 的需求、价格和品相门槛都定稿;批次“最优”标记须注明比较范围, + 不把未完成候选当作低分或断言全市场最优。 +- 通知先写持久化待投递事件,再在受限 I/O 通道发送;通知投递与业务计算解耦, + 相同业务事件的成功渠道不重发。进程崩溃后的未知投递结果标记 uncertain,默认等待人工重试, + 不无限自动重发不支持幂等键的渠道;新系统不保证外部网络严格 exactly-once。 + +生命周期补充: +- 计算 run 的租约/截止时间约束计算 worker 的结果提交;通知事件有独立投递租约,已合法提交的事件在 run 完成后可继续投递。 + run 超时不撤销此前成功提交的评估事件,也不允许超时后产生新评估事件。 +- 通知每次发送前重新核对任务存在/启用、当前任务版本、评估有效性和必要价格证据新鲜度; + 任务暂停/删除/条件改变时撤销尚未发送事件;证据过期转为 awaiting_revalidation。 + 已开始的外部 HTTP 发送不能保证撤回,明确记录 sent/failed/uncertain,不把已发送历史改成未发生。 +- uncertain 的人工重试也先重新核验;证据已旧则不能直接发出旧推荐。重新核验未实质变化时恢复原业务事件, + 不创建重复通知;证据变化时原事件标为 superseded,并按新证据是否满足条件决定创建新事件。 +- “仅恢复未完成步骤”不代表永远复用已过期结果:跨轮/重启恢复前检查任务版本及各步骤依赖指纹和新鲜度。 + 先刷新过期的价格/规格/可用性;若只是观察时间更新、实质内容未变,复用仍符合内容指纹的媒体/模型结果, + 不重复付费分析;实质内容变化才失效受影响的下游步骤。 + 恢复工作挂入当前有效 run,用当前已核验的观察记录建立评估引用;旧 run 历史保持不可变。 + +### 6.2 用户资源设置与推荐配置 + +任务调度与硬件敏感设置全部可见、服务端有范围校验;资源 profile 只是填写配置的快捷方式,不是第二套隐形配置。 +平台访问间隔、模型 RPM 和每轮预算不是设备检测可自动放宽的设置。 + +| 项目 | 低占用/首次启动默认 | 均衡(用户确认后) | 自定义硬上限 | +| --- | --- | --- | --- | +| active runs | 2(可手动降为 1) | 2 | 4 | +| 抓取 workers | 1 | 1 | 2(每账号导航仍为 1) | +| 模型请求并发 | 1 | 2 | 8 | +| 媒体处理并发 | 1 | 1 | 2 | +| 每个 FFmpeg 线程数 | 1 | 1 | 4 | +| 平台导航最小间隔 | 10 秒 | 10 秒 | 见 §6.1,不随设备调整 | +| 每请求图像上限 | 8 | 8 | 16,且受模型限制 | + +第一版不提供“一键最高性能”,避免把强硬件误认为可忽略平台和模型限流。并发增加会提高浏览器内存、 +请求排队、视频 CPU、模型费用消耗速度与 SQLite 竞争;提高线程数不承诺线性提速。 +资源保护优先级:应用硬上限/容器资源边界 > 用户已确认的资源上限 > profile 默认。 +运行时压力过高只可临时降低有效并发并显示原因,不偷偷改保存值;不会自动提升至未确认值。 + +硬件检测只读、本地执行,不上传硬件标识、不跑压力测试: +- Windows/macOS/Linux 检测 CPU 数、总内存和可用内存;可采用跨平台 psutil,依赖与镜像锁同步。 +- Linux 容器结合 cgroup v1/v2 的 CPU quota、cpuset、memory limit/current 与进程可用资源取较小可用值; + 可获取亲和性时也纳入。无限制、缺文件、权限不足、嵌套容器等情况显示检测依据/不确定性,不能拿宿主全量资源做保证。 +- 首次默认低占用;当有效 CPU 至少 4、有效内存上限至少 6 GiB、当前余量至少 3 GiB 时才建议均衡。 + 不满足或检测不完整建议低占用。保存建议前展示检测时间、容器/本地来源、拟变更字段和代价,用户点击确认后生效。 +- 不检测 GPU 或自动启用硬件解码;GPU/显存不是远程模型推理能力的可用资源。 + +配置变更按版本应用:只影响新派发工作。减少并发先停止派发、让当前请求结束再缩容, +媒体数量/模型变化使后续评估缓存失效,不自动批量付费重跑全部历史商品。 +外部 API 的有效 RPM/超时设置也记录在可见配置中;无有效值时拒绝保存,不以 0 表示无限。 +模型输入数量与资源预算都由同一 schema 供 UI/API/env/worker 使用,避免设置表有值而执行器不读取。 + +### 6.3 全站交互整理(不更换视觉风格) + +沿用现有简洁配色、按钮、字体和服务端模板;不使用前端设计类技能,不增加炫技动画、厚重组件库或 SPA。 +此次重心是导航、状态保持、分类、操作反馈和移动端可用性,不改变产品品牌或信息架构的基本入口。 + +列表上下文以规范化 URL 查询为事实来源,覆盖 task_id、价格可信状态、show、sort、分页/已加载范围。 +点击详情携带经过白名单构造的相对 return_to;详情返回、拉黑、恢复、删除、重新分析后回到原上下文。 +仅允许本站指定列表路径及合法参数,防止开放重定向;无来源时回标准列表,不能直接使用任意 Referer。 +浏览器原生后退/前进、刷新、直接粘贴链接都应得到一致筛选;滚动锚点和加载范围按查询键存在 sessionStorage, +不能跨查询错误恢复。恢复范围有上限,分段加载,商品被删除时返回相近位置而非空白页。 +不承诺动态数据集永远回到同一像素,但必须保留用户筛选和排序;登录 Cookie、密钥不进 URL/浏览器状态。 + +加载更多与 HTMX 刷新共享参数构造器、稳定排序键(加 ID 做同分 tie-breaker)和去重规则。 +自动刷新不重置加载范围/滚动/筛选、不清掉用户已选项,不替换正在编辑的表单; +交互期间可暂停刷新并显示“有新结果,点击更新”,不能靠每两秒整页换 DOM 达到实时。 +全选明确表示“已加载可见结果”,不默默选择整个数据库;危险操作确认显示数量,失败时保留选择及筛选。 + +设置按账号与登录、文本/视觉模型、消息通知、抓取与性能、媒体、部署与诊断分组; +类别使用锚点/轻量 tab(移动端可用),不是视觉重做。支持直接链接到分类,当前分类刷新/返回后保持。 +每分类独立保存,未提交分类不改;复选框仅在该分类的字段白名单内解释未勾选, +密钥留空保持原值,清除须显式确认。未保存更改切换分类/离开时提示,失败保留非敏感输入, +秘密不回显到 URL、localStorage、日志或 HTML 明文。浏览器及服务端都校验,失败不写半套配置。 +部署/启动级字段(数据库路径、绑定地址、容器运行模式等)只读或明确说明需改 env/重启, +不能把所有 Settings 字段无差别当作可从 Web 修改的运行时选项。 + +页面范围与行为验收: +- 仪表盘:区分排队、抓取、分析、待询价、待核验、失败;数字与任务状态一致,不显示虚假进度百分比。 +- 任务列表/详情:新增目标规格和待询价开关;运行按钮防重复提交,暂停/取消语义与 §6.1 一致, + 编辑保存后留在当前任务/原列表位置,清楚反馈是否有旧版本工作被取消。 +- 商品列表/详情:保持筛选与位置,区分标价/确价/人工报价和证据来源;未知价格不展示为 0 或“超值”。 +- 消息通知:保留筛选/分页,区分 pending/sent/failed/uncertain,重试仅影响所选渠道,不能靠前端刷新伪造成功。 +- 设置:分类、局部保存、配置代价、设备建议确认和生效范围;缺视频工具或模型上限不明时有操作指引。 +- 全局:统一加载、空结果、错误/成功反馈;错误 HTTP 响应不能被当作成功后直接 reload;危险操作确认、键盘可达、 + 窄屏表单/导航无横向溢出。无 JS 时核心筛选和表单提交仍能用;恢复滚动等增强可降级。 + +### 6.4 分阶段顺序、基准与发布边界 + +执行前先记录原行为与失败案例、建立隔离测试配置和性能基线;随后: +1. 规格/价格域及兼容迁移(阶段一),先稳定正确性。 +2. 媒体证据与可配置输入预算(阶段二),默认不增加付费视频请求。 +3. 有界流水线与资源配置(阶段三前半),将前两阶段接入统一执行器;旧执行器仅在过渡测试使用, + 交付时不得同时消费相同工作。再完成交互整理、性能对比、全项目缺陷修复与最终验收。 + +使用隔离数据库、固定合成媒体和 fake 平台/模型时钟测试;不以真实平台压测来证明性能。 +关键场景为慢模型下下一任务抓取仍可在平台节流允许后推进、并发峰值永不超过配置、压力降档不丢工作、 +任务版本变化拦截迟到写入/通知、重启仅恢复未完成步骤、挂起任务不会饿死其他任务。 +浏览器交互验证使用 Playwright 等功能测试,不调用前端设计技能;Windows 本地+Linux Docker 为实测目标, +macOS 及其他架构未实际运行须标注验证边界,不能因“跨平台库”就声称所有设备已测过。 +设计与实现均不新增操作系统专属启动前提;Linux 容器只读资源检测是能力增强,本地检测失败必须可降级。 + +## 7. 迁移、兼容与回退 + +开发和 Docker 验证使用独立数据库、缓存、容器名和未占用的回环端口,禁止挂载生产 data/;默认禁用真实通知与定时任务。 +生产升级前停写并制作可恢复备份;迁移按版本、幂等、事务执行,先验证临时旧库副本,不能运行未备份的破坏性迁移。 +新增表与字段保留原数据;旧评分标记 legacy_unverified,不转换为新规格的确定结论;原有通知历史不触发重发。 +旧任务 target_variant 为空、待询价外部通知关闭;旧单件下次成功详情抓取后生成默认 offer,未知 SKU 不凭区间构造。 +旧安装升级时保留原有每请求最多 4 张的视觉输入预算,视频仍关闭;新安装默认总上限 8、混合配额 4+4。 +迁移记录明确区分已有安装与新初始化,不能根据有没有 API Key 猜测;已有显式预算始终保留。 +旧用户可在媒体设置中查看从 4 提升到 8 的代价并显式确认;启用视频或应用硬件 profile 不隐式提高媒体总上限。 +低占用/均衡快捷配置只修改并发资源字段,媒体数量和视频开关另行确认,保持升级不默默增加模型用量。 +任务删除、商品删除要覆盖新关联记录,防止孤儿数据;同一外部 ID 跨任务仍隔离。 +关闭新视频能力不会影响原有图片功能;已升级数据库不承诺直接降级可写,完整回退采用备份数据库与旧镜像成对恢复。 +保留本机已忽略的 Windows 快捷脚本、私人文档和 .env;不提交凭据、真实数据库、视频缓存和个人资料。 +当前仅提交设计到本地专用分支;实施前审查远程变化,实施完成不擅自更新 main、release、Tag 或 GHCR。 + +## 8. 验收与交付 + +每阶段:先失败测试 → 最小实现 → 单测/集成测试 → 独立代码审查 → 修复 → 再验证;再进入下一阶段。 + +必要验收用例: +1. 配件 99、128GB 1200、256GB 1500:目标 256GB 使用 1500,预算筛选与历史比较正确,绝不拿 99 评分。 +2. 占位价 1 元、明确非售价或需询价:待确认、无性价比/本批最优,开关与去重符合约定。 +3. 冲突价、选项售罄、SKU 身份变更、详情失败、组合超限不被静默视为已核验;人工报价不会被自动覆盖。 +4. 同一规格降价才产生降价事件;不同规格切换、人工报价过期、条件变化均不冒充降价。 +5. 图片、纯视频、混合媒体、样品视频、过期链接、超大小、超时、缺 FFmpeg、SSRF/路径穿越均有回归用例。 +6. 旧库迁移两遍结果一致,任务/历史/快照/通知不丢失;备份恢复演练通过。 +7. Ruff、锁文件检查、全套 pytest、覆盖率不低于现有 80% 门槛;关键新裁决分支需直接覆盖,不能只靠总覆盖率。 +8. Docker 构建、SDK 导入、FFmpeg 检测、独立数据库启动、healthz、表单/详情/提醒预览与 noVNC 模式检查通过。 +9. 提供性能基线与改后对比,未运行项和受外部环境限制的项如实列出,不以合成样本代替真实平台验证。 +10. 媒体预算在 UI/API/env 一致,默认混合 4+4,单一媒体借额、不超过总上限;调整时展示代价,非法/超模型上限拒绝。 +11. 调度去重、公平性、背压、队列上限、资源峰值、429 退避、退出重启、暂停取消、迟到结果、未知投递结果均有回归测试。 +12. 容器限额覆盖宿主资源;检测失败推荐低占用;设备推荐不自动应用,也不提高平台/API 限额。 +13. 商品筛选→加载更多→详情→返回、原生后退/前进、刷新、拉黑/删除/重评均保留上下文;设置分类保存不会修改其他分类。 +14. 所有页面的加载/失败反馈、窄屏、无 JS 核心功能可用;保持原有简洁风格,不用前端设计技能。 + +真实平台适配需要合法可访问的多 SKU、询价、纯视频/混合媒体代表样本;采集夹具必须脱敏。 +若缺真实样本可完成领域模型、规则与夹具测试,但不能将真实 SKU 自动提取/视频覆盖宣称完成; +相关能力保持实验性/待验证并明确列为交付限制,不自动启用或发布稳定版。 + +## 9. 待批准的提议与实施者可决定事项 + +待用户随设计一并批准:待询价站内保留但外部提醒默认关闭;人工报价默认 7 天;视频抽帧可选且默认关闭; +分阶段完成三个阶段(含新版流水线和全站交互),真实样本验证不足时明确标为限制而非伪造兼容结论。 +以上默认值均为助手提议,不表示用户此前逐项确认。 +同样待批准的裁决:有效人工报价优先但来源切换不算降价;新鲜度过期暂停推荐、保留报价; +最低品相分大于 0 时未核验不得进入正常推荐。实施计划必须显式覆盖这些变化与回归测试。 +新版提议默认:首次使用低占用 profile(两个轻量 active run、每资源通道并发仍为 1),设备推荐需确认; +新安装媒体默认总上限 8、混合 4+4 可调,旧安装升级保留总上限 4,显式确认后才提升; +平台节流替代旧任务结束后的固定等待,导航默认间隔 10 秒、模型 RPM 默认 10;资源和输入硬上限如 §5.2/§6.2。 +用户已确认方向与方案,但这些精确默认数值和完整生命周期规则仍随本设计一并批准。 + +实施者可决定:内部模块命名、索引具体形式、帧去重算法及可维护的采样细节、测试工具与基准夹具组织; +不得改变不自动联系卖家、不用未知价排名、不触碰真实数据、不默默扩大资源预算等边界。 + +## 10. 独立设计审查记录 + +已由独立工程师审查初稿,发现三个会影响实施的重要歧义:人工报价与页面价的优先级、刷新失败/过期后的推荐资格、 +最低品相分对未核验商品是否构成硬门槛。已在 §4.4、§4.5、§5.3 补充确定规则,作为设计提议交由用户批准。 +审查未涉及实现正确性;具体代码、真实平台样本与性能结果仍须在后续阶段另行验证。 + +修订版 2 再次独立审查发现五项实施歧义,现已补充: +1. 默认两个 active run、资源 worker 仍为 1;手动设为 1 时明确跨任务串行代价。 +2. 通知采用独立投递生命周期,发送前重新核验任务版本/证据;完成计算不等于通知失效。 +3. 跨轮恢复先校验过期依赖,内容不变复用模型结果,内容变化才重做相关分析。 +4. 旧安装保留原每请求 4 张预算,新安装默认 8,硬件建议不默默改变媒体用量。 +5. 磁盘总预算包含临时文件,预留空间与背压确保配置的缓存上限有意义。 +以上补充均为助手提出的确定规则;用户已在本次对话确认修订设计,未开始实现。 diff --git a/docs/validation/2026-10-26-phase1-milestone.md b/docs/validation/2026-10-26-phase1-milestone.md new file mode 100644 index 0000000..4166681 --- /dev/null +++ b/docs/validation/2026-10-26-phase1-milestone.md @@ -0,0 +1,44 @@ +# 升级阶段一核心功能验证记录 + +状态:阶段一核心实现已落地;整个升级未完成,未合并、推送或发版。 + +## 已落地 + +- 应用采用无 import 副作用工厂;原 `python -m goodprice` 保持可用。 +- 规格身份、整分金额、明确价/区间/询价/冲突/未知价裁决。 +- 观察记录与人工报价分开保存;报价默认 7 天、支持撤销,证据过期不继续推荐。 +- 搜索页起售价不提前过滤目标规格,品相分析和性价比使用所匹配规格的确价。 +- 最低品相分不被未知评分绕过;有限搜索未见不冒充确定下架。 +- 实验性 DOM/Schema.org 提取、商品证据展示、任务目标规格和待询价提醒。 +- 单渠道失败不会被其他成功渠道遮蔽;未知投递结果不自动反复重试。 +- 旧库幂等迁移、旧通知基线兼容;用户运行数据未用于开发测试。 + +## 证据 + +- `uv lock --check`:通过。 +- `uv run --locked ruff check .`:通过。 +- `uv run --locked pytest -q --cov --cov-fail-under=80`:295 passed,覆盖率 88.44%。 +- 一项已知的上游 Starlette/httpx 弃用警告,不是测试失败。 +- 独立审查发现六项 Important,逐项编写失败用例后修复;另覆盖媒体移除和视觉请求错误使用标价的回归。 +- 最终清洁 Docker 构建成功,测试镜像 `second-eye:upgrade-phase1`;SDK 导入为 typesafe-sdk 0.7.2。 +- Linux 容器使用临时 tmpfs 数据、动态回环端口、禁用测试任务;healthz、五个主要页面、规格 API、确认和撤销报价通过。 +- 测试容器已删除,未替换现有应用镜像标签/生产容器,未挂载真实 data。 + +## 设计裁决与修复范围 + +- 阶段一使用串行适配器复用 offer 阶段契约;移除旧的重复价格/通知私有实现,避免两条业务规则并存。 + 代价:外部代码若调用旧私有方法需迁移,公开 run_task 入口保留。 +- 删除任务保留商品和报价历史、清除任务关联,而不是删除用户历史。代价:会保留无法再调度的历史商品。 +- 旧“详情失败继续按标价推荐/有限搜索三轮未见就确定下架”的测试按已批准语义更新,而非保留不可信行为。 +- Docker 两次构建遇到官方 Debian HTTP CDN 502,改为官方 HTTPS 源并启用三次有限重试;签名验证未关闭。 + 若 HTTPS 也不可达则仍明确构建失败,不做不安全回退。 + +## 未完成/未验证 + +- 尚无真实多 SKU/视频代表样本;不宣称已适配全部闲鱼 SKU 控件。 +- 有界持久化的跨轮候选续抓、阶段调度、资源保护与设备推荐仍需阶段三接入;当前串行适配器只报告超过30候选的 deferred 数量。 +- 视频采集、下载/解码配额、抽帧、媒体数量设置尚未实施。 +- 全站筛选返回状态、设置分类、局部保存、交互与性能对比尚未实施。 +- 列表证据投影目前有逐商品读取,阶段三需批量查询并实测 N+1;本记录不宣称已提升总体性能。 +- macOS、ARM 与真实平台端到端未测试;未修改真实数据库或正在使用的环境来模拟验收。 +- 阶段一尚不能视为整个17任务计划完成;后续执行须从进度记录继续,不能重复前三个已提交任务。 diff --git a/docs/validation/2026-10-26-phase2-milestone.md b/docs/validation/2026-10-26-phase2-milestone.md new file mode 100644 index 0000000..4297cf7 --- /dev/null +++ b/docs/validation/2026-10-26-phase2-milestone.md @@ -0,0 +1,27 @@ +# Media evidence stage validation + +This is an unreleased development branch. The production checkout, database and personal launchers were not changed. + +## Implemented + +- Configurable retained images/videos/frames, fair per-request input quotas, provider limit confirmation and old-install four-input default. +- Public-address media downloads with pinned socket destinations and original TLS hostname, per-hop redirect validation, byte/time limits and sanitized logs. +- Shared cache accounting for reservations and outputs, active-file pins, content-based decode reuse and bounded local-only FFmpeg processing. +- JPEG data URLs for chat and responses, allowed evidence IDs, shared-SKU/conflict safeguards, seller text in visual inputs/cache identity and manual forced reanalysis. +- Media settings with explicit cost confirmation, controlled image viewing independent of vision configuration, frame positions and scoreless observations/conflicts. +- Wheel package data now includes templates; installed applications can render pages outside the source checkout. + +## Verification + +- Before independent review: 337 passed, one host FFmpeg skip, coverage 88.94%. +- Review fixes and dormant stage-three foundations together: **356 passed, one skip, coverage 88.88%**. Ruff and lock check passed. Logs remain in ignored local execution scratch. +- One independent review found nine important defects, all addressed by regression tests. Duplicate content was regraded important because it wastes paid input slots, and deduplicated. +- A deliberately invalid Range response reproduces the pin leak when the response object is retained; cleanup now executes in a response-level `finally`. +- `second-eye:upgrade-phase2` built successfully. A container with networking disabled generated a synthetic video and verified actual FFmpeg probe/start-middle-end frames, image resizing, optional SDK import, application pages and media settings validation. No production volume or external model/notification credentials were used. +- Windows host lacks FFmpeg; application startup remains supported and video capability reports unavailable. The corresponding real-tool test was run in Linux Docker instead. + +## Limits and remaining work + +Actual Goofish video widgets and SKU/media associations still need sanitized real samples. Only main-media DOM adapters and direct HTTP(S) files are implemented; blob/HLS/DASH/DRM are explicitly unsupported. Live provider compatibility and real DNS/TLS timing were not exercised against external services. No macOS or ARM runtime was tested. + +The media stage does not complete the overall upgrade. Persistent execution foundations are being developed; the application still uses the old task queue until the staged coordinator, delivery lifecycle and resource controls are integrated. Full navigation/settings interaction work and final performance comparison remain required. No speedup is claimed here. diff --git a/docs/validation/2026-10-27-phase3-milestone.md b/docs/validation/2026-10-27-phase3-milestone.md new file mode 100644 index 0000000..19ff30a --- /dev/null +++ b/docs/validation/2026-10-27-phase3-milestone.md @@ -0,0 +1,87 @@ +# 分阶段执行与交互验证记录 + +本记录保存 `codex/offer-media-upgrade` 合入前的验证结果;验证阶段没有替换原服务或发布版本。后续版本发布信息见 [CHANGELOG](../../CHANGELOG.md)。 + +## 已接入的行为 + +- FastAPI 生命周期创建单个协调器,APScheduler 只提交信号;正式入口不再消费旧整任务队列。 +- 搜索、详情、解析、媒体、模型与聚合分阶段推进;持久化游标、版本、租约、容量、请求预算和恢复状态。 +- 同账号单页面导航、平台最小间隔和模型 RPM 分开限制;取消固定300秒整任务间隔。慢模型期间第二任务可推进抓取。 +- 浏览器所属线程复用,隔离页面上下文;媒体和模型资源独立限制。内存压力临时降并发或缓派,保留保存配置。 +- 独立通知 outbox 在发送前重新核验证据;成功渠道不重复,无法确定是否接受的投递不自动重发。 +- 需求/视觉缓存绑定内容、提示词和模型配置;性价比分项独立缓存,不因动态批次成员变化重复收费;当前有效项在本地比较。 +- 列表筛选/排序/加载范围与返回位置保留,设置分类单独保存,清除凭据需确认;运行时样式和HTMX随包提供。 + +## 审查与回归 + +阶段末由独立审查者只读检查整分支及新增生产接线;无 Critical、四项 Important、无其他 Minor。 +四项均先用合成数据复现失败,再修复,未进行重复审查: + +1. 评分完成后、聚合前人工报价改变:聚合事务复查评分价格上下文,按新价格重评。 +2. 搜索标题变化但详情失败:追加失效状态,保留历史,停止旧推荐和未发送事件。 +3. 渠道接受后响应连接断开:结果归为 uncertain;仅明确拒绝或连接尚未建立等安全失败自动重试。 +4. 删除商品后 SQLite 复用编号:业务键和发送前核验包含平台、外部商品身份及规格,避免漏发新商品或误投旧事件。 + +对应 `tests/test_final_review.py`,同时覆盖旧消息尚未投递的编号复用情况。 +其他回归包括:超额候选恢复、暂停后迟到结果、429 Retry-After、HTTP预算、配置变化、报价变化、慢模型价格过期刷新、 +单独重评不通知、签名地址内容去重、旧图片字段兼容、卖家风险/缓存/拉黑、通知操作、资源检测、缓存损坏恢复和异常关机。 + +## 离线性能对照 + +可重现命令: + +```sh +uv run --locked python scripts/benchmark_stages.py --rounds 3 --output output/benchmark +``` + +数据见 [原始记录](2026-10-27-pipeline-benchmark.json)。两种实现均处理相同30件确价合成商品、相同需求与评分输入,视觉关闭; +没有浏览器、真实网络、付费模型、真实账号或外部通知。导航/RPM使用压缩逻辑时钟;阶段时间为逻辑时间。 +Python内存由 tracemalloc 记录,RSS仅采样当前Python进程,不含浏览器/FFmpeg子进程。 +本机同时运行构建/回归,并非独占性能环境;三次样本的P95只是经验上界,不能当稳定尾延迟估计。 + +| 实现 | 冷/热 | 完整有效评分 | 模拟模型请求 | SQL次数 | 耗时中位数 / P95 | +| --- | --- | --- | --- | --- | --- | +| 串行适配层 | 冷 | 30/30 | 31 | 1387 | 6.29s / 7.81s | +| 串行适配层 | 热 | 30/30 | 0 | 757 | 2.89s / 7.00s | +| 分阶段协调器 | 冷 | 30/30 | 38 | 9308–9388 | 31.62s / 34.41s | +| 分阶段协调器 | 热 | 30/30 | 0 | 8769–8846 | 31.89s / 33.54s | + +协调器的持久化、校验与恢复有明显纯本地开销;小批及时评分增加冷轮请求数,不能宣称所有工作负载都更快。 +已验证的收益是慢模型与后续抓取可以重叠、浏览器复用、取消固定300秒任务间隔、列表查询批量化、热轮重复模型请求归零。 +后续优化空间是减少活跃轮次的状态轮询SQL,不能通过降低输入覆盖或突破平台间隔来美化指标。 + +## 执行决定与限制 + +- 使用主线程持续实现、阶段末一次独立审查;导航和设置先于生产接线完成。代价是必须整体复验,不能以组件测试冒充接线完成。 +- 本轮仅价格过期、已采集内容身份不变时刷新价格并复用原媒体;不声称媒体重新采集。代价是同地址媒体在该轮内变化,可能到下一轮普通采集才检测到。 +- 内容缓存绑定提示词/需求/模型等语义输入;磁盘大小、FFmpeg线程等运行参数不单独触发付费重评。代价是遗漏语义输入会错误复用,因此专门覆盖提示词、内容和配置失效测试。 +- 保留旧 CrawlService/TaskQueue 给导入调用者与回归使用,正式入口只运行协调器。代价是旧导入路径仍需维护。 +- 拒绝把来源不明、共享样品或未核验的价格/品相当作确定结果;真实闲鱼规格控件和视频结构仍待脱敏样本验证,功能保持实验性。 +- 本次本机Windows和Linux容器验证不代替macOS/ARM验收;同一数据目录只允许一个应用进程。 +- 审查者未独立判定性能、部署和全量测试,由本记录的实际执行结果负责;批准的价格刷新与单进程边界继续保留,代价如上。 + +## 最终验证 + +修复后执行 `pytest -q --cov --cov-fail-under=80`: + +| 环境 | 结果 | 覆盖率 | +| --- | --- | --- | +| Windows / Python3.11 | 463 passed,1 skipped | 88.46% | +| Linux amd64 / Docker / 2CPU、2GiB | 464 passed | 88.90% | + +Windows唯一跳过项是本机缺FFmpeg的实际视频测试;Linux容器实际执行该测试。 +两套全量均包括真实Chromium的6个页面交互测试、离线基准小样本契约及最终审查回归。 +`uv lock --check`、Ruff、`git diff --check` 通过;requirements.lock与uv按jev extra重新导出的依赖/哈希一致。 + +临时旧库已完成SQLite备份、两次幂等迁移、原始值/无伪造确价检查,以及还原备份和完整性检查。 +Docker唯一开发标签 `second-eye:upgrade-integration` 已构建/加载;没有推镜像、映射公网端口或挂载生产数据。 +临时容器在 `--network none` 下通过: + +- healthz、任务/商品/设置/通知页面、资源API和非法设置校验;安装后的wheel包含模板与静态文件。 +- 实际FFmpeg/ffprobe本地合成视频抽帧、图片缩放;SDK0.7.2可导入。 +- 实际Chromium浏览器复用、两次独立上下文关闭。 +- 2CPU/1GiB容器约束识别;noVNC关闭时端口不监听,开启时根路径/vnc.html可用且使用密码认证;缺密码拒绝启动。 +- 镜像中包含最终价格上下文复核与商品身份核验修复;测试容器均已移除。 + +CI已配置Windows/Linux和Docker构建门槛,但本分支未push,因此未声称远程Actions已运行。 +仍未验证真实平台组件、macOS/ARM;本记录确认开发分支验收范围,不等于发布新稳定版本。 diff --git a/docs/validation/2026-10-27-pipeline-benchmark.json b/docs/validation/2026-10-27-pipeline-benchmark.json new file mode 100644 index 0000000..9da97df --- /dev/null +++ b/docs/validation/2026-10-27-pipeline-benchmark.json @@ -0,0 +1,279 @@ +{ + "dataset": "synthetic-30", + "external_network_requests": 0, + "limits": "Same 30 exact offers, requirement/value inputs and disabled vision. Admission intervals use compressed logical time. No actual browser/model/video latency; stage timings are logical time. RSS sampled for this Python process, not subprocesses.", + "serial": { + "cold": { + "results": [ + { + "seconds": 7.8073082999999315, + "sql_count": 1387, + "python_peak_bytes": 1525549, + "process_peak_rss_bytes": 75296768, + "detail_calls": 30, + "model_attempts": 31, + "verified_offers": 30, + "stage_logical_seconds": {} + }, + { + "seconds": 6.28916819999904, + "sql_count": 1387, + "python_peak_bytes": 991453, + "process_peak_rss_bytes": 79220736, + "detail_calls": 30, + "model_attempts": 31, + "verified_offers": 30, + "stage_logical_seconds": {} + }, + { + "seconds": 4.674092200000814, + "sql_count": 1387, + "python_peak_bytes": 1053696, + "process_peak_rss_bytes": 79728640, + "detail_calls": 30, + "model_attempts": 31, + "verified_offers": 30, + "stage_logical_seconds": {} + } + ], + "median_seconds": 6.28916819999904, + "p95_seconds": 7.8073082999999315 + }, + "warm": { + "results": [ + { + "seconds": 6.999146799998925, + "sql_count": 757, + "python_peak_bytes": 240448, + "process_peak_rss_bytes": 80195584, + "detail_calls": 30, + "model_attempts": 0, + "verified_offers": 30, + "stage_logical_seconds": {} + }, + { + "seconds": 2.8926522999990993, + "sql_count": 757, + "python_peak_bytes": 279019, + "process_peak_rss_bytes": 81117184, + "detail_calls": 30, + "model_attempts": 0, + "verified_offers": 30, + "stage_logical_seconds": {} + }, + { + "seconds": 2.601198700000168, + "sql_count": 757, + "python_peak_bytes": 230785, + "process_peak_rss_bytes": 81944576, + "detail_calls": 30, + "model_attempts": 0, + "verified_offers": 30, + "stage_logical_seconds": {} + } + ], + "median_seconds": 2.8926522999990993, + "p95_seconds": 6.999146799998925 + } + }, + "staged": { + "cold": { + "results": [ + { + "seconds": 30.939778800000568, + "sql_count": 9388, + "python_peak_bytes": 3720124, + "process_peak_rss_bytes": 92418048, + "detail_calls": 30, + "model_attempts": 38, + "verified_offers": 30, + "stage_logical_seconds": { + "search": { + "median": 5.0, + "p95": 5.0 + }, + "detail": { + "median": 5.0, + "p95": 5.0 + }, + "parse": { + "median": 5.0, + "p95": 5.0 + }, + "analyze": { + "median": 5.0, + "p95": 5.0 + }, + "aggregate": { + "median": 5.0, + "p95": 10.0 + } + } + }, + { + "seconds": 34.40631290000056, + "sql_count": 9335, + "python_peak_bytes": 3371834, + "process_peak_rss_bytes": 94257152, + "detail_calls": 30, + "model_attempts": 38, + "verified_offers": 30, + "stage_logical_seconds": { + "search": { + "median": 5.0, + "p95": 5.0 + }, + "detail": { + "median": 5.0, + "p95": 5.0 + }, + "parse": { + "median": 5.0, + "p95": 5.0 + }, + "analyze": { + "median": 5.0, + "p95": 5.0 + }, + "aggregate": { + "median": 5.0, + "p95": 5.0 + } + } + }, + { + "seconds": 31.622776399999566, + "sql_count": 9308, + "python_peak_bytes": 3306201, + "process_peak_rss_bytes": 95559680, + "detail_calls": 30, + "model_attempts": 38, + "verified_offers": 30, + "stage_logical_seconds": { + "search": { + "median": 5.0, + "p95": 5.0 + }, + "detail": { + "median": 5.0, + "p95": 5.0 + }, + "parse": { + "median": 5.0, + "p95": 5.0 + }, + "analyze": { + "median": 5.0, + "p95": 5.0 + }, + "aggregate": { + "median": 5.0, + "p95": 5.0 + } + } + } + ], + "median_seconds": 31.622776399999566, + "p95_seconds": 34.40631290000056 + }, + "warm": { + "results": [ + { + "seconds": 28.582909599999766, + "sql_count": 8769, + "python_peak_bytes": 952933, + "process_peak_rss_bytes": 92221440, + "detail_calls": 30, + "model_attempts": 0, + "verified_offers": 30, + "stage_logical_seconds": { + "search": { + "median": 5.0, + "p95": 5.0 + }, + "detail": { + "median": 5.0, + "p95": 5.0 + }, + "parse": { + "median": 5.0, + "p95": 5.0 + }, + "analyze": { + "median": 0.0, + "p95": 0 + }, + "aggregate": { + "median": 5.0, + "p95": 5.0 + } + } + }, + { + "seconds": 31.885272199999235, + "sql_count": 8785, + "python_peak_bytes": 949210, + "process_peak_rss_bytes": 94892032, + "detail_calls": 30, + "model_attempts": 0, + "verified_offers": 30, + "stage_logical_seconds": { + "search": { + "median": 5.0, + "p95": 5.0 + }, + "detail": { + "median": 5.0, + "p95": 5.0 + }, + "parse": { + "median": 5.0, + "p95": 5.0 + }, + "analyze": { + "median": 0.0, + "p95": 0 + }, + "aggregate": { + "median": 5.0, + "p95": 5.0 + } + } + }, + { + "seconds": 33.539933300000484, + "sql_count": 8846, + "python_peak_bytes": 914595, + "process_peak_rss_bytes": 96477184, + "detail_calls": 30, + "model_attempts": 0, + "verified_offers": 30, + "stage_logical_seconds": { + "search": { + "median": 5.0, + "p95": 5.0 + }, + "detail": { + "median": 5.0, + "p95": 5.0 + }, + "parse": { + "median": 5.0, + "p95": 5.0 + }, + "analyze": { + "median": 0.0, + "p95": 0 + }, + "aggregate": { + "median": 5.0, + "p95": 10.0 + } + } + } + ], + "median_seconds": 31.885272199999235, + "p95_seconds": 33.539933300000484 + } + } +} \ No newline at end of file diff --git a/goodprice/analysis/jev_typesafe.py b/goodprice/analysis/jev_typesafe.py index 5299330..220bb5c 100644 --- a/goodprice/analysis/jev_typesafe.py +++ b/goodprice/analysis/jev_typesafe.py @@ -1,7 +1,7 @@ # 官方 TypeSafe Jev 后端:通过 typesafe-sdk 直连 api.typesafe.ai, # 返回与 adapter 后端(judge.JevJudger)同构的结论,供 crawl_service 统一裁决。 import logging -import os +import math from dataclasses import dataclass from typing import Any, Callable, Optional @@ -23,6 +23,8 @@ class TypeSafeJudger: auto_threshold: float = 0.85 # 测试注入用:接收 api_key,返回实现 system_one(state, questions) 的客户端 client_factory: Optional[Callable[[str], Any]] = None + single_attempt: bool = False + timeout: float = 60 @property def enabled(self) -> bool: @@ -40,6 +42,8 @@ def analyze_requirement( ) answer = self._ask(state, {"verdict": _noul_question()})["verdict"] noul = float(answer.noul) + if not math.isfinite(noul): + raise ValueError("判断概率无效") return RequirementVerdict( matched=noul >= 0.5, confidence=max(noul, 1.0 - noul), @@ -86,15 +90,33 @@ def _make_client(self): return self.client_factory(self.api_key) from typesafe_sdk import TypeSafeClient - try: - return TypeSafeClient(api_key=self.api_key) - except TypeError: - os.environ["TYPESAFE_API_KEY"] = self.api_key - return TypeSafeClient() + if self.single_attempt: + from typesafe_sdk import RetryPolicy + return TypeSafeClient(api_key=self.api_key, retry=RetryPolicy(max_retries=0), timeout=self.timeout) + return TypeSafeClient(api_key=self.api_key, timeout=self.timeout) + + def analyze_value(self, item: dict[str, Any]) -> dict[str, Any]: + state = (f"买家品相要求:{item.get('requirement') or '无'}\n" + f"商品:id={item.get('external_id')} 标题={item.get('title')} 价格={item.get('price')}元 " + f"品相分={item.get('condition_score') or '未评估'} " + f"瑕疵={'、'.join(str(d) for d in (item.get('defects') or [])[:5]) or '无'} " + f"卖家风险={item.get('seller_risk') or '未知'}") + answer = self._ask(state, {"value": _score_question()})["value"] + raw, confidence = float(answer.score), float(getattr(answer, "confidence", .5)) + if not math.isfinite(raw): + raise ValueError("评分无效") + return {"value_score": min(SCORE_MAX, max(SCORE_MIN, int(round(raw)) + 1)), + "probability": confidence if math.isfinite(confidence) else .5, + "reason": f"官方 Jev Score={raw:.1f}"} def _ask(self, state: str, questions: dict[str, Any]) -> dict[str, Any]: - response = self._make_client().system_one(state=state, questions=questions) - return response.answers + client = self._make_client() + try: + return client.system_one(state=state, questions=questions).answers + finally: + close = getattr(client, "close", None) + if close: + close() def _noul_question() -> dict[str, Any]: diff --git a/goodprice/analysis/judge.py b/goodprice/analysis/judge.py index a4612f9..417670a 100644 --- a/goodprice/analysis/judge.py +++ b/goodprice/analysis/judge.py @@ -2,6 +2,7 @@ # 与默认路径的区别:每次判断独立成请求并附带置信度,单件失败不拖累整批; # 置信度低于阈值的判断由调用方回落到原 LLMClient 路径(宁多勿漏)。 import logging +import math from dataclasses import dataclass from typing import Any, Callable, Optional @@ -69,7 +70,8 @@ def parse_value_typed(raw: str) -> dict[str, Any]: def _clamp01(value: Any) -> float: try: - return max(0.0, min(1.0, float(value))) + number = float(value) + return max(0.0, min(1.0, number)) if math.isfinite(number) else 0.5 except (TypeError, ValueError): return 0.5 @@ -153,6 +155,15 @@ def analyze_batch_value(self, items: list[dict[str, Any]]) -> dict[str, Any]: "confidences": confidences, } + def analyze_value(self, item: dict[str, Any]) -> dict[str, Any]: + """One request, with errors intact for the execution retry policy.""" + text = VALUE_TYPED_USER_TEMPLATE.format(requirement=item.get("requirement") or "无", + external_id=item.get("external_id"), title=item.get("title"), price=item.get("price"), + condition_score=item.get("condition_score") or "未评估", + defects="、".join(str(d) for d in (item.get("defects") or [])[:5]) or "无", + seller_risk=item.get("seller_risk") or RISK_UNKNOWN) + return self._ask(VALUE_TYPED_SYSTEM_PROMPT, text, parse_value_typed) + def _ask(self, system: str, text: str, parser) -> dict[str, Any]: payload = self.llm.payload([{"type": "text", "text": text}], system=system) return self.llm.complete(payload, parser=parser) diff --git a/goodprice/analysis/llm.py b/goodprice/analysis/llm.py index b779102..aeceb8a 100644 --- a/goodprice/analysis/llm.py +++ b/goodprice/analysis/llm.py @@ -82,6 +82,7 @@ def __init__( allow_image_fallback: bool = True, retry_delay: float = 5.0, api_format: str = "chat_completions", + max_attempts: int = 3, ): self.base_url = base_url.rstrip("/") self.api_key = api_key @@ -91,6 +92,9 @@ def __init__( self.allow_image_fallback = allow_image_fallback self.retry_delay = retry_delay self.api_format = api_format.strip().lower() + if not 1 <= max_attempts <= 3: + raise ValueError("Model attempts must be between 1 and 3") + self.max_attempts = max_attempts @property def enabled(self) -> bool: @@ -203,11 +207,11 @@ def _complete( return self._complete_responses(payload, parser) headers = {"Authorization": f"Bearer {self.api_key}"} with httpx.Client(transport=self._transport, timeout=self.timeout) as client: - for attempt in range(3): + for attempt in range(self.max_attempts): response = client.post( f"{self.base_url}/chat/completions", json=payload, headers=headers ) - if response.status_code == 429 and attempt < 2: + if response.status_code == 429 and attempt < self.max_attempts - 1: if self.retry_delay: time.sleep(self.retry_delay) continue @@ -256,11 +260,11 @@ def _complete_responses( } headers = {"Authorization": f"Bearer {self.api_key}"} with httpx.Client(transport=self._transport, timeout=self.timeout) as client: - for attempt in range(3): + for attempt in range(self.max_attempts): with client.stream( "POST", f"{self.base_url}/responses", json=response_payload, headers=headers ) as response: - if response.status_code == 429 and attempt < 2: + if response.status_code == 429 and attempt < self.max_attempts - 1: if self.retry_delay: time.sleep(self.retry_delay) continue diff --git a/goodprice/analysis/media_analysis.py b/goodprice/analysis/media_analysis.py new file mode 100644 index 0000000..01de61d --- /dev/null +++ b/goodprice/analysis/media_analysis.py @@ -0,0 +1,86 @@ +import base64 +import hashlib +import json + +from goodprice.analysis.json_extract import extract_json_object +from goodprice.media.types import MediaSelection +from goodprice.offers.types import OfferCandidate + +PROMPT_VERSION = "media-evidence-1" +SYSTEM = """根据提供的商品媒体证据评估品相。卖家声明不等于观察事实。 +视频帧仅覆盖列出的采样时间,不能声称看过整个视频、音频或验证动态故障。 +不判断实际售价,不推断未证实的型号/容量。不把样品品相当成特定SKU实物品相。 +返回JSON: condition_score(0到10或null), observations/seller_claims/conflicts/unverified(数组, +每项{text:字符串,evidence_ids:本次提供的编号数组})。任何观察和分数必须有本次证据支持。 +""" + + +def validate_evidence_references(verdict: dict, allowed_ids: set[str]) -> dict: + result = {} + cited = set() + for field in ("observations", "seller_claims", "conflicts", "unverified"): + items = verdict.get(field, []) + if not isinstance(items, list) or len(items) > 30: + raise ValueError("媒体输出格式无效") + cleaned = [] + for item in items: + if not isinstance(item, dict): + raise ValueError("媒体输出缺少证据关联") + ids = item.get("evidence_ids", []) + if not isinstance(ids, list) or any(not isinstance(i, str) or i not in allowed_ids for i in ids): + raise ValueError("模型引用了未提供的证据") + if field == "observations" and not ids: + raise ValueError("观察缺少证据") + cited.update(ids) + cleaned.append({"text": str(item.get("text", ""))[:1000], "evidence_ids": ids}) + result[field] = cleaned + score = verdict.get("condition_score") + if score is not None and (type(score) is not int or not 0 <= score <= 10 or not cited): + raise ValueError("品相评分缺少有效证据或超出范围") + result["condition_score"] = score + return result + + +def analysis_key(offer, selection, client, requirement, settings, seller_text="") -> str: + input_fields = {"media_image_count", "media_video_count", "media_frames_per_video", + "media_image_quota", "media_frame_quota", "media_total_limit", "video_analysis_enabled"} + values = [PROMPT_VERSION, SYSTEM, offer.key, dict(offer.attributes), requirement, seller_text, + [(e.content_hash, e.offer_key, e.timestamp_ms) for e in selection.items], selection.limitations, + {k: v for k, v in settings.model_dump(mode="json").items() if k in input_fields}, + client.model, client.base_url, client.api_format] + return hashlib.sha256(json.dumps(values, sort_keys=True).encode()).hexdigest() + + +def analyze_media(offer: OfferCandidate, selection: MediaSelection, client, + requirement: str, request_limit_bytes: int, seller_text: str = "") -> dict: + limitations = list(selection.limitations) + if not selection.items: + return {"condition_score": None, "observations": [], "limitations": limitations + ["no_visual_input"]} + descriptors = [{"id": e.id, "kind": e.kind, "offer_key": e.offer_key, + "video_id": e.video_id, "timestamp_ms": e.timestamp_ms} for e in selection.items] + content = [{"type": "text", "text": json.dumps({"offer_key": offer.key, + "attributes": dict(offer.attributes), "requirement": requirement, "seller_text": seller_text, + "evidence": descriptors, + "limitations": limitations}, ensure_ascii=False)}] + for evidence in selection.items: + if evidence.local_path.stat().st_size * 4 // 3 > request_limit_bytes: + raise ValueError("视觉请求超过字节预算,请降低输入设置") + url = "data:image/jpeg;base64," + base64.b64encode(evidence.local_path.read_bytes()).decode() + content.append({"type": "image_url", "image_url": {"url": url}}) + payload = client.payload(content, system=SYSTEM) + # Both HTTP API formats contain the same base64 inputs. Reserve conversion + # overhead instead of silently removing evidence to fit the request. + if len(json.dumps(payload).encode()) + 1024 > request_limit_bytes: + raise ValueError("视觉请求超过字节预算,请降低输入设置") + ids = {e.id for e in selection.items} + result = client.complete(payload, parser=lambda raw: validate_evidence_references(extract_json_object(raw), ids)) + if offer.key != "default" and any(e.offer_key != offer.key for e in selection.items): + result["condition_score"] = None + limitations.append("shared_media") + if result.get("conflicts"): + result["condition_score"] = None + limitations.append("conflicting_evidence") + result["limitations"] = limitations + result["evidence_ids"] = sorted(ids) + result["prompt_version"] = PROMPT_VERSION + return result diff --git a/goodprice/analysis/prompts.py b/goodprice/analysis/prompts.py index 331aabf..080d31a 100644 --- a/goodprice/analysis/prompts.py +++ b/goodprice/analysis/prompts.py @@ -3,6 +3,13 @@ REQUIREMENT_CRITERION = "商品是否满足买家的硬性需求" VALUE_CRITERION = "按当前价格是否划算" +INDEPENDENT_VALUE_SYSTEM = """根据每件商品自身的规格、已核验的价格、品相、需求和卖家风险,独立判断性价比。 +同一商品的评分不得随本次列表包含哪些其他商品而变化;缺少市场依据时保守评分,不编造行情。 +不能推断未核验的实际售价。每件只使用自己的输入,不借用其他商品的信息。 +返回JSON: {"items":[{"id":"输入external_id","value_score":1到10整数,"reason":"简短依据"}],"best":"其中一个id"}。 +items必须逐一覆盖输入且不能添加其他id;best仅为格式兼容,实际比较由程序按当前有效商品完成。 +""" + # 需求判断系统提示前缀:adapter 与 typesafe 两个后端共用。 REQUIREMENT_SYSTEM_PREFIX = "你是二手商品筛选助手。用户给出商品标题、卖家描述和买家需求," diff --git a/goodprice/config.py b/goodprice/config.py index feb741a..ca12d29 100644 --- a/goodprice/config.py +++ b/goodprice/config.py @@ -1,7 +1,7 @@ from functools import lru_cache from pathlib import Path -from pydantic import field_validator +from pydantic import Field, field_validator, model_validator from pydantic_settings import BaseSettings, SettingsConfigDict PROJECT_ROOT = Path(__file__).resolve().parent.parent @@ -19,6 +19,7 @@ "feishu_secret", "gotify_token", "jev_api_key", + "xianyu_cookie", } ) @@ -40,6 +41,7 @@ class Settings(BaseSettings): ) app_name: str = "闲鱼盯价助手" + data_root: Path = PROJECT_ROOT / "data" database_url: str = f"sqlite:///{DEFAULT_DB.as_posix()}" xianyu_cookie: str = "" llm_base_url: str = "" @@ -48,8 +50,8 @@ class Settings(BaseSettings): llm_api_format: str = "chat_completions" serverchan_sendkey: str = "" proxy: str = "" - default_crawl_interval_minutes: int = 20 - default_crawl_jitter_minutes: int = 10 + default_crawl_interval_minutes: int = Field(20, ge=1) + default_crawl_jitter_minutes: int = Field(10, ge=0) vision_base_url: str = "" vision_api_key: str = "" vision_model: str = "glm-4.6v-flash" @@ -60,7 +62,7 @@ class Settings(BaseSettings): feishu_enabled: bool = True gotify_url: str = "" gotify_token: str = "" - gotify_priority: int = 5 + gotify_priority: int = Field(5, ge=0, le=10) gotify_enabled: bool = True serverchan_enabled: bool = True wecom_robot_enabled: bool = True @@ -73,6 +75,45 @@ class Settings(BaseSettings): jev_auto_threshold: float = 0.85 jev_backend: str = "adapter" jev_api_key: str = "" + video_analysis_enabled: bool = False + media_image_count: int = Field(8, ge=1, le=32) + media_video_count: int = Field(1, ge=1, le=3) + media_frames_per_video: int = Field(6, ge=1, le=24) + media_image_quota: int = Field(4, ge=0, le=16) + media_frame_quota: int = Field(4, ge=0, le=16) + media_total_limit: int = Field(8, ge=1, le=16) + media_model_limit: int = Field(8, ge=1, le=16) + media_video_max_mib: int = Field(100, ge=10, le=200) + media_video_max_seconds: int = Field(180, ge=15, le=300) + media_cache_mib: int = Field(1024, ge=256, le=10240) + media_retention_days: int = Field(7, ge=1, le=30) + media_download_timeout: int = Field(60, ge=10, le=120) + media_decode_timeout: int = Field(30, ge=10, le=120) + media_request_max_mib: int = Field(16, ge=1, le=32) + media_ffmpeg_threads: int = Field(1, ge=1, le=4) + execution_active_runs: int = Field(2, ge=1, le=4) + execution_crawler_workers: int = Field(1, ge=1, le=2) + execution_model_workers: int = Field(1, ge=1, le=8) + execution_media_workers: int = Field(1, ge=1, le=2) + execution_platform_interval_seconds: int = Field(10, ge=5, le=300) + execution_model_rpm: int = Field(10, ge=1, le=60) + execution_global_items: int = Field(200, ge=1, le=200) + execution_task_items: int = Field(60, ge=1, le=60) + execution_run_details: int = Field(30, ge=1, le=100) + execution_run_http_attempts: int = Field(60, ge=1, le=200) + execution_run_minutes: int = Field(15, ge=5, le=60) + + @model_validator(mode="after") + def _media_budgets(self): + if self.media_total_limit > self.media_model_limit: + raise ValueError("视觉输入总上限不能超过已确认的模型上限(未知时为8)") + # Serial assets share the same reservation; retained outputs from earlier + # assets remain charged. JPEG outputs are capped at 2 MiB per image/frame. + required = self.media_video_max_mib + 2 * ( + self.media_image_count + self.media_video_count * self.media_frames_per_video) + if required > self.media_cache_mib: + raise ValueError("媒体缓存不足以容纳一次处理的下载和输出,请减少视频/采样预算") + return self @field_validator("jev_backend") @classmethod diff --git a/goodprice/crawler/base.py b/goodprice/crawler/base.py index 14a0749..9c92cd2 100644 --- a/goodprice/crawler/base.py +++ b/goodprice/crawler/base.py @@ -2,6 +2,9 @@ from datetime import datetime from typing import Optional +from goodprice.media.types import MediaCandidate +from goodprice.offers.types import OfferCandidate + @dataclass class ListingData: @@ -20,6 +23,8 @@ class ListingDetail: description: str = "" image_urls: list[str] = field(default_factory=list) variants: list[dict] = field(default_factory=list) + offers: list[OfferCandidate] = field(default_factory=list) + media: list[MediaCandidate] = field(default_factory=list) seller_uid: Optional[str] = None seller_name: Optional[str] = None credit_label: Optional[str] = None diff --git a/goodprice/crawler/offers.py b/goodprice/crawler/offers.py new file mode 100644 index 0000000..fd30ecd --- /dev/null +++ b/goodprice/crawler/offers.py @@ -0,0 +1,120 @@ +"""Conservative, bounded offer parsing. Unsupported platform state stays unknown. + +The data-* fixture format and Schema.org Product/Offer are supported adapters, +not a claim that every current Goofish SKU widget exposes these attributes. +""" +import json +import re +from dataclasses import replace +from datetime import datetime, timezone + +from bs4 import BeautifulSoup + +from goodprice.crawler import selectors as sel +from goodprice.offers.matching import stable_offer_key +from goodprice.offers.pricing import parse_money_cents +from goodprice.offers.types import OfferCandidate, PriceClaim + +_INQUIRY = re.compile(r"(?:标价.{0,5}(?:非|不代表|不为).{0,5}(?:售|价)|(?:价格|报价).{0,8}(?:私聊|咨询|面议)|(?:私聊|咨询).{0,8}(?:价格|报价))") + + +def _claim(raw, source, now, evidence_id, scope=None): + text = str(raw or "").strip()[:1000] + kind, amount, low, high = "unknown", None, None, None + if _INQUIRY.search(text): + kind = "inquiry" + elif "起" in text or re.search(r"\d\s*[-~至]\s*\d", text): + kind = "range_or_from" + numbers = re.findall(r"\d+(?:\.\d{1,2})?", text) + if numbers: + low = parse_money_cents(numbers[0]) + high = parse_money_cents(numbers[1]) if len(numbers) > 1 else None + else: + try: + amount = parse_money_cents(text) + kind = "exact" if amount > 0 else "unknown" + except ValueError: + pass + return PriceClaim(source, kind, text, now, evidence_id, amount, low, high, scope) + + +def parse_offer_candidates(html: str) -> list[OfferCandidate]: + now = datetime.now(timezone.utc) + soup = BeautifulSoup(html, "html.parser") + root = soup.select_one("[data-item-detail], main, [class*='detail-container--']") + result = [] + if root: + description = root.select_one("[data-item-description], " + sel.DETAIL_DESC) + disclaimer = description.get_text(" ", strip=True) if description else "" + global_claims = ((_claim(disclaimer, "description", now, "description"),) + if _INQUIRY.search(disclaimer) else ()) + nodes = root.select("[data-sku-id]") + for i, node in enumerate(nodes[:21]): + if i == 20: + result = [replace(o, claims=(*o.claims, PriceClaim( + "dom", "unknown", "规格获取不完整", now, "truncated"))) for o in result] + break + try: + attrs = json.loads(node.get("data-attributes", "{}")) + if not isinstance(attrs, dict) or any(not isinstance(v, str) for v in attrs.values()): + continue + key = stable_offer_key(node.get("data-sku-id"), attrs) + if not key: + continue + claim = _claim(node.get("data-price"), "sku", now, f"{key}:price", key) + available = "unavailable" if node.get("data-available") == "false" else "unknown" + if node.get("data-available") == "true": + available = "available" + result.append(OfferCandidate(key, attrs, available, (claim, *global_claims), + node.get("data-name", ""))) + except (ValueError, TypeError): + continue + if not nodes: + price = root.select_one("[data-item-price], " + sel.DETAIL_PRICE_RANGE) + if price or global_claims: + result.append(OfferCandidate("default", {}, "unknown", ( + _claim(price.get_text(" ", strip=True) if price else "", "dom", now, "detail:price"), + *global_claims), "默认商品")) + if not result: + products = [] + for script in soup.select('script[type="application/ld+json"]')[:10]: + try: + data = json.loads(script.string or "") + except (ValueError, TypeError): + continue + items = data if isinstance(data, list) else [data] + products.extend(d for d in items if isinstance(d, dict) and d.get("@type") == "Product") + if len(products) == 1: + product = products[0] + offers = product.get("offers", []) + offers = [offers] if isinstance(offers, dict) else offers + if isinstance(offers, list): + for i, data in enumerate(offers[:20]): + if not isinstance(data, dict) or data.get("priceCurrency", "CNY") != "CNY": + continue + attrs = data.get("itemOffered", {}).get("additionalProperty", []) if isinstance(data.get("itemOffered", {}), dict) else [] + attrs = {str(a["name"]): str(a["value"]) for a in attrs + if isinstance(a, dict) and "name" in a and "value" in a} if isinstance(attrs, list) else {} + key = stable_offer_key(data.get("sku"), attrs) or ("default" if len(offers) == 1 else None) + if key: + claim = _claim(data.get("price", ""), "sku", now, f"jsonld:{i}", key) + if len(offers) > 20: + claim = replace(claim, kind="unknown", amount_cents=None, text="规格获取不完整") + available = "unavailable" if "OutOfStock" in str(data.get("availability")) else "unknown" + if "InStock" in str(data.get("availability")): + available = "available" + description = str(product.get("description", "")) + disclaimers = (_claim(description, "description", now, "jsonld:description"),) if _INQUIRY.search(description) else () + result.append(OfferCandidate(key, attrs, available, (claim, *disclaimers), str(data.get("name", "")))) + if not result: + result = [OfferCandidate("default", claims=(PriceClaim( + "dom", "unknown", "未找到可核验的规格价格", now, "missing"),))] + merged = {} + for offer in result: + if offer.key in merged: + old = merged[offer.key] + merged[offer.key] = replace(old, claims=(*old.claims, *offer.claims, PriceClaim( + "dom", "conflict", "相同规格身份存在重复记录", now, "duplicate"))) + else: + merged[offer.key] = offer + return list(merged.values()) diff --git a/goodprice/crawler/parser.py b/goodprice/crawler/parser.py index 99a1adf..6649607 100644 --- a/goodprice/crawler/parser.py +++ b/goodprice/crawler/parser.py @@ -7,6 +7,8 @@ from goodprice.crawler import selectors as sel from goodprice.crawler.base import ListingData, ListingDetail, SellerData from goodprice.crawler.constants import GOOFISH_ORIGIN +from goodprice.crawler.offers import parse_offer_candidates +from goodprice.media.parser import extract_media_candidates BASE_URL = GOOFISH_ORIGIN _PRICE_RE = re.compile(r"(\d+(?:\.\d+)?)") @@ -113,6 +115,8 @@ def parse_detail_html(html: str) -> ListingDetail: credit_el = soup.select_one(sel.DETAIL_CREDIT_LABEL) credit_label = credit_el.get_text(strip=True) if credit_el else None return ListingDetail( + media=extract_media_candidates(html), + offers=parse_offer_candidates(html), description=desc[:2000], image_urls=images[:8], variants=variants, diff --git a/goodprice/crawler/xianyu.py b/goodprice/crawler/xianyu.py index f1d7164..28dff47 100644 --- a/goodprice/crawler/xianyu.py +++ b/goodprice/crawler/xianyu.py @@ -1,3 +1,4 @@ +import threading from contextlib import contextmanager from typing import Callable, Optional from urllib.parse import quote @@ -36,6 +37,9 @@ def __init__( self.proxy = proxy self.headless = headless self._playwright_factory = playwright_factory or sync_playwright + self._browser = None + self._owner = None + self._session_lock = threading.Lock() def _cookies(self) -> dict[str, str]: result: dict[str, str] = {} @@ -49,24 +53,49 @@ def _cookies(self) -> dict[str, str]: return result @contextmanager - def _open_page(self): - """统一的浏览器/上下文/Cookie 装配,三处抓取复用。""" + def session(self): + if not self._session_lock.acquire(blocking=False): + raise RuntimeError("Browser session already has an owning thread") + try: + with self._session(): + yield self + finally: + self._session_lock.release() + + @contextmanager + def _session(self): + if self._owner is not None: + raise RuntimeError("Browser session already has an owning thread") with self._playwright_factory() as playwright: - browser = playwright.chromium.launch( + self._browser = playwright.chromium.launch( headless=self.headless, proxy={"server": self.proxy} if self.proxy else None, ) + self._owner = threading.get_ident() try: - context = browser.new_context(user_agent=USER_AGENT) - context.add_cookies( - [ - {"name": k, "value": v, "domain": ".goofish.com", "path": "/"} - for k, v in self._cookies().items() - ] - ) - yield context.new_page() + yield self finally: - browser.close() + try: + self._browser.close() + finally: + self._browser, self._owner = None, None + + @contextmanager + def _open_page(self): + if self._browser is None: + with self.session(), self._open_page() as page: + yield page + return + if self._owner != threading.get_ident(): + raise RuntimeError("Browser must be used by its owning thread") + context = self._browser.new_context(user_agent=USER_AGENT) + try: + context.add_cookies([ + {"name": k, "value": v, "domain": ".goofish.com", "path": "/"} + for k, v in self._cookies().items()]) + yield context.new_page() + finally: + context.close() def search( self, keyword: str, max_items: int = 30, attempts: int = SEARCH_ATTEMPTS diff --git a/goodprice/db.py b/goodprice/db.py index 385a3c3..7b7d5f2 100644 --- a/goodprice/db.py +++ b/goodprice/db.py @@ -1,6 +1,6 @@ from pathlib import Path -from sqlalchemy import create_engine, text +from sqlalchemy import create_engine, inspect, text from sqlalchemy.orm import DeclarativeBase, sessionmaker _LISTINGS_DDL = """ @@ -9,6 +9,9 @@ external_id VARCHAR(200) NOT NULL, title VARCHAR(500) NOT NULL, price FLOAT NOT NULL, + offer_price_cents INTEGER, + offer_verified_until DATETIME, + offer_task_version INTEGER, url TEXT NOT NULL, image_urls JSON, seller VARCHAR(200), @@ -66,18 +69,48 @@ def init_db(database_url: str) -> None: from goodprice import models # noqa: F401 确保模型注册 factory = make_session_factory(database_url) - Base.metadata.create_all(factory().get_bind()) + engine = factory.kw["bind"] + legacy = inspect(engine).has_table("watch_tasks") + Base.metadata.create_all(engine) + # Record installation type before additive table creation hides its provenance. + from goodprice.models import SchemaVersion + + with factory() as session: + if session.get(SchemaVersion, 1) is None: + session.add(SchemaVersion(id=1, version=0, legacy_install=legacy)) + session.commit() + engine.dispose() def migrate_schema(session_factory) -> None: """幂等迁移:为已有数据库补齐新列。""" columns = { + "work_items": [ + ("started_at", "started_at DATETIME"), + ("finished_at", "finished_at DATETIME"), + ], + "listing_offers": [ + ("seen_in_latest_detail", "seen_in_latest_detail BOOLEAN DEFAULT 1"), + ("last_notified_task_version", "last_notified_task_version INTEGER"), + ("effective_price_state", "effective_price_state VARCHAR(20) DEFAULT 'unknown'"), + ("effective_price_cents", "effective_price_cents INTEGER"), + ("effective_price_source", "effective_price_source VARCHAR(30)"), + ("price_evidence_fingerprint", "price_evidence_fingerprint VARCHAR(64)"), + ("price_valid_until", "price_valid_until DATETIME"), + ("price_task_version", "price_task_version INTEGER"), + ], "watch_tasks": [ ("fetch_detail", "fetch_detail BOOLEAN DEFAULT 1"), ("min_price", "min_price FLOAT DEFAULT 0"), ("exclude_words", "exclude_words TEXT"), + ("target_variant", "target_variant TEXT DEFAULT ''"), + ("notify_pending_price", "notify_pending_price BOOLEAN DEFAULT 0"), + ("version", "version INTEGER DEFAULT 1"), ], "listings": [ + ("offer_price_cents", "offer_price_cents INTEGER"), + ("offer_verified_until", "offer_verified_until DATETIME"), + ("offer_task_version", "offer_task_version INTEGER"), ("description", "description TEXT"), ("requirement_match", "requirement_match BOOLEAN"), ("requirement_reason", "requirement_reason TEXT"), @@ -100,6 +133,9 @@ def migrate_schema(session_factory) -> None: ("blocked", "blocked BOOLEAN DEFAULT 0"), ], "notifications": [ + ("event_context", "event_context JSON"), + ("offer_id", "offer_id INTEGER"), + ("event_key", "event_key VARCHAR(128)"), ("title", "title TEXT"), ("content", "content TEXT"), ], @@ -120,6 +156,9 @@ def migrate_schema(session_factory) -> None: session.execute(text(f"ALTER TABLE {table} ADD COLUMN {ddl}")) if "listings" in existing_tables and _listing_unique_columns(session) != _NEW_LISTING_UNIQUE: _rebuild_listings(session) + from goodprice.migrations import migrate_offer_schema + + migrate_offer_schema(session) session.commit() diff --git a/goodprice/execution/__init__.py b/goodprice/execution/__init__.py new file mode 100644 index 0000000..e1d0a4d --- /dev/null +++ b/goodprice/execution/__init__.py @@ -0,0 +1 @@ +"""Persistent bounded execution; no background work on import.""" diff --git a/goodprice/execution/browser.py b/goodprice/execution/browser.py new file mode 100644 index 0000000..18b7788 --- /dev/null +++ b/goodprice/execution/browser.py @@ -0,0 +1,128 @@ +"""Browser actors close Playwright on the same thread that created it.""" +import logging +import queue +import threading +import time +from concurrent.futures import Future + +logger = logging.getLogger(__name__) +_local = threading.local() +_TRIM = object() + + +def current_browser(key, factory): + actor = getattr(_local, "actor", None) + if actor is None: + raise RuntimeError("Browser access requires its owning worker") + return actor.browser(key, factory) + + +class BrowserActor: + def __init__(self, name): + self.name = name + self.queue = queue.Queue(maxsize=1) + self._lock = threading.Lock() + self._thread = None + self._busy = False + self._stopped = False + self._context = self._browser = self._key = None + self._uses, self._opened = 0, 0 + + def available(self): + with self._lock: + return not self._busy and not self._stopped + + def submit(self, fn): + with self._lock: + if self._busy or self._stopped: + raise RuntimeError("Browser worker full or stopped") + self._busy = True + future = Future() + self.queue.put_nowait((fn, future)) + if self._thread is None: + self._thread = threading.Thread(target=self._loop, name=self.name) + self._thread.start() + return future + + def browser(self, key, factory): + if threading.current_thread() is not self._thread: + raise RuntimeError("Browser ownership violation") + if self._context is not None and (key != self._key or self._uses >= 100 or time.monotonic() - self._opened >= 600): + self._close_browser() + if self._context is None: + context = factory() + browser = context.__enter__() + self._context, self._browser = context, browser + self._key, self._opened, self._uses = key, time.monotonic(), 0 + self._uses += 1 + return self._browser + + def _close_browser(self): + context, self._context = self._context, None + self._browser, self._key = None, None + if context: + context.__exit__(None, None, None) + + def _loop(self): + _local.actor = self + try: + while True: + try: + work = self.queue.get(timeout=5) + except queue.Empty: + if self._context is not None and time.monotonic() - self._opened >= 600: + self._close_browser() + continue + if work is None: + break + if work is _TRIM: + try: + self._close_browser() + finally: + with self._lock: + self._busy = False + continue + fn, future = work + if self._stopped: + future.cancel() + break + if not future.set_running_or_notify_cancel(): + with self._lock: + self._busy = False + continue + result = error = None + try: + result = fn() + except BaseException as exc: + error = exc + try: + self._close_browser() + except Exception as close_error: + logger.warning("浏览器回收失败 (%s)", type(close_error).__name__) + with self._lock: + self._busy = False + if error: + future.set_exception(error) + else: + future.set_result(result) + finally: + try: + self._close_browser() + finally: + _local.actor = None + + def stop(self, timeout=10): + with self._lock: + self._stopped = True + try: + self.queue.put_nowait(None) + except queue.Full: + pass + if self._thread: + self._thread.join(timeout) + + def trim_idle(self): + with self._lock: + if self._thread and not self._busy and not self._stopped: + self._busy = True + self.queue.put_nowait(_TRIM) diff --git a/goodprice/execution/coordinator.py b/goodprice/execution/coordinator.py new file mode 100644 index 0000000..6a180bf --- /dev/null +++ b/goodprice/execution/coordinator.py @@ -0,0 +1,245 @@ +import logging +import math +import threading +import time +from datetime import datetime, timezone +from email.utils import parsedate_to_datetime + +from sqlalchemy.exc import OperationalError + +from goodprice.execution.types import Stage + +logger = logging.getLogger(__name__) + + +class Coordinator: + def __init__(self, store, limits, workers, clock=time.time, apply=None): + self.store, self.limits, self.workers, self.clock = store, limits, workers, clock + self._stop = threading.Event() + self._wake = threading.Event() + self._thread = None + self._futures = {} + self._delivery_futures = {} + self.apply = apply + self._last_renew = self.clock() + self._requested_limits = None + self.effective = None + + def submit(self, task_id): + result = self.store.submit(task_id, "crawl") + self._wake.set() + return result if result is not None else False + + def submit_reanalysis(self, listing_id): + from goodprice.models import Listing + with self.store.factory() as session: + listing = session.get(Listing, listing_id) + if listing is None or not listing.task_id: + return False + result = self.store.submit(listing.task_id, "reanalyze", listing_id) + self._wake.set() + return result if result is not None else False + + def pause(self, task_id): + self.store.invalidate_task(task_id) + + def running_ids(self): + from goodprice.models import TaskRun + with self.store.factory() as session: + return {r[0] for r in session.query(TaskRun.task_id).filter_by(state="active")} + + def queued_ids(self): + from goodprice.models import TaskRun + with self.store.factory() as session: + return [r[0] for r in session.query(TaskRun.task_id).filter( + TaskRun.state.in_(("queued", "waiting_capacity"))).order_by(TaskRun.id)] + + def wake(self): + self._wake.set() + + def describe(self, task_id): + from goodprice.models import TaskRun, WorkItem + with self.store.factory() as session: + run = session.query(TaskRun).filter_by(task_id=task_id).order_by(TaskRun.id.desc()).first() + if run is None: + return None + work = session.query(WorkItem).filter_by(run_id=run.id).all() + cursor = run.cursor or {} + remaining = set(cursor.get("remaining_ids", [])) + stages = sorted({item.stage for item in work if item.state == "running"}) + return {"run_id": run.id, "state": run.state, "stages": stages, + "planned": len(cursor.get("planned_ids", [])), + "completed": len(set(cursor.get("planned_ids", [])) - remaining), + "deferred_candidates": len(remaining), "model_http_attempts": cursor.get("_http_attempts", 0), + "cache_hits": sum(bool((item.output or {}).get("cache_hit")) for item in work), + "failed_steps": sum(item.state == "failed" for item in work)} + + def update_limits(self, limits): + self._requested_limits = limits + self._wake.set() + + def _apply_limits(self, limits): + self.workers.update_limits(limits) + with self.store.lock: + self.store.active_limit = limits.active_runs + self.store.global_limit = limits.global_items + self.store.task_limit = limits.task_items + self.store.run_minutes = limits.run_minutes + self.limits = limits + self._wake.set() + + def start(self): + if self._thread and self._thread.is_alive(): + return + self._stop.clear() + self.store.recover(self._now(), startup=True) + pipeline = getattr(self, "pipeline", None) + if pipeline: + pipeline.delivery.recover(self._now(), startup=True) + self._thread = threading.Thread(target=self._loop, name="second-eye-coordinator", daemon=True) + self._thread.start() + + def _now(self): + return datetime.fromtimestamp(self.clock(), timezone.utc) + + def tick(self): + now = self._now() + if self._requested_limits is not None: + self._apply_limits(self._requested_limits) + self._requested_limits = None + pipeline = getattr(self, "pipeline", None) + if pipeline: + from goodprice.execution.limits import ResourceLimits + requested = ResourceLimits.from_settings(pipeline.settings.get()) + if requested != self.limits: + self._apply_limits(requested) + self.effective = pipeline.resources(self.limits) + self.workers.update_limits(self.effective.limits) + if self.effective.low: + for actor in self.workers.browsers: + actor.trim_idle() + if self.clock() - self._last_renew >= 30: + for item in self._futures.values(): + self.store.renew(item.token, now) + self._last_renew = self.clock() + for future in list(self._delivery_futures): + if future.done(): + del self._delivery_futures[future] + try: + future.result() + except Exception as exc: + logger.warning("投递状态暂未完成 (%s)", type(exc).__name__) + for future, item in list(self._futures.items()): + if not future.done(): + continue + try: + result = future.result() + except Exception as exc: + del self._futures[future] + # Never log worker response bodies, URLs or credentials. + logger.warning("阶段 %s 失败 (%s)", item.stage.value, type(exc).__name__) + delay = self._retry_after(exc, now) + if delay: + provider = item.payload.get("provider_gate") + self.store.postpone_gate("provider:" + provider if provider else "model", now.timestamp() + delay) + hook = getattr(getattr(self, "pipeline", None), "failed", None) + self.store.fail(item, now, delay_seconds=delay, apply=hook) + self.store.finish_run(item.token.run_id, now) + continue + try: + self.store.complete(result, now, apply=self.apply) + except OperationalError: + # Keep the completed response in the bounded in-flight set. + # A transient SQLite lock must not repeat the paid HTTP call. + return + except Exception as exc: + logger.warning("结果写入失败 (%s)", type(exc).__name__) + self.store.fail(item, now, terminal=True) + del self._futures[future] + self.store.materialize(now) + self.store.finish_run(item.token.run_id, now) + self.store.recover(now) + self.store.materialize(now) + pipeline = getattr(self, "pipeline", None) + if pipeline: + for run_id in pipeline.maintain(self.store, now): + self.store.finish_run(run_id, now) + for stage in Stage: + while self.workers.available(stage): + lane = {Stage.SEARCH: "crawler", Stage.DETAIL: "crawler", Stage.MEDIA: "media"}.get(stage) + if self.effective and lane in self.effective.paused: + break + if pipeline and stage in {Stage.SEARCH, Stage.DETAIL} and any( + pipeline.platform_io(work) for work in self._futures.values()): + break + item = self.store.claim(stage, now) + if item is None: + break + if pipeline: + cached = pipeline.cached(item) + if cached is not None: + self.store.complete(cached, now, apply=self.apply) + self.store.materialize(now) + self.store.finish_run(item.token.run_id, now) + continue + if self.effective and item.payload.get("http") and "model" in self.effective.paused: + self.store.wait(item, now, 5) + break + if item.payload.get("http") and item.attempt > 3: + self.store.fail(item, now, terminal=True, apply=pipeline.failed) + self.store.finish_run(item.token.run_id, now) + continue + if item.payload.get("http") and not self.store.admit(item, now, + gate="model", interval=60 / self.limits.model_rpm, http_limit=self.limits.run_http_attempts, + provider=item.payload.get("provider_gate")): + break + if pipeline.platform_io(item) and not self.store.admit(item, now, + gate="platform", interval=self.limits.platform_interval_seconds): + break + future = self.workers.dispatch_item(item) + self._futures[future] = item + future.add_done_callback(lambda _: self._wake.set()) + if pipeline and self.workers.side_available("delivery"): + events = pipeline.delivery_events(now) + if events: + event_id = events[0] + future = self.workers.dispatch_side("delivery", lambda: pipeline.delivery.deliver(event_id, self._now())) + self._delivery_futures[future] = event_id + future.add_done_callback(lambda _: self._wake.set()) + + def _retry_after(self, exc, now): + response = getattr(exc, "response", None) + status = getattr(response, "status_code", None) or getattr(exc, "status", None) + if status != 429: + return None + headers = getattr(response, "headers", None) or getattr(exc, "headers", {}) + raw = headers.get("retry-after", "5") + try: + seconds = float(raw) + except (ValueError, TypeError): + try: + seconds = parsedate_to_datetime(raw).timestamp() - now.timestamp() + except (ValueError, TypeError, OverflowError): + return 5 + return max(0, min(seconds, 86400)) if math.isfinite(seconds) else 5 + + def _loop(self): + while not self._stop.is_set(): + delay = 1 + try: + self.tick() + delay = 1 if self._futures or self._delivery_futures or self.store.has_open_work() else 10 + except Exception as exc: + logger.warning("协调器暂未推进 (%s)", type(exc).__name__) + self._wake.wait(delay) + self._wake.clear() + + def stop(self, timeout=10): + self._stop.set() + self._wake.set() + if self._thread: + self._thread.join(timeout) + self.workers.stop() + pipeline = getattr(self, "pipeline", None) + if pipeline: + pipeline.registry.close() diff --git a/goodprice/execution/delivery.py b/goodprice/execution/delivery.py new file mode 100644 index 0000000..4db7191 --- /dev/null +++ b/goodprice/execution/delivery.py @@ -0,0 +1,242 @@ +"""A durable per-channel outbox. External delivery cannot promise exactly-once.""" +import time +import uuid +from contextlib import nullcontext +from datetime import timedelta + +import httpx + +from goodprice.execution.store import ExecutionStore, naive +from goodprice.models import ( + Listing, + ListingOffer, + Notification, + NotificationDelivery, + NotificationEvent, + WatchTask, +) +from goodprice.notify.base import DeliveryRejected, NotificationMessage +from goodprice.services.offer_pipeline import OfferPipeline + +SENDABLE = ("pending", "failed", "awaiting_revalidation", "retry_requested") + + +class DeliveryService: + def __init__(self, session_factory, notifiers, vision_enabled=True): + self.factory = session_factory + self.notifiers = notifiers + self.lock = ExecutionStore(session_factory).lock + self.vision_enabled = vision_enabled + + def enqueue(self, task_id, task_version, listing_id, offer_id, business_key, payload, *, session=None): + if not business_key or len(business_key) > 128: + raise ValueError("Invalid notification event key") + own = session is None + with self.lock, self.factory() if own else nullcontext(session) as session: + listing = session.get(Listing, listing_id) + offer = session.get(ListingOffer, offer_id) + if listing and offer: + payload = {**payload, "product_identity": [listing.platform, listing.external_id, offer.offer_key]} + existing = session.query(NotificationEvent).filter_by(business_key=business_key).first() + if existing: + rows = session.query(NotificationDelivery).filter_by(event_id=existing.id).all() + resumable = [r for r in rows if r.state in {"cancelled", "superseded", "awaiting_revalidation", "pending", "failed"}] + if resumable: + existing.task_version, existing.payload = task_version, payload + existing.state = "pending" + for row in resumable: + if row.state in {"cancelled", "superseded", "awaiting_revalidation"} and row.attempt < 3: + row.state, row.not_before = "pending", None + self._history(session, existing, row, "pending") + if own: + session.commit() + return existing.id + event = NotificationEvent(task_id=task_id, task_version=task_version, listing_id=listing_id, + offer_id=offer_id, business_key=business_key, payload=payload) + session.add(event) + session.flush() + notifiers = self.notifiers() if callable(self.notifiers) else self.notifiers + for channel in notifiers: + prior_sent = session.query(Notification).filter_by(event_key=business_key, channel=channel, status="sent").first() + row = NotificationDelivery(event_id=event.id, channel=channel, + state="sent" if prior_sent else "pending") + session.add(row) + session.flush() + if not prior_sent: + session.add(Notification(task_id=task_id, listing_id=listing_id, offer_id=offer_id, + event_key=business_key, channel=channel, status="pending", title=payload["title"], + content=payload["content"], event_context={"delivery_id": row.id, "task_version": task_version})) + if own: + session.commit() + return event.id + + def _validate(self, event, now): + with self.factory() as session: + task = session.get(WatchTask, event.task_id) + listing = session.get(Listing, event.listing_id) + if task is None or not task.enabled or task.version != event.task_version or listing is None or listing.blocked or listing.task_id != event.task_id: + return "cancelled" + identity = event.payload.get("product_identity") + if identity and identity[:2] != [listing.platform, listing.external_id]: + return "cancelled" + score = listing.condition_score + media_key = (listing.condition_detail or {}).get("media_cache_key") + try: + vision = self.vision_enabled() if callable(self.vision_enabled) else self.vision_enabled + offers = OfferPipeline(self.factory, vision_enabled=vision).snapshot(event.listing_id, now) + except LookupError: + return "cancelled" + current = next((o for o in offers if o["id"] == event.offer_id), None) + if current is None: + return "cancelled" + if identity and identity[2] != current["key"]: + return "cancelled" + payload = event.payload + if not current.get("scope_current", True): + return "awaiting_revalidation" + if payload.get("offer_evaluation"): + evaluation = current.get("evaluation") or {} + if evaluation.get("status", "ready") not in {"ready", "completed"}: + return "awaiting_revalidation" + score = evaluation.get("condition_score") + media_key = evaluation.get("media_cache_key") + if (current["fingerprint"] != payload["fingerprint"] or score != payload.get("condition_score") + or media_key != payload.get("media_cache_key")): + return "superseded" + if payload.get("pending"): + if not current.get("observation_fresh"): + return "awaiting_revalidation" + return "valid" if current["price_state"] != "exact" else "superseded" + if current["price_state"] != "exact": + return "awaiting_revalidation" + if current["source"] != payload["source"] or current["effective_price_cents"] != payload["amount_cents"]: + return "superseded" + if not current.get("evaluation", {}).get("eligible"): + return "superseded" + return "valid" + + def _history(self, session, event, row, state, detail=None): + history = session.query(Notification).filter_by(event_key=event.business_key, + channel=row.channel).order_by(Notification.id.desc()).first() + if history: + history.status, history.detail = state, detail + + def _invalidate(self, event_id, state, now=None): + with self.lock, self.factory() as session: + event = session.get(NotificationEvent, event_id) + event.state = state + for row in session.query(NotificationDelivery).filter(NotificationDelivery.event_id == event_id, + NotificationDelivery.state.in_(SENDABLE)): + row.state = state + if state == "awaiting_revalidation" and now: + row.not_before = naive(now) + timedelta(seconds=30) + self._history(session, event, row, state) + session.commit() + return state + + def deliver(self, event_id, now): + began = time.monotonic() + with self.factory() as session: + event = session.get(NotificationEvent, event_id) + if event is None: + return "cancelled" + deliveries = session.query(NotificationDelivery).filter_by(event_id=event_id).order_by(NotificationDelivery.id).all() + if any(row.state in SENDABLE for row in deliveries): + valid = self._validate(event, now) + if valid != "valid": + return self._invalidate(event.id, valid, now) + for original in deliveries: + if original.state not in SENDABLE or (original.not_before and original.not_before > naive(now)): + continue + check_time = now + timedelta(seconds=time.monotonic() - began) + valid = self._validate(event, check_time) + if valid != "valid": + return self._invalidate(event.id, valid, check_time) + with self.lock, self.factory() as session: + row = session.get(NotificationDelivery, original.id) + # Last short write-gate before crossing the external boundary. + task = session.get(WatchTask, event.task_id) + if task is None or not task.enabled or task.version != event.task_version: + return self._invalidate(event.id, "cancelled") + if row.state not in SENDABLE or (row.attempt >= 3 and row.state != "retry_requested"): + continue + lease = uuid.uuid4().hex + row.state, row.lease_id = "sending", lease + row.lease_until = naive(check_time) + timedelta(seconds=120) + row.attempt += 1 + session.commit() + status, detail = "sent", None + notifiers = self.notifiers() if callable(self.notifiers) else self.notifiers + notifier = notifiers.get(original.channel) + try: + if notifier is None: + raise DeliveryRejected("channel unavailable") + notifier.send(NotificationMessage(event.payload["title"], event.payload["content"], event.payload["url"])) + except (DeliveryRejected, httpx.ConnectError, httpx.ConnectTimeout, httpx.PoolTimeout): + status, detail = "failed", "渠道不可用或投递失败,未记录响应中的敏感信息" + except httpx.HTTPStatusError as exc: + status = "failed" if 400 <= exc.response.status_code < 500 and exc.response.status_code != 408 else "uncertain" + detail = "渠道明确拒绝请求" if status == "failed" else "服务端投递结果未知,请核验后人工重试(可能重复)" + except Exception: + status, detail = "uncertain", "投递结果未知,请核验后人工重试(可能重复)" + with self.lock, self.factory() as session: + row = session.get(NotificationDelivery, original.id) + if row.lease_id != lease: + # Recovery has already classified this interrupted send. + continue + row.state, row.lease_id, row.lease_until = status, None, None + row.not_before = naive(now) + timedelta(seconds=min(300, 5 * 2**row.attempt)) + self._history(session, event, row, status, detail) + if status == "sent": + listing = session.get(Listing, event.listing_id) + if listing: + listing.notified_at = naive(check_time) + offer = session.get(ListingOffer, event.offer_id) + if offer and not event.payload.get("pending"): + offer.last_notified_score = event.payload.get("satisfaction", offer.last_notified_score) + offer.last_notified_amount, offer.last_notified_source = event.payload["amount_cents"], event.payload["source"] + offer.last_notified_task_version = event.task_version + session.commit() + with self.lock, self.factory() as session: + states = {r.state for r in session.query(NotificationDelivery).filter_by(event_id=event_id)} + state = next((s for s in ["uncertain", "sending", "failed", "awaiting_revalidation", "pending", "cancelled_manual", "cancelled", "superseded"] if s in states), "sent") + session.get(NotificationEvent, event_id).state = state + session.commit() + return state + + def queue_retry(self, delivery_id, now): + with self.factory() as session: + row = session.get(NotificationDelivery, delivery_id) + if row is None: + return "cancelled" + event = session.get(NotificationEvent, row.event_id) + if row.state in {"sent", "sending", "cancelled_manual"}: + return row.state + valid = self._validate(event, now) + if valid != "valid": + return self._invalidate(event.id, valid, now) + with self.lock, self.factory() as session: + row = session.get(NotificationDelivery, delivery_id) + if row.state not in {"sent", "sending", "cancelled_manual"}: + row.state, row.not_before = "retry_requested", None + session.commit() + return "pending" + + def retry(self, delivery_id, now): + status = self.queue_retry(delivery_id, now) + if status != "pending": + return status + with self.factory() as session: + event_id = session.get(NotificationDelivery, delivery_id).event_id + return self.deliver(event_id, now) + + def recover(self, now, *, startup=False): + with self.lock, self.factory() as session: + rows = session.query(NotificationDelivery).filter_by(state="sending") + if not startup: + rows = rows.filter(NotificationDelivery.lease_until <= naive(now)) + for row in rows: + row.state, row.lease_id = "uncertain", None + event = session.get(NotificationEvent, row.event_id) + self._history(session, event, row, "uncertain", "进程中断,投递结果未知;不会自动重发") + session.commit() diff --git a/goodprice/execution/limits.py b/goodprice/execution/limits.py new file mode 100644 index 0000000..f7ca703 --- /dev/null +++ b/goodprice/execution/limits.py @@ -0,0 +1,51 @@ +import threading +from dataclasses import dataclass + + +@dataclass(frozen=True) +class ResourceLimits: + active_runs: int = 2 + crawler_workers: int = 1 + model_workers: int = 1 + media_workers: int = 1 + platform_interval_seconds: int = 10 + model_rpm: int = 10 + global_items: int = 200 + task_items: int = 60 + run_details: int = 30 + run_http_attempts: int = 60 + run_minutes: int = 15 + + def __post_init__(self): + bounds = {"active_runs": (1, 4), "crawler_workers": (1, 2), "model_workers": (1, 8), + "media_workers": (1, 2), "platform_interval_seconds": (5, 300), "model_rpm": (1, 60), + "global_items": (1, 200), "task_items": (1, 60), "run_details": (1, 100), + "run_http_attempts": (1, 200), "run_minutes": (5, 60)} + for field, (minimum, maximum) in bounds.items(): + if not minimum <= getattr(self, field) <= maximum: + raise ValueError(f"{field} must be {minimum}..{maximum}") + + @classmethod + def from_settings(cls, settings): + return cls(**{name: getattr(settings, f"execution_{name}") for name in cls.__dataclass_fields__}) + + +class RateGate: + def __init__(self, interval_seconds): + self.interval = interval_seconds + self._next = {} + self._lock = threading.Lock() + + def ready_at(self, key, now): + with self._lock: + return max(now, self._next.get(key, now)) + + def reserve(self, key, now): + with self._lock: + allowed = max(now, self._next.get(key, now)) + self._next[key] = allowed + self.interval + return allowed + + def postpone(self, key, until): + with self._lock: + self._next[key] = max(until, self._next.get(key, until)) diff --git a/goodprice/execution/media_registry.py b/goodprice/execution/media_registry.py new file mode 100644 index 0000000..20926e1 --- /dev/null +++ b/goodprice/execution/media_registry.py @@ -0,0 +1,93 @@ +"""In-flight file references; no database objects or browser objects cross lanes.""" +import hashlib +import threading +from contextlib import contextmanager + +from goodprice.media.budget import MediaBudget +from goodprice.media.processing import MediaProcessor + + +class MediaRegistry: + def __init__(self, processor_factory=None): + self._entries = {} + self._lock = threading.RLock() + self._processor_factory = processor_factory + + def has(self, key): + with self._lock: + return key in self._entries + + def restore(self, key, metadata, settings): + from goodprice.media.types import MediaEvidence + budget = MediaBudget(settings.data_root / "media-cache", settings.media_cache_mib * 1024**2, + settings.media_retention_days * 86400) + evidence = [] + try: + with budget.state.lock: + for e in metadata["evidence"]: + path = budget.checked(budget.root / e["path"]) + if not path.is_file(): + return False + with path.open("rb") as source: + if hashlib.file_digest(source, "sha256").hexdigest() != e["hash"]: + return False + evidence.append(MediaEvidence(e["id"], e["kind"], e["hash"], path, e["offer_key"], + e["video_id"], e["timestamp_ms"])) + pin = budget.pin([e.local_path for e in evidence]) + pin.__enter__() + except (ValueError, OSError): + return False + with self._lock: + if key in self._entries: + pin.__exit__(None, None, None) + else: + self._entries[key] = (pin, budget, tuple(evidence), tuple(metadata.get("limitations", []))) + return True + + def prepare(self, key, candidates, settings): + with self._lock: + if key in self._entries: + return self._metadata(self._entries[key]) + processor = (self._processor_factory(settings) if self._processor_factory else + MediaProcessor(MediaBudget(settings.data_root / "media-cache", settings.media_cache_mib * 1024**2, + settings.media_retention_days * 86400))) + context = processor.session(candidates, settings) + evidence, limitations = context.__enter__() + entry = (context, processor.budget, tuple(evidence), tuple(limitations)) + with self._lock: + if key in self._entries: + context.__exit__(None, None, None) + else: + self._entries[key] = entry + return self._metadata(self._entries[key]) + + def _metadata(self, entry): + _, budget, evidence, limitations = entry + return {"evidence": [{"id": e.id, "kind": e.kind, "hash": e.content_hash, + "path": e.local_path.relative_to(budget.root).as_posix(), "offer_key": e.offer_key, + "video_id": e.video_id, "timestamp_ms": e.timestamp_ms} for e in evidence], + "limitations": list(limitations)} + + @contextmanager + def pin(self, key): + with self._lock: + entry = self._entries.get(key) + if entry is None: + raise LookupError("媒体证据需恢复") + _, budget, evidence, limitations = entry + pin = budget.pin([e.local_path for e in evidence]) + pin.__enter__() + try: + yield evidence, limitations + finally: + pin.__exit__(None, None, None) + + def retain(self, needed): + with self._lock: + unused = [key for key in self._entries if key not in needed] + for key in unused: + context, *_ = self._entries.pop(key) + context.__exit__(None, None, None) + + def close(self): + self.retain(set()) diff --git a/goodprice/execution/model_provider.py b/goodprice/execution/model_provider.py new file mode 100644 index 0000000..18350e8 --- /dev/null +++ b/goodprice/execution/model_provider.py @@ -0,0 +1,58 @@ +"""Each provider operation performs at most one HTTP attempt.""" +import json + +from goodprice.analysis.jev_typesafe import TypeSafeJudger +from goodprice.analysis.judge import JevJudger +from goodprice.analysis.llm import LLMClient, parse_batch_value_json +from goodprice.analysis.media_analysis import analyze_media + + +class ModelProvider: + def _client(self, settings, vision=False): + prefix = "vision" if vision else "llm" + return LLMClient(getattr(settings, f"{prefix}_base_url"), getattr(settings, f"{prefix}_api_key"), + getattr(settings, f"{prefix}_model"), api_format=getattr(settings, f"{prefix}_api_format"), + max_attempts=1, retry_delay=0, allow_image_fallback=False) + + def _judger(self, settings): + if not settings.jev_enabled: + return None + if settings.jev_backend == "typesafe": + return TypeSafeJudger(settings.jev_api_key, settings.jev_auto_threshold, single_attempt=True) + return JevJudger(self._client(settings), settings.jev_auto_threshold) + + def enabled(self, role, settings): + if role == "visual": + return settings.vision_enabled and self._client(settings, True).enabled + if role.endswith("_fallback"): + return self._client(settings).enabled + judger = self._judger(settings) + return self._client(settings).enabled or bool(judger and judger.enabled) + + def requirement(self, data, settings, fallback=False): + judger = None if fallback else self._judger(settings) + if judger and judger.enabled: + verdict = judger.analyze_requirement(data["title"], data["description"], data["requirement"]) + if verdict.confidence >= settings.jev_auto_threshold: + return {"matched": verdict.matched, "reason": verdict.reason} + return {"fallback": True} + return self._client(settings).analyze_requirement(data["title"], data["description"], data["requirement"]) + + def visual(self, candidate, selection, data, settings): + return analyze_media(candidate, selection, self._client(settings, True), data["requirement"], + settings.media_request_max_mib * 1024**2, data["description"]) + + def value(self, items, settings, fallback=False): + judger = None if fallback else self._judger(settings) + if judger and judger.enabled: + if len(items) != 1: + raise ValueError("类型化单件判断必须逐请求调度") + result = judger.analyze_value(items[0]) + if result["probability"] < settings.jev_auto_threshold: + return {"fallback": True} + key = items[0]["external_id"] + return {"scores": {key: result["value_score"]}, "best": key, "reasons": {key: result["reason"]}} + from goodprice.analysis.prompts import INDEPENDENT_VALUE_SYSTEM + client = self._client(settings) + return client.complete(client.payload([{"type": "text", "text": json.dumps(items, ensure_ascii=False)}], + system=INDEPENDENT_VALUE_SYSTEM), parser=parse_batch_value_json) diff --git a/goodprice/execution/pipeline.py b/goodprice/execution/pipeline.py new file mode 100644 index 0000000..3e8e8d7 --- /dev/null +++ b/goodprice/execution/pipeline.py @@ -0,0 +1,882 @@ +"""Production stage graph. Workers return data; the coordinator applies writes.""" +import hashlib +import json +import time +from dataclasses import asdict +from datetime import datetime, timedelta, timezone +from types import SimpleNamespace + +from goodprice.analysis.media_analysis import analysis_key +from goodprice.crawler.base import ListingData +from goodprice.crawler.xianyu import XianyuAdapter +from goodprice.execution.browser import current_browser +from goodprice.execution.coordinator import Coordinator +from goodprice.execution.delivery import DeliveryService +from goodprice.execution.limits import ResourceLimits +from goodprice.execution.media_registry import MediaRegistry +from goodprice.execution.model_provider import ModelProvider +from goodprice.execution.protection import ResourceGuard +from goodprice.execution.store import ExecutionStore, naive +from goodprice.execution.types import Stage, WorkResult, thaw +from goodprice.execution.workers import WorkerPools +from goodprice.media.selection import select_media +from goodprice.media.types import MediaCandidate, MediaEvidence, MediaSelection +from goodprice.models import ( + AnalysisCache, + Listing, + ListingOffer, + ManualQuote, + MediaAsset, + NotificationDelivery, + OfferEvaluationRecord, + OfferObservation, + Seller, + TaskRun, + WatchTask, + WorkItem, +) +from goodprice.offers.evaluation import assess_offer +from goodprice.offers.pricing import decide_price, utc +from goodprice.offers.service import OfferService, decode_offer, encode_offer +from goodprice.offers.types import OfferCandidate, PriceClaim, QuoteInput +from goodprice.services.crawl_service import CrawlService +from goodprice.services.offer_pipeline import OfferPipeline, StaleEvaluation +from goodprice.services.satisfaction import compute_satisfaction + +PENDING = {"requirement", "media", "visual", "value_pending", "value_running", "aggregate", "refreshing"} + + +def digest(value): + return hashlib.sha256(json.dumps(value, sort_keys=True, default=str).encode()).hexdigest() + + +def node(stage, identity, payload): + return {"stage": stage.value, "fingerprint": digest(identity), "payload": payload} + + +class Pipeline: + def __init__(self, factory, settings, adapter_factory=None, provider=None, + notifier_factory=None, processor_factory=None, clock=time.time): + self.factory, self.settings, self.clock = factory, settings, clock + self.adapter_factory = adapter_factory + self.provider = provider or ModelProvider() + self.registry = MediaRegistry(processor_factory) + self.notifier_factory = notifier_factory or (lambda: {}) + self.delivery = DeliveryService(factory, self.notifier_factory, + vision_enabled=lambda: self.provider.enabled("visual", self.settings.get())) + self.upserter = CrawlService(factory, None, None, None, [], settings) + self.store = None + self.guard = ResourceGuard() + + def now(self): + return datetime.fromtimestamp(self.clock(), timezone.utc) + + def config_key(self, runtime): + from goodprice.analysis import prompts + values = runtime.model_dump(mode="json") + return digest([{k: v for k, v in values.items() if k.startswith(("llm_", "jev_"))}, + {k: v for k, v in vars(prompts).items() if k.isupper() and isinstance(v, str)}, "typed-contract-1"]) + + def model_config(self, role, runtime): + if role == "visual": + from goodprice.analysis.media_analysis import PROMPT_VERSION, SYSTEM + fields = {"vision_enabled", "vision_base_url", "vision_api_key", "vision_model", "vision_api_format", + "video_analysis_enabled", "media_image_count", "media_video_count", "media_frames_per_video", + "media_image_quota", "media_frame_quota", "media_total_limit", "media_model_limit"} + return digest([{k: v for k, v in runtime.model_dump(mode="json").items() if k in fields}, PROMPT_VERSION, SYSTEM]) + return self.config_key(runtime) + + def provider_gate(self, role, runtime): + from urllib.parse import urlsplit + if role == "visual": + url, key = runtime.vision_base_url, runtime.vision_api_key + elif runtime.jev_enabled and runtime.jev_backend == "typesafe" and not role.endswith("fallback"): + url, key = "https://api.typesafe.ai", runtime.jev_api_key + else: + url, key = runtime.llm_base_url, runtime.llm_api_key + try: + parsed = urlsplit(url) + host = (parsed.scheme, parsed.hostname, parsed.port) + except ValueError: + host = "invalid" + return digest([host, digest(key)]) + + def _adapter(self, runtime): + factory = self.adapter_factory or (lambda: XianyuAdapter(runtime.xianyu_cookie, runtime.proxy).session()) + return current_browser(digest([runtime.xianyu_cookie, runtime.proxy]), factory) + + def _context(self, item): + with self.factory() as session: + task = session.get(WatchTask, item.token.task_id) + if not task or not task.enabled or task.version != item.token.task_version: + raise StaleEvaluation("任务条件已变化") + task_data = {c.name: getattr(task, c.name) for c in WatchTask.__table__.columns} + listing = session.get(Listing, item.payload.get("listing_id")) if item.payload.get("listing_id") else None + if listing is not None and listing.task_id != item.token.task_id: + raise StaleEvaluation("商品引用已不属于当前任务") + listing_data = {c.name: getattr(listing, c.name) for c in Listing.__table__.columns} if listing else None + return task_data, listing_data + + def _owned(self, session, row, run): + if row is None or row.task_version != run.task_version or row.payload.get("run_id") != run.id: + return False + offer = session.get(ListingOffer, row.offer_id) + listing = session.get(Listing, offer.listing_id) if offer else None + return bool(listing and listing.task_id == run.task_id) + + def platform_io(self, item): + if item.stage == Stage.SEARCH: + return not item.payload.get("target_id") + if item.stage == Stage.DETAIL: + with self.factory() as session: + task = session.get(WatchTask, item.token.task_id) + return bool(task and task.fetch_detail) + return False + + def cached(self, item): + if item.stage == Stage.DETAIL and item.payload.get("role") == "seller": + with self.factory() as session: + seller = session.query(Seller).filter_by(platform="xianyu", seller_uid=item.payload["seller_uid"]).first() + if seller and seller.last_fetched_at and self.now() - utc(seller.last_fetched_at) < timedelta(days=7): + return WorkResult(item.token, item.stage, item.input_fingerprint, {"seller_cached": True}) + if item.stage != Stage.ANALYZE: + return None + runtime = self.settings.get() + if item.payload.get("model_cfg") != self.model_config(item.payload["role"], runtime): + return WorkResult(item.token, item.stage, item.input_fingerprint, {"configuration_changed": True, "no_http": True}) + if item.payload.get("force"): + return None + if item.payload["role"] == "value": + scores = self._cached_values(item.payload) + if len(scores) == len(item.payload["items"]): + return WorkResult(item.token, item.stage, item.input_fingerprint, + {"verdict": {"scores": scores, "best": max(scores, key=scores.get)}, "cache_hit": True}) + return None + with self.factory() as session: + cached = session.query(WorkItem).filter(WorkItem.stage == Stage.ANALYZE.value, + WorkItem.state == "done", WorkItem.payload["cache_key"].as_string() == item.payload["cache_key"], + WorkItem.output["verdict"].as_string().is_not(None)).order_by(WorkItem.id.desc()).first() + if cached and "verdict" in (cached.output or {}): + return WorkResult(item.token, item.stage, item.input_fingerprint, + {"verdict": cached.output["verdict"], "cache_hit": True}) + return None + + def _cached_values(self, payload): + if payload.get("force"): + return {} + keys = payload.get("item_cache_keys", {}) + with self.factory() as session: + rows = {r.key: r.payload for r in session.query(AnalysisCache).filter(AnalysisCache.key.in_(list(keys.values())))} + return {i: rows[key]["score"] for i, key in keys.items() if key in rows} + + def worker(self, item): + task, listing = self._context(item) + runtime = self.settings.get() + payload = thaw(item.payload) + if item.stage == Stage.SEARCH: + if payload.get("target_id"): + return {"manual_ids": [payload["target_id"]]} + data = self._adapter(runtime).search(task["keyword"], max_items=200, attempts=1) + return {"items": [{**asdict(row), "published_at": row.published_at.isoformat() if row.published_at else None} + for row in data[:200]], "source_truncated": max(0, len(data) - 200)} + if item.stage == Stage.DETAIL: + if not listing: + return {"missing": True} + if payload.get("role") == "seller": + try: + return {"seller": asdict(self._adapter(runtime).fetch_seller(payload["seller_uid"]))} + except Exception: + return {"seller_unavailable": True} + if not task["fetch_detail"]: + return {"disabled": True} + detail = self._adapter(runtime).fetch_detail(listing["url"]) + media = detail.media or [MediaCandidate(digest(["adapter-image", url]), "image", url, + source_locator="adapter:detail-image") for url in detail.image_urls[:32]] + return {"detail": {"description": detail.description, "image_urls": detail.image_urls, + "variants": detail.variants, "offers": [encode_offer(o) for o in detail.offers], + "media": [asdict(m) for m in media], "seller_uid": detail.seller_uid, + "seller_name": detail.seller_name, "credit_label": detail.credit_label, + "positive_rate": detail.positive_rate, "sold_count": detail.sold_count}} + if item.stage == Stage.MEDIA: + runtime = self.guard.media_settings(runtime) + with self.factory() as session: + sources = session.query(MediaAsset).filter(MediaAsset.listing_id == payload["listing_id"], + MediaAsset.source_url != "", MediaAsset.status != "superseded").order_by(MediaAsset.id).all() + candidates = [MediaCandidate(r.key, r.kind, r.source_url, r.offer_key, r.source_locator, + status="unsupported" if r.status == "unsupported" else "pending") for r in sources] + return self.registry.prepare(payload["group"], candidates, runtime) + if item.stage == Stage.ANALYZE: + if payload.get("model_cfg") != self.model_config(payload["role"], runtime): + return {"configuration_changed": True, "no_http": True} + role = payload["role"] + if role == "value": + cached = self._cached_values(payload) + missing = [r for r in payload["items"] if r["external_id"] not in cached] + verdict = self.provider.value(missing, runtime, payload.get("fallback", False)) if missing else {"scores": {}, "best": None} + if not verdict.get("fallback"): + ids = {row["external_id"] for row in missing} + scores = verdict.get("scores") + if (not isinstance(scores, dict) or set(scores) != ids or + any(type(v) not in (int, float) or not 0 <= v <= 10 for v in scores.values()) or + verdict.get("best") not in ids | {None}): + raise ValueError("模型评分必须覆盖本次商品且只能引用本次商品") + verdict = {**verdict, "scores": {**cached, **scores}} + return {"verdict": verdict} + with self.factory() as session: + row = session.get(OfferEvaluationRecord, payload["eval_id"]) + observation = session.get(OfferObservation, row.payload["observation_id"]) + candidate = decode_offer(observation.payload) + data = {"title": listing["title"], "description": f"{listing['description'] or ''}\n规格:{dict(candidate.attributes)}", + "requirement": task["condition_requirement"]} + if role.startswith("requirement"): + return {"verdict": self.provider.requirement(data, runtime, role.endswith("fallback"))} + with self.registry.pin(payload["group"]): + selection, scoped_candidate = self._selection(payload, candidate, runtime) + data["description"] = f"{listing['title']}\n{listing['description'] or ''}\n规格:{dict(scoped_candidate.attributes)}" + verdict = self.provider.visual(scoped_candidate, selection, data, runtime) + return {"verdict": verdict} + return {} + + def _selection(self, payload, candidate, runtime): + from pathlib import Path + raw = payload["evidence"] + scoped = [e for e in raw if e["offer_key"] == candidate.key] + shared = not scoped and payload.get("multiple", False) + chosen = scoped or [e for e in raw if e["offer_key"] is None] + root = runtime.data_root / "media-cache" + evidence = [MediaEvidence(e["id"], e["kind"], e["hash"], root / Path(e["path"]), + e["offer_key"], e["video_id"], e["timestamp_ms"]) for e in chosen] + selection = select_media([e for e in evidence if e.kind == "image"], + [e for e in evidence if e.kind == "frame"], runtime.media_image_quota, runtime.media_frame_quota, runtime.media_total_limit) + selection = MediaSelection(selection.items, selection.omitted_ids, + tuple(dict.fromkeys(selection.limitations + tuple(payload.get("limitations", []))))) + return selection, OfferCandidate("shared", {}) if shared else candidate + + def apply(self, result, session): + work = session.query(WorkItem).filter_by(run_id=result.token.run_id, lease_id=result.token.lease_id).one() + run = session.get(TaskRun, result.token.run_id) + task = session.get(WatchTask, run.task_id) + runtime = self.settings.get() + data, payload = thaw(result.output), work.payload + if data.get("configuration_changed") or (result.stage == Stage.ANALYZE and + payload.get("model_cfg") != self.model_config(payload["role"], runtime)): + run.cursor = {**(run.cursor or {}), "_draining": True, "_drain_reason": "configuration"} + session.query(WorkItem).filter(WorkItem.run_id == run.id, WorkItem.state.in_(("pending", "retry")), + WorkItem.stage.in_(("search", "detail", "media", "analyze"))).update({"state": "deferred"}) + return {} + if payload.get("listing_id"): + listing = session.get(Listing, payload["listing_id"]) + if listing is not None and listing.task_id != run.task_id: + return {} + if result.stage == Stage.SEARCH: + return self._search(session, run, task, data, runtime) + if result.stage == Stage.DETAIL: + return self._detail(session, run, task, payload, data) + if result.stage == Stage.PARSE: + return self._parse(session, run, task, payload, runtime) + if result.stage == Stage.MEDIA: + return self._media(session, run, payload, data, runtime, result.input_fingerprint) + if result.stage == Stage.ANALYZE: + return self._analyzed(session, run, task, payload, data["verdict"], runtime) + if result.stage == Stage.AGGREGATE: + return self._aggregate(session, run, task, payload, runtime) + return {} + + def _search(self, session, run, task, data, runtime): + seen = [] + for item in data.get("items", []): + if self.upserter._excluded(task, item["title"]): + continue + item = dict(item) + item["published_at"] = datetime.fromisoformat(item["published_at"]) if item.get("published_at") else None + listing, _ = self.upserter._upsert_listing(session, task, ListingData(**item)) + listing.status, listing.missed_count = "active", 0 + seen.append(listing.id) + priorities = data.get("manual_ids") or list((run.cursor or {}).get("remaining_ids", [])) + ids = list(dict.fromkeys(priorities + seen))[:200] + ids = [i for i in ids if (session.get(Listing, i) is not None and session.get(Listing, i).task_id == task.id)] + run.cursor = {**(run.cursor or {}), "remaining_ids": ids, "seen_ids": seen, + "source_truncated": data.get("source_truncated", 0), "planned_ids": ids[:runtime.execution_run_details]} + task.last_run_at, task.last_run_count, task.last_error = naive(self.now()), (task.last_run_count or 0) + 1, None + if run.kind == "crawl": + for listing in session.query(Listing).filter_by(task_id=task.id): + if listing.id not in seen: + listing.missed_count += 1 + if listing.missed_count >= 3: + listing.status = "unseen" + return {"next": [node(Stage.DETAIL, ["detail", i], {"listing_id": i}) for i in ids[:runtime.execution_run_details]]} + + def _detail(self, session, run, task, payload, data): + listing = session.get(Listing, payload["listing_id"]) + if listing is None or data.get("missing"): + self._listing_done(session, run, payload["listing_id"]) + return {} + if payload.get("role") == "seller": + from goodprice.services.seller_service import compute_risk + seller = session.query(Seller).filter_by(platform=task.platform, seller_uid=payload["seller_uid"]).first() + captured = data.get("seller") + if captured: + if seller is None: + seller = Seller(platform=task.platform, seller_uid=payload["seller_uid"]) + session.add(seller) + seller.nickname, seller.tags = captured["nickname"], captured["tags"] + seller.positive_count, seller.total_count = captured["positive_count"], captured["total_count"] + seller.positive_rate = captured["positive_count"] / captured["total_count"] if captured["positive_count"] is not None and captured["total_count"] else None + seller.last_fetched_at = naive(self.now()) + risk = listing.seller_risk or {} + if seller and seller.blocked: + listing.blocked = True + level, reason = compute_risk(seller, risk.get("credit_label"), risk.get("positive_rate")) + listing.seller_risk = {**risk, "risk_level": level, "reason": reason} + return {"next": [node(Stage.PARSE, ["parse", listing.id, payload["eval_ids"]], + {"listing_id": listing.id, "eval_ids": payload["eval_ids"]})]} + detail = data.get("detail", {}) + if payload.get("price_refresh") and detail: + candidates = [decode_offer(o) for o in detail.get("offers", [])] + if candidates and self._source_key(session, listing, detail) == payload["source_key"]: + for candidate in candidates: + OfferService(self.factory).record_observation(listing.id, candidate, session=session, now=self.now()) + refreshed = [] + for i in payload["eval_ids"]: + row = session.get(OfferEvaluationRecord, i) + observation = session.query(OfferObservation).filter_by(offer_id=row.offer_id).order_by(OfferObservation.id.desc()).first() + row.payload = {**row.payload, "observation_id": observation.id, "status": "parse"} + row.payload = {**row.payload, **self._price_context(session, row)} + refreshed.append(observation.id) + return {"next": [node(Stage.PARSE, ["price_refreshed", payload["eval_ids"], refreshed], + {"listing_id": listing.id, "eval_ids": payload["eval_ids"], "reuse_inputs": True})]} + for i in payload["eval_ids"]: + row = session.get(OfferEvaluationRecord, i) + if row: + row.payload = {**row.payload, "status": "superseded", "eligible": False} + if detail: + listing.offer_verified_until, listing.value_score, listing.best_of_batch = None, None, False + listing.satisfaction = 0 + listing.description, listing.image_urls = detail["description"], detail["image_urls"][:32] + listing.variants = detail["variants"] + listing.seller_uid, listing.seller_name = detail["seller_uid"], detail["seller_name"] + listing.seller_risk = {"credit_label": detail["credit_label"], "positive_rate": detail["positive_rate"], "sold_count": detail["sold_count"]} + from goodprice.services.seller_service import compute_risk + level, reason = compute_risk(None, detail["credit_label"], detail["positive_rate"]) + listing.seller_risk = {**listing.seller_risk, "risk_level": level, "reason": reason} + listing.status, listing.missed_count = "active", 0 + sources = {r.key: r for r in session.query(MediaAsset).filter(MediaAsset.listing_id == listing.id, MediaAsset.source_url != "")} + for old in sources.values(): + old.status = "superseded" + for candidate in detail["media"]: + row = sources.get(candidate["key"]) + if row is None: + row = MediaAsset(listing_id=listing.id, key=candidate["key"], kind=candidate["kind"]) + session.add(row) + row.source_url, row.source_locator, row.offer_key = candidate["url"], candidate["source_locator"], candidate["offer_key"] + row.status, row.relative_path, row.content_hash = candidate["status"], None, None + candidates = [decode_offer(o) for o in detail.get("offers", [])] + if not candidates: + candidates = [OfferCandidate("default", {}, claims=(PriceClaim("dom", "unknown", + "详情未核验" if not data.get("disabled") else "详情抓取关闭", self.now(), "detail:missing"),))] + ids = OfferPipeline(self.factory, self.provider.enabled("visual", self.settings.get())).evaluate_listing( + listing.id, task.id, candidates, self.now(), session=session, expected_version=task.version, + ready=False, per_offer_results={}) + included = {session.get(OfferEvaluationRecord, i).offer_id for i in ids} + for old in session.query(ListingOffer).filter_by(listing_id=listing.id): + if old.id not in included: + old.seen_in_latest_detail = False + old.effective_price_state, old.effective_price_cents, old.price_valid_until = "unknown", None, None + group = digest([run.id, listing.id, ids]) + for i in ids: + row = session.get(OfferEvaluationRecord, i) + row.payload = {**row.payload, "run_id": run.id, "group": group, "multiple": len(ids) > 1, "status": "parse"} + if listing.seller_uid and detail: + return {"next": [node(Stage.DETAIL, ["seller", listing.id, ids], + {"role": "seller", "listing_id": listing.id, "seller_uid": listing.seller_uid, "eval_ids": ids})]} + return {"next": [node(Stage.PARSE, ["parse", listing.id, ids], {"listing_id": listing.id, "eval_ids": ids})]} + + def _parse(self, session, run, task, payload, runtime): + following = [] + for i in payload["eval_ids"]: + row = session.get(OfferEvaluationRecord, i) + offer = session.get(ListingOffer, row.offer_id) + base = {"listing_id": payload["listing_id"], "eval_id": i, "group": row.payload["group"], "force": run.kind == "reanalyze"} + gates = self._assessment(session, row, runtime) + if gates["state"] == "exact" and not gates["price_eligible"]: + row.payload = {**row.payload, "status": "aggregate"} + following.append(node(Stage.AGGREGATE, ["aggregate", i, row.payload["observation_id"]], base)) + continue + if payload.get("reuse_inputs") and row.payload.get("requirement_done"): + following.extend(self._after_requirement(session, row, base, runtime)) + continue + if row.payload["match"] != "matched" or not task.fetch_detail: + row.payload = {**row.payload, "status": "aggregate", "requirement_match": None} + following.append(node(Stage.AGGREGATE, ["aggregate", i], base)) + elif task.condition_requirement and self.provider.enabled("requirement", runtime): + listing = session.get(Listing, payload["listing_id"]) + key = digest(["requirement", listing.id, offer.offer_key, listing.title, listing.description, + task.condition_requirement, task.target_variant, self.config_key(runtime)]) + row.payload = {**row.payload, "status": "requirement"} + following.append(node(Stage.ANALYZE, ["requirement", i, row.payload["observation_id"], key], {**base, "role": "requirement", "cache_key": key, "http": True, + "model_cfg": self.model_config("requirement", runtime), "provider_gate": self.provider_gate("requirement", runtime)})) + else: + row.payload = {**row.payload, "requirement_match": None, "requirement_done": True} + following.extend(self._after_requirement(session, row, base, runtime)) + return {"next": following} + + def _after_requirement(self, session, row, base, runtime): + if row.payload.get("media_done"): + return self._ready_for_value(session, row, base, runtime) + if row.payload.get("requirement_match") is False or not self.provider.enabled("visual", runtime): + row.payload = {**row.payload, "condition_score": None, "visual_scope": None} + return self._ready_for_value(session, row, base, runtime) + row.payload = {**row.payload, "status": "media"} + return [node(Stage.MEDIA, ["media", row.id, row.payload["observation_id"]], {**base, "role": "media"})] + + def _media(self, session, run, payload, data, runtime, generation): + row = session.get(OfferEvaluationRecord, payload["eval_id"]) + if not self._owned(session, row, run) or row.payload.get("status") in {"completed", "superseded", "refreshing"}: + return {} + assets = {r.key: r for r in session.query(MediaAsset).filter_by(listing_id=payload["listing_id"])} + for e in data["evidence"]: + asset_key = f"evidence:{e['id']}" + asset = assets.get(asset_key) + if asset is None: + asset = MediaAsset(listing_id=payload["listing_id"], key=asset_key, kind=e["kind"]) + session.add(asset) + assets[asset_key] = asset + asset.offer_key, asset.content_hash, asset.relative_path = e["offer_key"], e["hash"], e["path"] + asset.parent_video_key, asset.timestamp_ms, asset.status = e["video_id"], e["timestamp_ms"], "ready" + base = {**payload, **data, "role": "visual", "multiple": row.payload["multiple"]} + offer = decode_offer(session.get(OfferObservation, row.payload["observation_id"]).payload) + selection, scoped = self._selection(base, offer, runtime) + if not selection.items: + row.payload = {**row.payload, "condition_score": None, "visual_scope": None, + "media": {"limitations": list(selection.limitations) + ["no_visual_input"]}} + return {"next": self._ready_for_value(session, row, base, runtime)} + listing = session.get(Listing, payload["listing_id"]) + client_identity = SimpleNamespace(model=runtime.vision_model, base_url=runtime.vision_base_url, api_format=runtime.vision_api_format) + seller_text = f"{listing.title}\n{listing.description or ''}\n规格:{dict(scoped.attributes)}" + key = digest([listing.id, digest(runtime.vision_api_key), analysis_key(scoped, selection, client_identity, + session.get(WatchTask, run.task_id).condition_requirement, runtime, seller_text)]) + row.payload = {**row.payload, "status": "visual", "media_cache_key": key} + return {"next": [node(Stage.ANALYZE, ["visual", row.id, key, generation], {**base, "cache_key": key, "http": True, + "model_cfg": self.model_config("visual", runtime), "provider_gate": self.provider_gate("visual", runtime)})]} + + def _analyzed(self, session, run, task, payload, verdict, runtime): + if payload["role"] == "value": + if verdict.get("fallback"): + if self.provider.enabled("value_fallback", runtime): + key = digest([payload["cache_key"], "fallback"]) + return {"next": [node(Stage.ANALYZE, ["value_fallback", payload["eval_ids"], key], + {**payload, "cache_key": key, "fallback": True, + "provider_gate": self.provider_gate("value_fallback", runtime)})]} + for i in payload["eval_ids"]: + row = session.get(OfferEvaluationRecord, i) + row.payload = {**row.payload, "status": "unverified", "eligible": False, + "reason": "评分置信度不足,文本回落模型尚未配置"} + return {} + following = [] + for item_id, score in verdict["scores"].items(): + key = payload["item_cache_keys"][item_id] + cached = session.get(AnalysisCache, key) + if cached is None: + session.add(AnalysisCache(key=key, payload={"score": score})) + else: + cached.payload = {"score": score} + for i in payload["eval_ids"]: + row = session.get(OfferEvaluationRecord, i) + if not self._owned(session, row, run) or row.payload.get("status") in {"completed", "superseded", "refreshing"}: + continue + if self._expired(session, row): + following.extend(self._refresh_price(session, run, payload["listing_ids"][str(i)])) + continue + current = self._price_context(session, row) + if current != payload.get("price_contexts", {}).get(str(i)): + row.payload = {**row.payload, **current, "value_score": None, + "status": "value_pending", "value_ready_at": self.now().isoformat()} + continue + value = verdict.get("scores", {}).get(f"offer:{row.offer_id}") + row.payload = {**row.payload, "value_score": value, "value_price_context": current, "status": "aggregate"} + following.append(node(Stage.AGGREGATE, ["aggregate", i, payload["cache_key"]], {"listing_id": payload["listing_ids"][str(i)], "eval_id": i})) + return {"next": following} + row = session.get(OfferEvaluationRecord, payload["eval_id"]) + if not self._owned(session, row, run) or row.payload.get("status") in {"completed", "superseded", "refreshing"}: + return {} + if payload["role"].startswith("requirement"): + if verdict.get("fallback"): + role = "requirement_fallback" + if not self.provider.enabled(role, runtime): + row.payload = {**row.payload, "status": "unverified", "eligible": False, + "reason": "判断置信度不足,文本回落模型尚未配置"} + return {} + key = digest([payload["cache_key"], role]) + return {"next": [node(Stage.ANALYZE, [role, row.id, key], {**payload, "role": role, "cache_key": key, + "model_cfg": self.model_config(role, runtime), "provider_gate": self.provider_gate(role, runtime)})]} + row.payload = {**row.payload, "requirement_match": verdict.get("matched"), "requirement_reason": verdict.get("reason", ""), "requirement_done": True} + return {"next": self._after_requirement(session, row, payload, runtime)} + offer = session.get(ListingOffer, row.offer_id) + from goodprice.analysis.media_analysis import validate_evidence_references + candidate = decode_offer(session.get(OfferObservation, row.payload["observation_id"]).payload) + selection, scoped = self._selection(payload, candidate, runtime) + clean = validate_evidence_references(verdict, {e.id for e in selection.items}) + limitations = list(dict.fromkeys(list(selection.limitations) + list(verdict.get("limitations", [])))) + if scoped.key == "shared": + clean["condition_score"] = None + limitations.append("shared_media") + if clean.get("conflicts"): + clean["condition_score"] = None + limitations.append("conflicting_evidence") + clean.update(limitations=list(dict.fromkeys(limitations)), evidence_ids=[e.id for e in selection.items], + media_cache_key=payload["cache_key"], omitted_evidence_ids=list(selection.omitted_ids)) + row.payload = {**row.payload, "media": clean, "condition_score": clean.get("condition_score"), + "visual_scope": offer.offer_key if clean.get("condition_score") is not None else None, "media_done": True} + return {"next": self._ready_for_value(session, row, payload, runtime)} + + def _ready_for_value(self, session, row, payload, runtime): + if not self._assessment(session, row, runtime)["eligible"] or not self.provider.enabled("value", runtime): + row.payload = {**row.payload, "value_score": None, "status": "aggregate"} + return [node(Stage.AGGREGATE, ["aggregate", row.id], {"listing_id": payload["listing_id"], "eval_id": row.id})] + row.payload = {**row.payload, "status": "value_pending", "value_ready_at": self.now().isoformat()} + return [] + + def _aggregate(self, session, run, task, payload, runtime): + original = session.get(OfferEvaluationRecord, payload["eval_id"]) + if not self._owned(session, original, run) or original.payload.get("status") == "completed": + return {} + if (run.cursor or {}).get("_drain_reason") == "configuration": + original.payload = {**original.payload, "status": "deferred", "eligible": False} + return {} + if self._expired(session, original): + return {"next": self._refresh_price(session, run, payload["listing_id"])} + current_price = self._price_context(session, original) + if original.payload.get("value_score") is not None and original.payload.get("value_price_context") != current_price: + original.payload = {**original.payload, **current_price, "value_score": None, + "status": "value_pending", "value_ready_at": self.now().isoformat()} + return {} + listing = session.get(Listing, payload["listing_id"]) + offer = session.get(ListingOffer, original.offer_id) + if not offer.seen_in_latest_detail: + original.payload = {**original.payload, "status": "superseded", "eligible": False} + self._listing_done(session, run, listing.id) + return {} + if listing is None or offer is None: + return {} + observation = session.query(OfferObservation).filter_by(offer_id=offer.id).order_by(OfferObservation.id.desc()).first() + candidate = decode_offer(observation.payload) + values = dict(original.payload) + ids = OfferPipeline(self.factory, self.provider.enabled("visual", runtime)).evaluate_listing( + listing.id, task.id, [candidate], self.now(), session=session, expected_version=task.version, + per_offer_results={candidate.key: values}, record_observations=False) + final = session.get(OfferEvaluationRecord, ids[0]) + final.payload = {**final.payload, "run_id": run.id, "status": "ready", "media": values.get("media"), + "value_score": values.get("value_score"), "media_cache_key": values.get("media_cache_key")} + original.payload = {**original.payload, "status": "completed", "final_id": final.id} + if final.payload["eligible"]: + score = compute_satisfaction(SimpleNamespace(requirement_match=values.get("requirement_match"), + condition_score=values.get("condition_score"), value_score=values.get("value_score"), seller_risk=listing.seller_risk), + vision_enabled=self.provider.enabled("visual", runtime)) + final.payload = {**final.payload, "satisfaction": score} + if listing.offer_task_version != task.version or listing.offer_price_cents is None or score >= listing.satisfaction: + listing.condition_score, listing.condition_detail = values.get("condition_score"), values.get("media") + listing.requirement_match, listing.requirement_reason = values.get("requirement_match"), values.get("requirement_reason") + listing.satisfaction, listing.value_score = score, values.get("value_score") + listing.offer_price_cents, listing.offer_task_version = final.payload["amount_cents"], task.version + listing.offer_verified_until = datetime.fromisoformat(final.payload["fresh_until"]) + session.flush() + self.rank_ready(session, run) + if run.kind != "reanalyze": + self._event(session, task, listing, offer, final, score, values) + elif (run.kind != "reanalyze" and task.notify_pending_price and task.fetch_detail + and final.payload["state"] != "exact" and final.payload["match"] != "mismatched" + and offer.availability != "unavailable" and not listing.blocked and self.notifier_factory()): + conditions = digest({name: getattr(task, name) for name in ("keyword", "target_variant", "condition_requirement", "min_condition_score", "min_price", "max_price")}) + key = digest(["pending", task.id, listing.platform, listing.external_id, offer.offer_key, + conditions, offer.id, final.payload["fingerprint"]]) + self.delivery.enqueue(task.id, task.version, listing.id, offer.id, key, + {"title": f"[待询价] {listing.title}", "content": "当前售价尚未核实,不参与正常性价比排名。请自行联系卖家核实具体规格和实际价格。", + "url": listing.url, "pending": True, "fingerprint": final.payload["fingerprint"], + "source": None, "amount_cents": None, "condition_score": values.get("condition_score"), + "media_cache_key": values.get("media_cache_key"), "offer_evaluation": True}, session=session) + self._listing_done(session, run, listing.id) + return {} + + def _event(self, session, task, listing, offer, final, score, values): + if not self.notifier_factory(): + return + price = final.payload + if offer.last_notified_source == "legacy_baseline": + offer.last_notified_score, offer.last_notified_amount = score, price["amount_cents"] + offer.last_notified_source, offer.last_notified_task_version = price["source"], task.version + return + changed_source = offer.last_notified_source != price["source"] + reduced = (not changed_source and offer.last_notified_amount is not None and price["amount_cents"] < offer.last_notified_amount) + if offer.last_notified_task_version in {None, task.version} and offer.last_notified_score is not None and score <= offer.last_notified_score and not changed_source and not reduced: + return + conditions = digest({name: getattr(task, name) for name in ("keyword", "platform", "target_variant", "min_price", "max_price", "exclude_words", "condition_requirement", "min_condition_score")}) + key = digest([task.id, listing.platform, listing.external_id, offer.offer_key, + conditions, offer.id, price["fingerprint"], price["source"], price["amount_cents"], score, values.get("media_cache_key")]) + self.delivery.enqueue(task.id, task.version, listing.id, offer.id, key, + {"title": f"[{task.keyword}] {listing.title}", "content": f"规格:{offer.name or dict(offer.attributes) or '默认商品'}\n" + f"价格:{price['amount_cents'] / 100:g}元({'人工报价' if price['source'] == 'manual' else '页面确价'})\n" + f"品相:{values.get('condition_score') if values.get('condition_score') is not None else '未核验'}\n" + f"性价比:{values.get('value_score') if values.get('value_score') is not None else '未评估'}", + "url": listing.url, "fingerprint": price["fingerprint"], "source": price["source"], + "amount_cents": price["amount_cents"], "condition_score": values.get("condition_score"), + "media_cache_key": values.get("media_cache_key"), "satisfaction": score, "offer_evaluation": True}, session=session) + + def rank_ready(self, session, run): + rows = session.query(OfferEvaluationRecord).filter(OfferEvaluationRecord.payload["run_id"].as_integer() == run.id, + OfferEvaluationRecord.payload["status"].as_string() == "ready").all() + offer_ids = [r.offer_id for r in rows] + offers = {o.id: o for o in session.query(ListingOffer).filter(ListingOffer.id.in_(offer_ids))} + ready = [r for r in rows if r.payload.get("eligible") and r.offer_id in offers and + offers[r.offer_id].seen_in_latest_detail and offers[r.offer_id].price_valid_until is not None and + offers[r.offer_id].price_valid_until > naive(self.now()) and + offers[r.offer_id].price_task_version == run.task_version and + r.payload.get("amount_cents") == offers[r.offer_id].effective_price_cents and + r.payload.get("source") == offers[r.offer_id].effective_price_source and + r.payload.get("fingerprint") == offers[r.offer_id].price_evidence_fingerprint] + valued = [r for r in ready if r.payload.get("value_score") is not None] + best = max(valued, key=lambda r: (r.payload.get("satisfaction", 0), -r.payload["amount_cents"], -r.offer_id)) if len(valued) > 1 else None + listings = {p.id: p for p in session.query(Listing).filter(Listing.id.in_({o.listing_id for o in offers.values()}))} + for row in ready: + row.payload = {**row.payload, "best_of_batch": row is best, "batch_scope_count": len(valued)} + for listing in listings.values(): + current = [r for r in ready if offers[r.offer_id].listing_id == listing.id] + if not current: + listing.offer_verified_until, listing.value_score, listing.best_of_batch = None, None, False + listing.satisfaction = 0 + continue + chosen = max(current, key=lambda r: r.payload.get("satisfaction", 0)) + listing.satisfaction, listing.value_score = chosen.payload.get("satisfaction", 0), chosen.payload.get("value_score") + listing.offer_price_cents = min(r.payload["amount_cents"] for r in current) + listing.offer_verified_until = min(datetime.fromisoformat(r.payload["fresh_until"]) for r in current) + listing.offer_task_version = run.task_version + listing.best_of_batch = bool(best and offers[best.offer_id].listing_id == listing.id) + listing.value_batch_at = naive(self.now()) + + def _listing_done(self, session, run, listing_id): + rows = session.query(OfferEvaluationRecord).join(ListingOffer).filter(ListingOffer.listing_id == listing_id, + OfferEvaluationRecord.payload["run_id"].as_integer() == run.id).all() + if any((row.payload or {}).get("status") in PENDING | {"parse"} for row in rows): + return + run.cursor = {**(run.cursor or {}), "remaining_ids": [i for i in (run.cursor or {}).get("remaining_ids", []) if i != listing_id]} + + def can_finish(self, session, run): + return session.query(OfferEvaluationRecord.id).filter(OfferEvaluationRecord.payload["run_id"].as_integer() == run.id, + OfferEvaluationRecord.payload["status"].as_string().in_(PENDING | {"parse"})).first() is None + + def maintain(self, store, now): + self._restore_media(store) + with store.lock, self.factory() as session: + runs = session.query(TaskRun).filter(TaskRun.state.in_(("active", "waiting_capacity"))).all() + needed = set() + for run in runs: + rows = session.query(OfferEvaluationRecord).filter(OfferEvaluationRecord.payload["run_id"].as_integer() == run.id).all() + if (run.cursor or {}).get("_draining"): + running = session.query(WorkItem).filter_by(run_id=run.id, state="running").all() + active_ids = {i for work in running for i in (work.payload.get("eval_ids") or [work.payload.get("eval_id")]) if i} + for row in rows: + if row.id not in active_ids and row.payload.get("status") in {"requirement", "media", "visual", "value_pending", "value_running"}: + row.payload = {**row.payload, "status": "deferred", "eligible": False, + "reason": "本轮模型预算已用尽,未完成部分等待后续执行"} + for row in rows: + if row.payload.get("status") in {"requirement", "media", "visual", "refreshing"}: + needed.add(row.payload["group"]) + pending = [row for row in rows if row.payload.get("status") == "value_pending"] + if pending and not (run.cursor or {}).get("_draining"): + other = any(row.payload.get("status") in {"parse", "requirement", "media", "visual"} for row in rows) + oldest = min(datetime.fromisoformat(row.payload["value_ready_at"]) for row in pending) + runtime = self.settings.get() + if len(pending) >= 5 or not other or (now - oldest).total_seconds() >= 30: + batch = pending[:1 if runtime.jev_enabled else 30] + self._value_batch(session, store, run, batch, runtime) + session.commit() + self.registry.retain(needed) + return [run.id for run in runs] + + def resources(self, limits): + return self.guard.evaluate(limits) + + def _restore_media(self, store): + with self.factory() as session: + pending = session.query(WorkItem).filter(WorkItem.stage == Stage.ANALYZE.value, + WorkItem.state.in_(("pending", "retry")), WorkItem.payload["role"].as_string() == "visual").all() + runtime = self.settings.get() + for work in pending: + key = work.payload["group"] + if self.registry.has(key) or self.registry.restore(key, work.payload, runtime): + continue + with store.lock, self.factory() as session: + current = session.get(WorkItem, work.id) + if current.state not in {"pending", "retry"}: + continue + current.state = "superseded" + session.flush() + if not store._room(session, work.task_id): + current.state = "pending" + session.commit() + continue + session.add(WorkItem(run_id=work.run_id, task_id=work.task_id, stage=Stage.MEDIA.value, + input_fingerprint=digest(["restore_media", work.id]), + payload={"listing_id": work.payload["listing_id"], "eval_id": work.payload["eval_id"], + "group": key, "force": work.payload.get("force", False), "role": "media"})) + row = session.get(OfferEvaluationRecord, work.payload["eval_id"]) + row.payload = {**row.payload, "status": "media"} + session.commit() + + def failed(self, session, work): + ids = work.payload.get("eval_ids") or [work.payload.get("eval_id")] + for i in ids: + row = session.get(OfferEvaluationRecord, i) if i else None + if row and row.payload.get("status") != "completed": + row.payload = {**row.payload, "status": "unverified", "eligible": False, + "reason": "本轮步骤失败,保留证据并等待后续重试"} + task = session.get(WatchTask, work.task_id) + if task: + task.last_error = f"{work.stage} 阶段未完成,未完成项将保留重试" + + def delivery_events(self, now): + from sqlalchemy import or_ + with self.factory() as session: + rows = session.query(NotificationDelivery).filter( + NotificationDelivery.state.in_(("pending", "failed", "awaiting_revalidation", "retry_requested")), + or_(NotificationDelivery.not_before.is_(None), NotificationDelivery.not_before <= naive(now)), + or_(NotificationDelivery.attempt < 3, NotificationDelivery.state == "retry_requested") + ).order_by(NotificationDelivery.id).limit(20).all() + return list(dict.fromkeys(r.event_id for r in rows)) + + def _value_batch(self, session, store, run, rows, runtime): + if not store._room(session, run.task_id): + if not session.query(WorkItem.id).filter(WorkItem.run_id == run.id, + WorkItem.state.in_(("pending", "running", "retry"))).first(): + run.state = "waiting_capacity" + return + items, listing_ids, contexts = [], {}, {} + for row in rows: + if row.payload.get("status") != "value_pending": + continue + current = self._price_context(session, row) + row.payload = {**row.payload, **current} + if current["state"] != "exact": + if self._expired(session, row): + for next_node in self._refresh_price(session, run, session.get(ListingOffer, row.offer_id).listing_id): + if store._room(session, run.task_id) and not session.query(WorkItem.id).filter_by( + run_id=run.id, stage=next_node["stage"], input_fingerprint=next_node["fingerprint"]).first(): + session.add(WorkItem(run_id=run.id, task_id=run.task_id, stage=next_node["stage"], + input_fingerprint=next_node["fingerprint"], payload=next_node["payload"])) + session.flush() + continue + row.payload = {**row.payload, "status": "unverified", "eligible": False, + "reason": "价格或规格证据已过期,等待重新抓取"} + continue + offer = session.get(ListingOffer, row.offer_id) + listing = session.get(Listing, offer.listing_id) + task = session.get(WatchTask, run.task_id) + items.append({"external_id": f"offer:{offer.id}", "title": f"{listing.title} {dict(offer.attributes)}", + "price": row.payload["amount_cents"] / 100, "condition_score": row.payload.get("condition_score"), + "defects": (row.payload.get("media") or {}).get("observations", []), + "seller_risk": (listing.seller_risk or {}).get("risk_level"), "requirement": task.condition_requirement}) + listing_ids[str(row.id)] = listing.id + contexts[str(row.id)] = current + rows = [row for row in rows if str(row.id) in contexts] + if not rows: + return + key = digest(["value", items, self.config_key(runtime)]) + item_keys = {item["external_id"]: digest(["independent-value-1", item, self.config_key(runtime)]) for item in items} + identity = digest(["value", [r.id for r in rows], [r.payload["observation_id"] for r in rows], key, contexts]) + if not session.query(WorkItem.id).filter_by(run_id=run.id, stage=Stage.ANALYZE.value, input_fingerprint=identity).first(): + session.add(WorkItem(run_id=run.id, task_id=run.task_id, stage=Stage.ANALYZE.value, input_fingerprint=identity, + payload={"role": "value", "listing_id": next(iter(listing_ids.values())), "listing_ids": listing_ids, + "eval_ids": [r.id for r in rows], "items": items, "cache_key": key, "http": True, + "item_cache_keys": item_keys, + "price_contexts": contexts, + "model_cfg": self.model_config("value", runtime), "provider_gate": self.provider_gate("value", runtime), + "force": run.kind == "reanalyze"})) + for row in rows: + row.payload = {**row.payload, "status": "value_running"} + session.flush() + + def _price_context(self, session, row): + offer = session.get(ListingOffer, row.offer_id) + listing = session.get(Listing, offer.listing_id) + task = session.get(WatchTask, listing.task_id) + observation = session.query(OfferObservation).filter_by(offer_id=offer.id).order_by(OfferObservation.id.desc()).first() + candidate = decode_offer(observation.payload) + quote = session.query(ManualQuote).filter_by(offer_id=offer.id, revoked=False).order_by(ManualQuote.id.desc()).first() + value = None if quote is None else QuoteInput(candidate.key, quote.amount_cents, utc(quote.confirmed_at), + utc(quote.expires_at), quote.evidence_fingerprint, quote.revoked) + decision = decide_price(candidate, value, self.now(), task.interval_minutes * 60 if task and task.fetch_detail else 0) + return {"state": decision.state, "source": decision.source, "amount_cents": decision.amount_cents, + "fingerprint": observation.fingerprint} + + def _expired(self, session, row): + observation = session.get(OfferObservation, row.payload["observation_id"]) + offer = session.get(ListingOffer, row.offer_id) + listing = session.get(Listing, offer.listing_id) + task = session.get(WatchTask, listing.task_id) + return bool(task and task.fetch_detail and + (self.now() - utc(observation.observed_at)).total_seconds() >= task.interval_minutes * 60) + + def _assessment(self, session, row, runtime): + offer = session.get(ListingOffer, row.offer_id) + listing = session.get(Listing, offer.listing_id) + task = session.get(WatchTask, listing.task_id) + observation = session.query(OfferObservation).filter_by(offer_id=offer.id).order_by(OfferObservation.id.desc()).first() + candidate = decode_offer(observation.payload) + quote = session.query(ManualQuote).filter_by(offer_id=offer.id, revoked=False).order_by(ManualQuote.id.desc()).first() + value = None if quote is None else QuoteInput(candidate.key, quote.amount_cents, utc(quote.confirmed_at), + utc(quote.expires_at), quote.evidence_fingerprint, quote.revoked) + return assess_offer(task, listing, candidate, value, self.now(), vision_enabled=self.provider.enabled("visual", runtime), + condition_score=row.payload.get("condition_score"), requirement_match=row.payload.get("requirement_match"), + visual_valid=row.payload.get("visual_scope") == candidate.key) + + def _source_key(self, session, listing, detail=None): + from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit + def canonical(url): + try: + parts = urlsplit(url) + ignored = {"sig", "sign", "signature", "token", "expires", "ossaccesskeyid", "x-oss-signature", "x-oss-security-token", "x-oss-credential", "x-oss-date", "x-oss-expires"} + query = urlencode([(k, v) for k, v in parse_qsl(parts.query) if k.lower() not in ignored]) + return urlunsplit((parts.scheme, parts.netloc, parts.path, query, "")) + except ValueError: + return "unsupported" + if detail is None: + description = listing.description + media = [(r.kind, canonical(r.source_url), r.offer_key) for r in session.query(MediaAsset).filter( + MediaAsset.listing_id == listing.id, MediaAsset.source_url != "", MediaAsset.status != "superseded")] + offers = [(r.offer_key, r.attributes) for r in session.query(ListingOffer).filter_by(listing_id=listing.id)] + else: + description = detail["description"] + media = [(r["kind"], canonical(r["url"]), r["offer_key"]) for r in detail["media"]] + offers = [(o["key"], o["attributes"]) for o in detail["offers"]] + return digest([listing.title, description, sorted(media, key=str), sorted(offers, key=str)]) + + def _refresh_price(self, session, run, listing_id): + listing = session.get(Listing, listing_id) + rows = session.query(OfferEvaluationRecord).join(ListingOffer).filter( + ListingOffer.listing_id == listing_id, OfferEvaluationRecord.payload["run_id"].as_integer() == run.id).all() + rows = [r for r in rows if r.payload.get("status") not in {"ready", "completed", "superseded"}] + observations = [r.payload["observation_id"] for r in rows] + for row in rows: + row.payload = {**row.payload, "status": "refreshing", "eligible": False} + return [node(Stage.DETAIL, ["price_refresh", listing_id, observations], { + "listing_id": listing_id, "price_refresh": True, "eval_ids": [r.id for r in rows], + "source_key": self._source_key(session, listing)})] + + +def build_pipeline(factory, settings_service, *, adapter_factory=None, provider=None, + notifier_factory=None, processor_factory=None, clock=time.time): + if notifier_factory is None: + def notifier_factory(): + from goodprice.notify.log import LogNotifier + from goodprice.notify.registry import build_enabled_notifiers + return dict([("log", LogNotifier()), *build_enabled_notifiers(settings_service.get())]) + pipeline = Pipeline(factory, settings_service, adapter_factory, provider, notifier_factory, processor_factory, clock) + limits = ResourceLimits.from_settings(settings_service.get()) + store = ExecutionStore(factory, global_limit=limits.global_items, task_limit=limits.task_items, + active_limit=limits.active_runs, run_minutes=limits.run_minutes) + store.finish_guard = pipeline.can_finish + store.completion_state = lambda session, run: "deferred" if (run.cursor or {}).get("remaining_ids") else "completed" + pipeline.store = store + controller = Coordinator(store, limits, WorkerPools(limits, {stage: pipeline.worker for stage in Stage if stage != Stage.NOTIFY}), + clock=clock, apply=pipeline.apply) + controller.pipeline = pipeline + return controller diff --git a/goodprice/execution/protection.py b/goodprice/execution/protection.py new file mode 100644 index 0000000..701ff3d --- /dev/null +++ b/goodprice/execution/protection.py @@ -0,0 +1,47 @@ +"""Read-only resource protection; saved user settings are never rewritten.""" +import time +from dataclasses import dataclass, replace + +from goodprice.execution.limits import ResourceLimits +from goodprice.resources import detect_resources + + +@dataclass(frozen=True) +class EffectiveResources: + limits: ResourceLimits + paused: frozenset[str] + reasons: tuple[str, ...] + low: bool + + +class ResourceGuard: + def __init__(self, detector=None, clock=time.monotonic): + self.detector, self.clock = detector or detect_resources, clock + self._at, self._snapshot = 0, None + + def evaluate(self, configured): + if self._snapshot is None or self.clock() - self._at >= 5: + self._snapshot, self._at = self.detector(), self.clock() + snapshot = self._snapshot + available = snapshot.available_memory_bytes + low = snapshot.source == "unknown" or available is None or available < 1024**3 + paused, reasons = set(), [] + limits = configured + if low: + limits = replace(configured, crawler_workers=1, model_workers=1, media_workers=1) + reasons.append("内存余量较小或检测不完整,实际工作并发暂限为1") + if available is not None: + for lane, reserve_mib in {"crawler": 256, "media": 192, "model": 128}.items(): + if available < reserve_mib * 1024**2: + paused.add(lane) + if paused: + reasons.append("可用内存不足,暂缓新请求并保留待执行步骤") + return EffectiveResources(limits, frozenset(paused), tuple(reasons), low) + + def media_settings(self, settings): + from goodprice.media.budget import BudgetExceeded + current = self.evaluate(ResourceLimits.from_settings(settings)) + if "media" in current.paused: + raise BudgetExceeded("媒体处理等待内存余量") + return settings.model_copy(update={"execution_media_workers": current.limits.media_workers, + "media_ffmpeg_threads": 1 if current.low else settings.media_ffmpeg_threads}) diff --git a/goodprice/execution/store.py b/goodprice/execution/store.py new file mode 100644 index 0000000..8dd1145 --- /dev/null +++ b/goodprice/execution/store.py @@ -0,0 +1,389 @@ +import threading +import uuid +from datetime import datetime, timedelta, timezone + +from goodprice.execution.types import RunToken, Stage, WorkItemDTO, freeze, thaw +from goodprice.models import ( + AppSetting, + NotificationDelivery, + NotificationEvent, + TaskRun, + WatchTask, + WorkItem, +) + +_locks = {} +_locks_guard = threading.Lock() +OPEN_WORK = ("pending", "running", "retry") +OPEN_RUN = ("queued", "active", "waiting_capacity") + + +def naive(value): + return value.astimezone(timezone.utc).replace(tzinfo=None) if value.tzinfo else value + + +def invalidate_in_session(session, task_id): + session.query(TaskRun).filter(TaskRun.task_id == task_id, TaskRun.state.in_(OPEN_RUN)).update( + {"state": "cancelled", "followup": False}) + session.query(WorkItem).filter(WorkItem.task_id == task_id, WorkItem.state.in_(OPEN_WORK)).update({"state": "cancelled"}) + events = session.query(NotificationEvent.id).filter_by(task_id=task_id) + session.query(NotificationDelivery).filter(NotificationDelivery.event_id.in_(events), + NotificationDelivery.state.in_(("pending", "failed", "awaiting_revalidation"))).update({"state": "cancelled"}, synchronize_session=False) + + +class ExecutionStore: + def __init__(self, session_factory, global_limit=200, task_limit=60, active_limit=2, + lease_seconds=120, run_minutes=15): + self.factory = session_factory + self.global_limit, self.task_limit, self.active_limit = global_limit, task_limit, active_limit + self.lease_seconds, self.run_minutes = lease_seconds, run_minutes + self._last_task = {} + with _locks_guard: + self.lock = _locks.setdefault(str(session_factory.kw["bind"].url), threading.RLock()) + + def _room(self, session, task_id): + query = session.query(WorkItem).filter(WorkItem.state.in_(OPEN_WORK)) + return query.count() < self.global_limit and query.filter_by(task_id=task_id).count() < self.task_limit + + def _submit(self, session, task, kind, target_id, *, accepted_followup=False): + room = self._room(session, task.id) + if not room and not accepted_followup: + return None + run = TaskRun(task_id=task.id, task_version=task.version, kind=kind, target_id=target_id) + used = session.query(TaskRun.cursor["resume_run_id"].as_integer()).filter( + TaskRun.task_id == task.id, TaskRun.cursor["resume_run_id"].as_integer().is_not(None)) + previous = session.query(TaskRun).filter(TaskRun.task_id == task.id, TaskRun.task_version == task.version, + TaskRun.state.in_(("deferred", "failed")), TaskRun.kind == kind, TaskRun.target_id == target_id, + TaskRun.id.not_in(used)).order_by(TaskRun.id.desc()).first() + if previous: + run.cursor = {**{k: v for k, v in previous.cursor.items() if k not in {"_draining", "_http_attempts"}}, "resume_run_id": previous.id} + session.add(run) + session.flush() + if room: + self._add_root(session, run) + else: + run.cursor = {**(run.cursor or {}), "_root_pending": True} + return run.id + + def _add_root(self, session, run): + session.add(WorkItem(run_id=run.id, task_id=run.task_id, stage=Stage.SEARCH.value, + payload={"target_id": run.target_id} if run.target_id else {})) + + def _promote_followup(self, session, run): + cursor = dict(run.cursor or {}) + manual = cursor.pop("_manual_followup", None) + due = run.followup + run.cursor, run.followup = cursor, False + task = session.get(WatchTask, run.task_id) + if task is None or not task.enabled or task.version != run.task_version: + return + if manual: + next_id = self._submit(session, task, "reanalyze", manual["target_id"], accepted_followup=True) + session.get(TaskRun, next_id).followup = due + elif due: + self._submit(session, task, "crawl", None, accepted_followup=True) + + def submit(self, task_id: int, kind: str = "crawl", target_id=None): + with self.lock, self.factory() as session: + task = session.get(WatchTask, task_id) + if task is None or not task.enabled: + return None + existing = session.query(TaskRun).filter(TaskRun.task_id == task_id, TaskRun.state.in_(OPEN_RUN)).first() + if existing and existing.task_version == task.version: + if kind == "reanalyze": + if existing.kind == kind and existing.target_id == target_id: + return existing.id + pending = (existing.cursor or {}).get("_manual_followup") + if pending and pending["target_id"] != target_id: + return None + existing.cursor = {**(existing.cursor or {}), "_manual_followup": {"target_id": target_id}} + elif existing.state != "queued" or existing.kind == "reanalyze": + existing.followup = True + session.commit() + return existing.id + if existing: + invalidate_in_session(session, task_id) + result = self._submit(session, task, kind, target_id) + session.commit() + return result + + def enqueue(self, run_id, stage, fingerprint, payload): + with self.lock, self.factory() as session: + run = session.get(TaskRun, run_id) + if run is None or run.state not in OPEN_RUN or not self._room(session, run.task_id): + return False + if session.query(WorkItem).filter_by(run_id=run_id, stage=stage.value, input_fingerprint=fingerprint).first(): + return True + session.add(WorkItem(run_id=run_id, task_id=run.task_id, stage=stage.value, + input_fingerprint=fingerprint, payload=thaw(payload))) + session.commit() + return True + + def set_cursor(self, run_id, cursor): + with self.lock, self.factory() as session: + run = session.get(TaskRun, run_id) + if run is None or run.state not in OPEN_RUN: + return False + protected = {k: v for k, v in (run.cursor or {}).items() if k.startswith("_") or k == "resume_run_id"} + run.cursor = {**thaw(cursor), **protected} + session.commit() + return True + + def _valid(self, session, run, now): + task = session.get(WatchTask, run.task_id) + return (task is not None and task.enabled and task.version == run.task_version + and run.state in OPEN_RUN and (run.deadline_at is None or run.deadline_at > now)) + + def claim(self, stage: Stage, now: datetime): + now = naive(now) + with self.lock, self.factory() as session: + active = session.query(TaskRun).filter_by(state="active").count() + from sqlalchemy import or_ + waiting_ready = session.query(WorkItem.id).join(TaskRun, TaskRun.id == WorkItem.run_id).filter( + TaskRun.state == "waiting_capacity", WorkItem.state.in_(("pending", "retry")), + or_(WorkItem.not_before.is_(None), WorkItem.not_before <= now)).first() is not None + rows = session.query(WorkItem).join(TaskRun, TaskRun.id == WorkItem.run_id).filter( + WorkItem.stage == stage.value, WorkItem.state.in_(("pending", "retry")), + TaskRun.state.in_(OPEN_RUN)).order_by(WorkItem.attempt, WorkItem.id).all() + task_ids = sorted({row.task_id for row in rows}) + last = self._last_task.get(stage, 0) + rotated = [i for i in task_ids if i > last] + [i for i in task_ids if i <= last] + order = {key: i for i, key in enumerate(rotated)} + rows.sort(key=lambda row: (order[row.task_id], row.id)) + for item in rows: + run = session.get(TaskRun, item.run_id) + if not self._valid(session, run, now): + invalidate_in_session(session, run.task_id) + continue + if item.not_before and item.not_before > now: + continue + cache_key = (item.payload or {}).get("cache_key") + if stage == Stage.ANALYZE and cache_key and session.query(WorkItem.id).filter( + WorkItem.state == "running", WorkItem.stage == stage.value, + WorkItem.payload["cache_key"].as_string() == cache_key).first(): + continue + if run.state != "active": + if run.state == "queued" and waiting_ready: + continue + if active >= self.active_limit: + continue + run.state = "active" + if run.started_at is None: + run.started_at = now + run.deadline_at = now + timedelta(minutes=self.run_minutes) + item.state, item.lease_id = "running", uuid.uuid4().hex + item.started_at = now + item.lease_until = min(run.deadline_at, now + timedelta(seconds=self.lease_seconds)) + item.attempt += 1 + self._last_task[stage] = item.task_id + session.commit() + return WorkItemDTO(item.id, RunToken(run.id, run.task_id, run.task_version, item.lease_id), + stage, item.input_fingerprint, freeze(item.payload), item.attempt, + run.deadline_at.replace(tzinfo=timezone.utc)) + session.commit() + return None + + def complete(self, result, now: datetime, *, apply=None): + now = naive(now) + with self.lock, self.factory() as session: + run = session.get(TaskRun, result.token.run_id) + if run is None or not self._valid(session, run, now) or run.task_version != result.token.task_version: + return False + from sqlalchemy import update + gate = session.execute(update(WatchTask).where(WatchTask.id == run.task_id, + WatchTask.version == run.task_version, WatchTask.enabled.is_(True)).values( + version=WatchTask.version).execution_options(synchronize_session=False)) + if gate.rowcount != 1: + return False + item = session.query(WorkItem).filter_by(run_id=run.id, lease_id=result.token.lease_id, + state="running", stage=result.stage.value, input_fingerprint=result.input_fingerprint).first() + if item is None or item.lease_until <= now: + return False + output = thaw(result.output) + if apply: + applied = apply(result, session) + if applied: + output.update(thaw(applied)) + if output.get("no_http") and item.payload.get("_admitted_lease") == result.token.lease_id: + run.cursor = {**(run.cursor or {}), "_http_attempts": max(0, (run.cursor or {}).get("_http_attempts", 0) - 1)} + item.state, item.output, item.lease_id = "done", output, None + item.finished_at = now + session.commit() + return True + + def finish_run(self, run_id, now): + with self.lock, self.factory() as session: + run = session.get(TaskRun, run_id) + if run is None or not self._valid(session, run, naive(now)): + return False + guard = getattr(self, "finish_guard", None) + if guard and not guard(session, run): + return False + if session.query(WorkItem).filter(WorkItem.run_id == run_id, WorkItem.state.in_(OPEN_WORK)).count(): + return False + if any((item.output or {}).get("next") for item in session.query(WorkItem).filter_by(run_id=run_id, state="done")): + return False + run.state = "failed" if session.query(WorkItem).filter_by(run_id=run_id, state="failed").first() else "completed" + state_for = getattr(self, "completion_state", None) + if state_for and run.state == "completed": + run.state = state_for(session, run) + session.flush() + self._promote_followup(session, run) + session.commit() + return True + + def recover(self, now, *, startup=False): + now = naive(now) + count = 0 + with self.lock, self.factory() as session: + for item in session.query(WorkItem).filter_by(state="running"): + if startup or item.lease_until and item.lease_until <= now: + item.state, item.lease_id = "pending", None + count += 1 + for run in session.query(TaskRun).filter(TaskRun.state.in_(OPEN_RUN)).all(): + task = session.get(WatchTask, run.task_id) + if task is None or not task.enabled or task.version != run.task_version: + invalidate_in_session(session, run.task_id) + elif run.deadline_at and run.deadline_at <= now: + run.state = "deferred" + session.query(WorkItem).filter(WorkItem.run_id == run.id, + WorkItem.state.in_(OPEN_WORK)).update({"state": "deferred", "lease_id": None}) + session.flush() + self._promote_followup(session, run) + session.query(NotificationDelivery).filter(NotificationDelivery.state == "sending", + NotificationDelivery.lease_until <= now).update({"state": "uncertain"}) + session.commit() + return count + + def invalidate_task(self, task_id): + with self.lock, self.factory() as session: + invalidate_in_session(session, task_id) + session.commit() + + def materialize(self, now): + """Durably drain bounded fan-out; unexpanded outputs survive a restart.""" + with self.lock, self.factory() as session: + from sqlalchemy import func + for run in session.query(TaskRun).filter(TaskRun.state == "queued", + TaskRun.cursor["_root_pending"].as_boolean().is_(True)).order_by(TaskRun.id).all(): + if not self._room(session, run.task_id): + break + cursor = dict(run.cursor) + cursor.pop("_root_pending", None) + run.cursor = cursor + self._add_root(session, run) + session.flush() + active_runs = session.query(TaskRun.id).filter(TaskRun.state.in_(OPEN_RUN)) + rows = session.query(WorkItem).filter(WorkItem.state == "done", WorkItem.run_id.in_(active_runs), + func.json_array_length(WorkItem.output["next"]) > 0).order_by(WorkItem.id).all() + for item in rows: + pending = list((item.output or {}).get("next", [])) + if not pending: + continue + run = session.get(TaskRun, item.run_id) + if not self._valid(session, run, naive(now)): + continue + expanded = 0 + for node in pending: + if (run.cursor or {}).get("_draining") and node["stage"] not in {"parse", "aggregate"}: + expanded += 1 + continue + if not self._room(session, run.task_id): + if not session.query(WorkItem.id).filter(WorkItem.run_id == run.id, + WorkItem.state.in_(OPEN_WORK)).first(): + run.state = "waiting_capacity" + break + stage = Stage(node["stage"]) + fingerprint = str(node.get("fingerprint", ""))[:64] + if not session.query(WorkItem).filter_by(run_id=run.id, stage=stage.value, input_fingerprint=fingerprint).first(): + session.add(WorkItem(run_id=run.id, task_id=run.task_id, stage=stage.value, + input_fingerprint=fingerprint, payload=thaw(node.get("payload", {})))) + session.flush() + expanded += 1 + item.output = {**item.output, "next": pending[expanded:]} + session.commit() + + def fail(self, item, now, *, terminal=False, delay_seconds=None, apply=None): + with self.lock, self.factory() as session: + row = session.get(WorkItem, item.id) + if row is None or row.state != "running" or row.lease_id != item.token.lease_id: + return + row.state = "retry" if row.attempt < 3 and not terminal else "failed" + row.lease_id = None + row.not_before = naive(now) + timedelta(seconds=max(2 ** row.attempt, delay_seconds or 0)) + if row.state == "failed" and apply: + run = session.get(TaskRun, row.run_id) + if self._valid(session, run, naive(now)): + apply(session, row) + session.commit() + + def admit(self, item, now, *, gate, interval, http_limit=None, provider=None): + """Reserve one real request; cache hits never pass through this method.""" + with self.lock, self.factory() as session: + row = session.get(WorkItem, item.id) + run = session.get(TaskRun, item.token.run_id) + if not run or not self._valid(session, run, naive(now)) or row.state != "running" or row.lease_id != item.token.lease_id: + return False + key = "$execution:gate:" + gate + entry = session.get(AppSetting, key) + allowed = float(entry.value) if entry else 0 + if provider: + cooldown = session.get(AppSetting, "$execution:gate:provider:" + provider) + allowed = max(allowed, float(cooldown.value) if cooldown else 0) + if allowed > now.timestamp(): + row.state, row.lease_id = "pending", None + row.attempt = max(0, row.attempt - 1) + row.not_before = datetime.fromtimestamp(min(allowed, run.deadline_at.replace(tzinfo=timezone.utc).timestamp()), timezone.utc).replace(tzinfo=None) + session.commit() + return False + cursor = dict(run.cursor or {}) + if http_limit is not None and cursor.get("_http_attempts", 0) >= http_limit: + cursor["_draining"] = True + run.cursor = cursor + row.state, row.lease_id = "deferred", None + session.query(WorkItem).filter(WorkItem.run_id == run.id, WorkItem.state.in_(("pending", "retry")), + WorkItem.stage.in_(("search", "detail", "media", "analyze"))).update({"state": "deferred"}) + session.commit() + return False + if http_limit is not None: + run.cursor = {**cursor, "_http_attempts": cursor.get("_http_attempts", 0) + 1} + row.payload = {**row.payload, "_admitted_lease": item.token.lease_id} + until = str(now.timestamp() + interval) + if entry: + entry.value = until + else: + session.add(AppSetting(key=key, value=until)) + session.commit() + return True + + def postpone_gate(self, gate, until): + with self.lock, self.factory() as session: + key = "$execution:gate:" + gate + row = session.get(AppSetting, key) + if row: + row.value = str(max(float(row.value), until)) + else: + session.add(AppSetting(key=key, value=str(until))) + session.commit() + + def wait(self, item, now, delay): + with self.lock, self.factory() as session: + row = session.get(WorkItem, item.id) + if row and row.state == "running" and row.lease_id == item.token.lease_id: + row.state, row.lease_id = "pending", None + row.attempt = max(0, row.attempt - 1) + row.not_before = naive(now) + timedelta(seconds=delay) + session.commit() + + def renew(self, token, now): + with self.lock, self.factory() as session: + run = session.get(TaskRun, token.run_id) + if run and self._valid(session, run, naive(now)): + row = session.query(WorkItem).filter_by(run_id=run.id, state="running", lease_id=token.lease_id).first() + if row: + row.lease_until = min(run.deadline_at, naive(now) + timedelta(seconds=self.lease_seconds)) + session.commit() + + def has_open_work(self): + with self.factory() as session: + return session.query(WorkItem.id).filter(WorkItem.state.in_(OPEN_WORK)).first() is not None diff --git a/goodprice/execution/types.py b/goodprice/execution/types.py new file mode 100644 index 0000000..438c2a3 --- /dev/null +++ b/goodprice/execution/types.py @@ -0,0 +1,61 @@ +from dataclasses import dataclass +from datetime import datetime +from enum import Enum +from types import MappingProxyType +from typing import Mapping + + +class Stage(str, Enum): + SEARCH = "search" + DETAIL = "detail" + PARSE = "parse" + MEDIA = "media" + ANALYZE = "analyze" + AGGREGATE = "aggregate" + NOTIFY = "notify" + + +def freeze(value): + if isinstance(value, dict): + return MappingProxyType({k: freeze(v) for k, v in value.items()}) + if isinstance(value, (tuple, list)): + return tuple(freeze(v) for v in value) + return value + + +def thaw(value): + if isinstance(value, Mapping): + return {k: thaw(v) for k, v in value.items()} + if isinstance(value, (tuple, list)): + return [thaw(v) for v in value] + return value + + +@dataclass(frozen=True) +class RunToken: + run_id: int + task_id: int + task_version: int + lease_id: str + + +@dataclass(frozen=True) +class WorkItemDTO: + id: int + token: RunToken + stage: Stage + input_fingerprint: str + payload: Mapping + attempt: int + deadline_at: datetime + + +@dataclass(frozen=True) +class WorkResult: + token: RunToken + stage: Stage + input_fingerprint: str + output: Mapping + + def __post_init__(self): + object.__setattr__(self, "output", freeze(self.output)) diff --git a/goodprice/execution/workers.py b/goodprice/execution/workers.py new file mode 100644 index 0000000..b5ecd84 --- /dev/null +++ b/goodprice/execution/workers.py @@ -0,0 +1,78 @@ +import threading +from concurrent.futures import ThreadPoolExecutor + +from goodprice.execution.browser import BrowserActor +from goodprice.execution.types import Stage, WorkResult + +STAGE_LANE = {Stage.SEARCH: "crawler", Stage.DETAIL: "crawler", Stage.PARSE: "local", + Stage.MEDIA: "media", Stage.ANALYZE: "model", Stage.AGGREGATE: "local", Stage.NOTIFY: "delivery"} + + +class WorkerPools: + """The coordinator checks slots before dispatch: no unbounded executor queue.""" + def __init__(self, limits, handlers): + self.handlers = handlers + self.capacities = {"crawler": limits.crawler_workers, "model": limits.model_workers, + "media": limits.media_workers, "local": 1, "delivery": 1} + # Threads are created lazily. Hard ceilings allow later confirmed raises + # without replacing an executor that still owns active work. + ceilings = {"model": 8, "media": 2, "local": 1, "delivery": 1} + self.pools = {name: ThreadPoolExecutor(max_workers=count, thread_name_prefix=f"second-eye-{name}") + for name, count in ceilings.items()} + self._busy = dict.fromkeys(self.capacities, 0) + self._lock = threading.Lock() + self.browsers = [BrowserActor(f"second-eye-browser-{i}") for i in range(2)] + + def update_limits(self, limits): + with self._lock: + self.capacities.update(crawler=limits.crawler_workers, model=limits.model_workers, + media=limits.media_workers) + + def available(self, stage): + lane = STAGE_LANE[stage] + with self._lock: + return (self._busy[lane] < self.capacities[lane] and stage in self.handlers + and (lane != "crawler" or any(actor.available() for actor in self.browsers))) + + def dispatch_item(self, item): + lane = STAGE_LANE[item.stage] + with self._lock: + if self._busy[lane] >= self.capacities[lane]: + raise RuntimeError("Resource lane full") + actor = next((a for a in self.browsers if a.available()), None) if lane == "crawler" else None + if lane == "crawler" and actor is None: + raise RuntimeError("Browser lane full") + self._busy[lane] += 1 + def execute(): + try: + output = self.handlers[item.stage](item) + return WorkResult(item.token, item.stage, item.input_fingerprint, output) + finally: + with self._lock: + self._busy[lane] -= 1 + return actor.submit(execute) if actor else self.pools[lane].submit(execute) + + def side_available(self, lane): + with self._lock: + return self._busy[lane] < self.capacities[lane] + + def dispatch_side(self, lane, fn): + if lane == "crawler": + raise ValueError("Browser operations require an owning actor") + with self._lock: + if self._busy[lane] >= self.capacities[lane]: + raise RuntimeError("Resource lane full") + self._busy[lane] += 1 + def execute(): + try: + return fn() + finally: + with self._lock: + self._busy[lane] -= 1 + return self.pools[lane].submit(execute) + + def stop(self): + for actor in self.browsers: + actor.stop() + for pool in self.pools.values(): + pool.shutdown(wait=False, cancel_futures=True) diff --git a/goodprice/main.py b/goodprice/main.py index 7f2bc09..3c23193 100644 --- a/goodprice/main.py +++ b/goodprice/main.py @@ -7,22 +7,27 @@ import uvicorn from fastapi import FastAPI +from fastapi.responses import JSONResponse +from fastapi.staticfiles import StaticFiles from goodprice.config import Settings, get_settings from goodprice.db import init_db, migrate_schema +from goodprice.media.budget import BudgetExceeded from goodprice.scheduler import _sync_tasks, build_scheduler from goodprice.services.crawl_service import CrawlService, TaskRunGuard from goodprice.services.seller_service import SellerService from goodprice.services.settings_service import SettingsService -from goodprice.services.task_queue import DEFAULT_GAP_SECONDS, TaskQueue from goodprice.services.task_service import TaskService +from goodprice.web.media_routes import router as media_router +from goodprice.web.offer_routes import router as offer_router +from goodprice.web.resource_routes import router as resource_router from goodprice.web.routes import router +from goodprice.web.settings_sections import router as settings_sections_router LOG_DIR = Path(__file__).resolve().parent.parent / "data" / "logs" -def _setup_logging() -> None: - LOG_DIR.mkdir(parents=True, exist_ok=True) +def _setup_logging(settings: Optional[Settings] = None) -> None: formatter = logging.Formatter("%(asctime)s %(levelname)s %(name)s: %(message)s") root = logging.getLogger() root.setLevel(logging.INFO) @@ -32,8 +37,10 @@ def _setup_logging() -> None: console.setFormatter(formatter) root.addHandler(console) if not os.environ.get("PYTEST_CURRENT_TEST"): + log_dir = (settings.data_root / "logs") if settings else LOG_DIR + log_dir.mkdir(parents=True, exist_ok=True) file_handler = RotatingFileHandler( - LOG_DIR / "app.log", + log_dir / "app.log", maxBytes=5 * 1024 * 1024, backupCount=3, encoding="utf-8", @@ -42,7 +49,6 @@ def _setup_logging() -> None: root.addHandler(file_handler) -_setup_logging() logger = logging.getLogger(__name__) @@ -50,8 +56,11 @@ def _make_crawl_service(session_factory, settings_service, guard): runtime = settings_service.get() from goodprice.analysis.llm import LLMClient from goodprice.crawler.xianyu import XianyuAdapter + from goodprice.media.budget import MediaBudget + from goodprice.media.processing import MediaProcessor from goodprice.notify.log import LogNotifier from goodprice.notify.registry import build_enabled_notifiers + from goodprice.services.media_service import MediaService adapter = XianyuAdapter(cookie=runtime.xianyu_cookie, proxy=runtime.proxy) seller_service = SellerService(session_factory, adapter=adapter) @@ -82,6 +91,9 @@ def _make_crawl_service(session_factory, settings_service, guard): settings_service=settings_service, guard=guard, seller_service=seller_service, + media_service=MediaService(session_factory, MediaProcessor(MediaBudget( + runtime.data_root / "media-cache", runtime.media_cache_mib * 1024**2, + runtime.media_retention_days * 86400))), ) @@ -99,8 +111,10 @@ def build_app( settings: Optional[Settings] = None, session_factory=None, with_scheduler: bool = True, + execution_factory=None, ) -> FastAPI: settings = settings or get_settings() + _setup_logging(settings) if session_factory is None: init_db(settings.database_url) from goodprice.db import make_session_factory @@ -115,64 +129,67 @@ def build_app( settings_service = SettingsService(session_factory, base=settings) login_session = _make_login_session(settings_service, settings) task_service = TaskService(session_factory) - from goodprice.services.satisfaction import backfill_satisfaction - - backfill_satisfaction(session_factory, vision_enabled=settings_service.get().vision_enabled) guard = TaskRunGuard() - def run_job(task_id: int) -> None: - logger.info("任务 %s 开始执行", task_id) - try: - stats = _make_crawl_service(session_factory, settings_service, guard).run_task(task_id) - logger.info("任务 %s 执行完成: %s", task_id, stats) - except Exception: - logger.exception("任务 %s 执行失败", task_id) - - def run_reanalyze(listing_id: int) -> None: - logger.info("重新分析商品 %s", listing_id) - try: - _make_crawl_service(session_factory, settings_service, guard).reanalyze_listing(listing_id) - except Exception: - logger.exception("重新分析商品 %s 失败", listing_id) - - task_queue = TaskQueue(run_job, gap_seconds=DEFAULT_GAP_SECONDS) + from goodprice.execution.pipeline import build_pipeline + task_queue = (execution_factory or build_pipeline)(session_factory, settings_service) + guard.provider = task_queue.running_ids scheduler = ( - build_scheduler(session_factory, task_queue.submit, task_service) + build_scheduler(session_factory, task_queue.submit, task_service, + jitter_seconds=lambda: settings_service.get().default_crawl_jitter_minutes * 60) if with_scheduler else None ) def sync_scheduler() -> None: if scheduler is not None: - _sync_tasks(session_factory, task_queue.submit, task_service, scheduler) + _sync_tasks(session_factory, task_queue.submit, task_service, scheduler, + jitter_seconds=lambda: settings_service.get().default_crawl_jitter_minutes * 60) @asynccontextmanager async def lifespan(app: FastAPI): task_queue.start() - if scheduler is not None: - app.state.scheduler = scheduler - app.state.scheduler.start() - yield - if scheduler is not None: - app.state.scheduler.shutdown(wait=False) - task_queue.stop() + try: + if scheduler is not None: + app.state.scheduler = scheduler + app.state.scheduler.start() + yield + finally: + try: + if scheduler is not None and scheduler.running: + scheduler.shutdown(wait=False) + finally: + task_queue.stop() app = FastAPI(title=settings.app_name, lifespan=lifespan) + @app.exception_handler(BudgetExceeded) + async def media_busy_handler(request, exc): + return JSONResponse({"detail": "媒体处理繁忙或磁盘预算不足,请稍后重试"}, + status_code=503, headers={"Retry-After": "5"}) + app.mount("/static", StaticFiles(directory=str(Path(__file__).parent / "web" / "static")), name="static") app.state.session_factory = session_factory app.state.settings_service = settings_service app.state.task_service = task_service app.state.run_job = task_queue.submit - app.state.run_reanalyze = run_reanalyze + app.state.run_reanalyze = task_queue.submit_reanalysis + app.state.coordinator = task_queue + app.state.resource_guard = getattr(getattr(task_queue, "pipeline", None), "guard", None) app.state.guard = guard app.state.sync_scheduler = sync_scheduler app.state.task_queue = task_queue app.state.login_session = login_session app.include_router(router) + app.include_router(offer_router) + app.include_router(media_router) + app.include_router(settings_sections_router) + app.include_router(resource_router) return app -app = build_app() +def create_app() -> FastAPI: + """ASGI factory: importing the module never opens a database or starts work.""" + return build_app() def main() -> None: - uvicorn.run("goodprice.main:app", host="0.0.0.0", port=8000, reload=False) + uvicorn.run("goodprice.main:create_app", factory=True, host="0.0.0.0", port=8000, reload=False) diff --git a/goodprice/media/__init__.py b/goodprice/media/__init__.py new file mode 100644 index 0000000..0fa0c87 --- /dev/null +++ b/goodprice/media/__init__.py @@ -0,0 +1 @@ +"""Bounded media evidence; importing this package performs no I/O.""" diff --git a/goodprice/media/budget.py b/goodprice/media/budget.py new file mode 100644 index 0000000..872541c --- /dev/null +++ b/goodprice/media/budget.py @@ -0,0 +1,158 @@ +"""Single-process disk accounting, shared by all media workers for a root. + +Pending directories are reservations, including every intermediate output. +Only one application process may own a data directory (also true of scheduler). +""" +import shutil +import threading +import time +import uuid +from contextlib import contextmanager +from dataclasses import dataclass, field +from pathlib import Path + + +class BudgetExceeded(RuntimeError): + pass + + +@dataclass +class _State: + lock: threading.RLock = field(default_factory=threading.RLock) + reservations: dict = field(default_factory=dict) + pins: dict = field(default_factory=dict) + processing: int = 0 + + def __post_init__(self): + self.condition = threading.Condition(self.lock) + + +_states = {} +_states_lock = threading.Lock() + + +@dataclass +class BudgetReservation: + token: str + bytes_reserved: int + directory: Path + budget: "MediaBudget" + + def __enter__(self): + return self + + def __exit__(self, *args): + self.budget.release(self.token) + + +class MediaBudget: + def __init__(self, root: Path, max_bytes: int, retention_seconds: int, + safety_bytes: int = 64 * 1024**2): + self.root = root.resolve() + self.root.mkdir(parents=True, exist_ok=True) + self.max_bytes = max_bytes + self.retention_seconds = retention_seconds + self.safety_bytes = safety_bytes + with _states_lock: + if self.root not in _states: + # Crash leftovers are never served. No external path is followed. + for path in self.root.glob("pending-*"): + if path.is_dir() and not path.is_symlink(): + shutil.rmtree(path) + _states[self.root] = _State() + self.state = _states[self.root] + + def checked(self, path: Path) -> Path: + absolute = path.resolve() + if not absolute.is_relative_to(self.root) or path.is_symlink(): + raise ValueError("Media path outside controlled cache") + return absolute + + def _files(self): + return [p for p in self.root.rglob("*") if p.is_file() and not p.is_symlink() + and p.resolve().is_relative_to(self.root) + and not any(part.startswith("pending-") for part in p.relative_to(self.root).parts)] + + @property + def usage_bytes(self): + with self.state.lock: + return sum(p.stat().st_size for p in self._files()) + sum( + reservation.bytes_reserved for reservation in self.state.reservations.values()) + + def reserve(self, key: str, bytes_needed: int) -> BudgetReservation: + if bytes_needed < 1 or bytes_needed > self.max_bytes: + raise BudgetExceeded("单项媒体预算超过缓存容量,请降低文件或采样上限") + with self.state.lock: + files = sorted(self._files(), key=lambda p: p.stat().st_mtime) + for path in files: + if self.state.pins.get(path.resolve(), 0): + continue + if (time.time() - path.stat().st_mtime >= self.retention_seconds + or self.usage_bytes + bytes_needed > self.max_bytes): + path.unlink(missing_ok=True) + if (self.usage_bytes + bytes_needed > self.max_bytes or + shutil.disk_usage(self.root).free < bytes_needed + self.safety_bytes): + raise BudgetExceeded("媒体磁盘预算暂不可用,请稍后重试或降低预算") + token = uuid.uuid4().hex + directory = self.root / f"pending-{token}" + directory.mkdir() + reservation = BudgetReservation(token, bytes_needed, directory, self) + self.state.reservations[token] = reservation + return reservation + + def commit(self, token: str, paths: list[Path]) -> None: + with self.state.lock: + reservation = self.state.reservations[token] + paths = [self.checked(p) for p in paths] + if any(not p.is_relative_to(reservation.directory) for p in paths): + raise ValueError("Output does not belong to this reservation") + # Check ALL files, not only outputs supplied by the caller. + size = sum(p.stat().st_size for p in reservation.directory.rglob("*") if p.is_file()) + if size > reservation.bytes_reserved: + raise BudgetExceeded("媒体输出超过预留空间") + # Keep output paths stable; remove the pending prefix via move then + # return paths through committed_paths. No file can escape its root. + destination = self.root / f"asset-{token}" + relative = [p.relative_to(reservation.directory) for p in paths] + kept = set(paths) + for p in reservation.directory.rglob("*"): + if p.is_file() and p.resolve() not in kept: + p.unlink() + reservation.directory.rename(destination) + reservation.committed_paths = [destination / p for p in relative] + del self.state.reservations[token] + + def release(self, token: str) -> None: + with self.state.lock: + reservation = self.state.reservations.pop(token, None) + if reservation and reservation.directory.exists(): + self.checked(reservation.directory) + shutil.rmtree(reservation.directory) + + @contextmanager + def processing_slot(self, limit, timeout_seconds=5): + with self.state.condition: + if not self.state.condition.wait_for(lambda: self.state.processing < limit, timeout_seconds): + raise BudgetExceeded("媒体处理繁忙,请稍后重试") + self.state.processing += 1 + try: + yield + finally: + with self.state.condition: + self.state.processing -= 1 + self.state.condition.notify_all() + + @contextmanager + def pin(self, paths): + with self.state.lock: + checked = [self.checked(p) for p in paths] + for p in checked: + self.state.pins[p] = self.state.pins.get(p, 0) + 1 + try: + yield + finally: + with self.state.lock: + for p in checked: + self.state.pins[p] -= 1 + if not self.state.pins[p]: + del self.state.pins[p] diff --git a/goodprice/media/cache.py b/goodprice/media/cache.py new file mode 100644 index 0000000..bc00b00 --- /dev/null +++ b/goodprice/media/cache.py @@ -0,0 +1,19 @@ +import hashlib +import json + + +def processing_key(content_hash: str, parameters: dict) -> str: + return hashlib.sha256(json.dumps([content_hash, parameters], sort_keys=True).encode()).hexdigest() + + +def find_cached(budget, key): + """Caller holds budget lock until cached inputs have been pinned.""" + for manifest in budget.root.glob(f"asset-*/{key}.json"): + try: + data = json.loads(manifest.read_text(encoding="utf-8")) + paths = [budget.checked(manifest.parent / item["file"]) for item in data] + if all(path.is_file() for path in paths): + return data, paths + except (OSError, ValueError, KeyError, TypeError): + continue + return None diff --git a/goodprice/media/download.py b/goodprice/media/download.py new file mode 100644 index 0000000..ccdc643 --- /dev/null +++ b/goodprice/media/download.py @@ -0,0 +1,202 @@ +"""Public HTTP media only; no environment proxy, cookies or second DNS lookup.""" +import ipaddress +import logging +import socket +import threading +import time +import uuid +from contextvars import ContextVar +from urllib.parse import urljoin, urlsplit + +import httpx +from httpcore import ConnectError, ReadTimeout +from httpcore._backends.base import NetworkStream +from httpcore._backends.sync import SyncBackend, SyncStream + +from goodprice.media.budget import MediaBudget +from goodprice.media.types import MediaCandidate + + +class MediaDownloadError(RuntimeError): + pass + + +_media_request = ContextVar("media_request", default=False) + + +class _RedactMediaRequest(logging.Filter): + def filter(self, record): + if _media_request.get(): + record.msg, record.args = "媒体 HTTP 请求(地址已隐藏)", () + return True + + +_log_filter = _RedactMediaRequest() +logging.getLogger("httpx").addFilter(_log_filter) + + +def resolve_public(host, port): + return list({row[4][0] for row in socket.getaddrinfo(host, port, type=socket.SOCK_STREAM)}) + + +_dns_slots = threading.BoundedSemaphore(2) + + +def bounded_resolve(resolver, host, port, deadline): + # OS DNS cannot be safely interrupted on every supported platform. Limit + # outstanding daemon lookups, and never keep the media worker waiting past + # its deadline or create an unbounded resolver backlog. + if not _dns_slots.acquire(timeout=max(0, deadline-time.monotonic())): + raise MediaDownloadError("媒体域名解析繁忙或超时") + done, result = threading.Event(), [] + def lookup(): + try: + result.append((True, resolver(host, port))) + except Exception: + result.append((False, None)) + finally: + _dns_slots.release() + done.set() + try: + threading.Thread(target=lookup, name="media-dns", daemon=True).start() + except Exception: + _dns_slots.release() + raise + if not done.wait(max(0, deadline-time.monotonic())) or not result[0][0]: + raise MediaDownloadError("媒体域名解析失败或超时") + return result[0][1] + + +def public_addresses(url, resolver): + parsed = urlsplit(url) + if (parsed.scheme not in {"http", "https"} or not parsed.hostname or parsed.username + or parsed.password or parsed.port not in {None, 80, 443}): + raise MediaDownloadError("媒体地址不受支持") + host = parsed.hostname + try: + literal = ipaddress.ip_address(host) + addresses = [str(literal)] + except ValueError: + addresses = resolver(host, parsed.port or (443 if parsed.scheme == "https" else 80)) + if not addresses or any(not ipaddress.ip_address(a).is_global for a in addresses): + raise MediaDownloadError("媒体地址必须指向公网") + # Reject IPv4-mapped/private transition targets as well. + for addr in addresses: + ip = ipaddress.ip_address(addr) + if isinstance(ip, ipaddress.IPv6Address) and (ip.ipv4_mapped or ip.sixtofour or ip.teredo): + raise MediaDownloadError("媒体地址类型不受支持") + return host, parsed.port or (443 if parsed.scheme == "https" else 80), addresses[0] + + +class DeadlineStream(NetworkStream): + def __init__(self, stream, deadline): + self.stream, self.deadline = stream, deadline + + def _timeout(self, timeout): + remaining = self.deadline - time.monotonic() + if remaining <= 0: + raise ReadTimeout("媒体下载超时") + return min(timeout, remaining) if timeout is not None else remaining + + def read(self, max_bytes, timeout=None): + return self.stream.read(max_bytes, self._timeout(timeout)) + + def write(self, buffer, timeout=None): + return self.stream.write(buffer, self._timeout(timeout)) + + def start_tls(self, ssl_context, server_hostname=None, timeout=None): + return DeadlineStream(self.stream.start_tls(ssl_context, server_hostname, self._timeout(timeout)), self.deadline) + + def close(self): + self.stream.close() + + def get_extra_info(self, info): + return self.stream.get_extra_info(info) + + +class PinnedBackend(SyncBackend): + def __init__(self, pins, deadline=None): + self.pins = pins + self.deadline = deadline + + def connect_tcp(self, host, port, timeout=None, local_address=None, socket_options=None): + addr = self.pins.get((host, port)) + if addr is None: + raise MediaDownloadError("未经验证的媒体目标") + ip = ipaddress.ip_address(addr) + sock = socket.socket(socket.AF_INET6 if ip.version == 6 else socket.AF_INET, socket.SOCK_STREAM) + try: + sock.settimeout(timeout) + for option in socket_options or []: + sock.setsockopt(*option) + # socket.connect receives a numeric address; hostname cannot rebind. + sock.connect((str(ip), port)) + stream = SyncStream(sock) + return DeadlineStream(stream, self.deadline) if self.deadline else stream + except OSError as exc: + sock.close() + raise ConnectError("媒体连接失败") from exc + + +class MediaDownloader: + def __init__(self, budget: MediaBudget, transport=None, resolver=None): + self.budget = budget + self.transport = transport + self.resolver = resolver or resolve_public + + def download(self, candidate: MediaCandidate, max_bytes: int, timeout_seconds: int, + reservation=None): + own = reservation is None + reservation = reservation or self.budget.reserve(candidate.key, max_bytes) + path = reservation.directory / f"{uuid.uuid4().hex}.part" + deadline = time.monotonic() + timeout_seconds + logging_token = _media_request.set(True) + try: + url = candidate.url + for _ in range(6): + host, port, addr = public_addresses(url, lambda host, port: bounded_resolve(self.resolver, host, port, deadline)) + remaining = deadline - time.monotonic() + if remaining <= 0: + raise MediaDownloadError("媒体下载超时") + transport = self.transport + if transport is None: + transport = httpx.HTTPTransport(trust_env=False, retries=0) + # httpcore 1.x backend preserves original Host and TLS SNI; + # only the socket destination is replaced with verified IP. + transport._pool._network_backend = PinnedBackend({(host, port): addr}, deadline) + with httpx.Client(transport=transport, trust_env=False, timeout=remaining, + follow_redirects=False) as client: + with client.stream("GET", url, headers={"Accept-Encoding": "identity"}) as response: + if response.is_redirect: + url = urljoin(url, response.headers.get("location", "")) + continue + response.raise_for_status() + if response.headers.get("content-encoding", "identity").lower() != "identity": + raise MediaDownloadError("媒体响应压缩格式不受支持") + length = response.headers.get("content-length") + if length and (not length.isdigit() or int(length) > max_bytes): + raise MediaDownloadError("媒体文件超过大小上限") + size = 0 + with path.open("xb") as output: + for chunk in response.iter_bytes(): + size += len(chunk) + if size > max_bytes or time.monotonic() > deadline: + raise MediaDownloadError("媒体下载超过字节或时间预算") + output.write(chunk) + if not size: + raise MediaDownloadError("媒体文件为空") + if own: + self.budget.commit(reservation.token, [path]) + return reservation.committed_paths[0] + return path + raise MediaDownloadError("媒体重定向次数超过限制") + except Exception as exc: + path.unlink(missing_ok=True) + if isinstance(exc, MediaDownloadError): + raise + # Never propagate signed URL, cookies, DNS details or response body. + raise MediaDownloadError("媒体下载失败") from None + finally: + _media_request.reset(logging_token) + if own: + self.budget.release(reservation.token) diff --git a/goodprice/media/frames.py b/goodprice/media/frames.py new file mode 100644 index 0000000..0d909d9 --- /dev/null +++ b/goodprice/media/frames.py @@ -0,0 +1,102 @@ +import hashlib +import json +import math +import shutil +import subprocess +import time +from pathlib import Path + +from goodprice.media.types import FrameFile, VideoInfo + +FORMATS = "mov,mp4,m4a,3gp,3g2,mj2,matroska,webm,avi" +MAX_PIXELS = 24_000_000 +MAX_OUTPUT_BYTES = 2 * 1024**2 + + +class MediaDecodeError(ValueError): + pass + + +def _local(path): + if not isinstance(path, Path) or not path.is_file() or path.is_symlink(): + raise MediaDecodeError("仅处理已下载的本地媒体文件") + return str(path.resolve()) + + +def _run(tool, args, timeout): + binary = shutil.which(tool) + if not binary: + raise MediaDecodeError("FFmpeg/ffprobe 未安装,视频未分析") + try: + result = subprocess.run([binary, *args], stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, timeout=max(.01, timeout), + check=True, text=True, encoding="utf-8") + except subprocess.TimeoutExpired: + raise MediaDecodeError("媒体解码超时") from None + except (subprocess.CalledProcessError, OSError): + raise MediaDecodeError("媒体文件解码失败") from None + return result.stdout + + +def probe_video(path: Path, timeout_seconds: int) -> VideoInfo: + path = _local(path) + raw = _run("ffprobe", ["-v", "error", "-protocol_whitelist", "file", "-format_whitelist", FORMATS, + "-select_streams", "v:0", "-show_entries", "stream=width,height:format=duration", + "-of", "json", path], timeout_seconds) + try: + data = json.loads(raw) + stream = data["streams"][0] + info = VideoInfo(float(data["format"]["duration"]), int(stream["width"]), int(stream["height"])) + if (not math.isfinite(info.duration_seconds) or info.duration_seconds <= 0 + or min(info.width, info.height) <= 0 or info.width * info.height > MAX_PIXELS): + raise ValueError() + return info + except (ValueError, TypeError, KeyError, IndexError): + raise MediaDecodeError("媒体时长或分辨率无效/超限") from None + + +def extract_frames(path: Path, count: int, output_dir: Path, timeout_seconds: int, + threads: int, max_seconds: int = 180) -> list[FrameFile]: + if not 1 <= count <= 24 or not 1 <= threads <= 4: + raise MediaDecodeError("抽帧数量或线程配置无效") + started = time.monotonic() + info = probe_video(path, timeout_seconds) + if info.duration_seconds > max_seconds: + raise MediaDecodeError("视频时长超过上限,未分析") + output_dir.mkdir(parents=True, exist_ok=True) + result, seen = [], set() + # Last point precedes the end by 250ms to avoid an empty seek at EOF. + end = max(0, info.duration_seconds - .25) + timestamps = [round(1000 * end * i / max(1, count - 1)) for i in range(count)] + try: + for i, timestamp in enumerate(timestamps): + output = output_dir / f"frame-{i}.jpg" + remaining = timeout_seconds - (time.monotonic() - started) + if remaining <= 0: + raise MediaDecodeError("媒体解码超时") + _run("ffmpeg", ["-v", "error", "-nostdin", "-y", "-protocol_whitelist", "file", + "-format_whitelist", FORMATS, "-threads", str(threads), + "-ss", str(timestamp / 1000), "-i", _local(path), + "-map", "0:v:0", "-an", "-sn", "-dn", "-frames:v", "1", + "-vf", "scale=1280:1280:force_original_aspect_ratio=decrease", + "-filter_threads", str(threads), "-threads", str(threads), + "-fs", str(MAX_OUTPUT_BYTES), str(output)], remaining) + if not output.is_file() or output.stat().st_size > MAX_OUTPUT_BYTES: + raise MediaDecodeError("视频帧输出无效或超过预算") + from PIL import Image, ImageFilter, ImageStat + with Image.open(output) as image: + image.verify() + with Image.open(output) as image: + # Flat or very blurred frames add no reliable visual evidence. + variance = ImageStat.Stat(image.convert("L").filter(ImageFilter.FIND_EDGES)).var[0] + digest = hashlib.sha256(output.read_bytes()).hexdigest() + if digest in seen or variance < 8: + output.unlink() + continue + seen.add(digest) + result.append(FrameFile(output, timestamp, digest)) + return result + except Exception: + for output in output_dir.glob("frame-*.jpg"): + output.unlink(missing_ok=True) + raise diff --git a/goodprice/media/parser.py b/goodprice/media/parser.py new file mode 100644 index 0000000..1701383 --- /dev/null +++ b/goodprice/media/parser.py @@ -0,0 +1,41 @@ +import hashlib +from urllib.parse import urljoin, urlsplit + +from bs4 import BeautifulSoup + +from goodprice.crawler.constants import GOOFISH_ORIGIN +from goodprice.media.types import MediaCandidate + + +def extract_media_candidates(html: str) -> list[MediaCandidate]: + soup = BeautifulSoup(html, "html.parser") + root = soup.select_one("[data-item-detail], main, [class*='detail-container']") + if root is None: + return [] + container = root.select_one("[data-item-media], [class*='item-slider'], [class*='main-media']") + nodes = (container.select("img, video") if container else + root.select("img.ant-image-img, [data-item-video]")) + result = [] + seen = set() + for node in nodes: + if node.find_parent(["aside", "nav"]) or node.find_parent(attrs={"data-recommend": True}): + continue + kind = "video" if node.name == "video" else "image" + source = node.select_one("source[src]") if kind == "video" else None + raw = node.get("src") or (source.get("src") if source else "") or node.get("data-src") + if not raw: + continue + url = urljoin(GOOFISH_ORIGIN, raw) + sku = node.get("data-sku-id") + scope = f"sku:{sku}" if sku else None + identity = (kind, url, scope) + if identity in seen: + continue + seen.add(identity) + parsed = urlsplit(url) + supported = parsed.scheme in {"http", "https"} and not parsed.path.lower().endswith((".m3u8", ".mpd")) and not node.get("data-drm") + result.append(MediaCandidate( + hashlib.sha256(repr(identity).encode()).hexdigest()[:32], kind, url, scope, + f"main-media:{len(result)}", node.get("poster"), + "pending" if supported else "unsupported")) + return result diff --git a/goodprice/media/processing.py b/goodprice/media/processing.py new file mode 100644 index 0000000..c2baffe --- /dev/null +++ b/goodprice/media/processing.py @@ -0,0 +1,105 @@ +import hashlib +import json +import warnings +from contextlib import ExitStack, contextmanager +from pathlib import Path + +from PIL import Image, UnidentifiedImageError + +from goodprice.media.cache import find_cached, processing_key +from goodprice.media.download import MediaDownloader, MediaDownloadError +from goodprice.media.frames import MAX_OUTPUT_BYTES, MAX_PIXELS, extract_frames +from goodprice.media.types import MediaEvidence + + +def prepare_image(path: Path, max_edge: int = 1280) -> Path: + output = path.with_suffix(".jpg") + try: + with warnings.catch_warnings(): + warnings.simplefilter("error", Image.DecompressionBombWarning) + with Image.open(path) as image: + if image.format not in {"JPEG", "PNG", "WEBP"} or image.width * image.height > MAX_PIXELS: + raise ValueError("图片格式或解码像素超过上限") + image.load() + image.thumbnail((max_edge, max_edge)) + image.convert("RGB").save(output, "JPEG", quality=85) + if output.stat().st_size > MAX_OUTPUT_BYTES: + raise ValueError("图片输出超过预算") + return output + except (OSError, UnidentifiedImageError, Image.DecompressionBombWarning, Image.DecompressionBombError): + output.unlink(missing_ok=True) + raise ValueError("图片解码失败或像素超限") from None + + +class MediaProcessor: + def __init__(self, budget, downloader=None): + self.budget = budget + self.downloader = downloader or MediaDownloader(budget) + + def process(self, candidates, settings): + with self.session(candidates, settings) as (evidence, limitations): + return evidence + + @contextmanager + def session(self, candidates, settings): + """Pins every selected file through the consumer's model request.""" + evidence, limitations = [], [] + counts = {"image": 0, "video": 0} + with ExitStack() as pins: + with self.budget.processing_slot(settings.execution_media_workers): + for candidate in candidates: + limit = settings.media_image_count if candidate.kind == "image" else settings.media_video_count + if counts[candidate.kind] >= limit: + limitations.append("retention_limit") + continue + counts[candidate.kind] += 1 + if candidate.status == "unsupported": + limitations.append("unsupported_media") + continue + if candidate.kind == "video" and not settings.video_analysis_enabled: + limitations.append("video_disabled") + continue + download_size = (settings.media_video_max_mib if candidate.kind == "video" else 10) * 1024**2 + output_size = MAX_OUTPUT_BYTES * (settings.media_frames_per_video if candidate.kind == "video" else 1) + try: + with self.budget.reserve(candidate.key, download_size + output_size + 65536) as reservation: + raw = self.downloader.download(candidate, download_size, settings.media_download_timeout, + reservation=reservation) + with raw.open("rb") as source: + source_hash = hashlib.file_digest(source, "sha256").hexdigest() + parameters = {"version": 1, "kind": candidate.kind, "edge": 1280, + "frames": settings.media_frames_per_video, + "max_seconds": settings.media_video_max_seconds} + key = processing_key(source_hash, parameters) + with self.budget.state.lock: + cached = find_cached(self.budget, key) + if cached: + data, paths = cached + pins.enter_context(self.budget.pin(paths)) + if not cached: + if candidate.kind == "image": + outputs = [prepare_image(raw)] + data = [{"file": outputs[0].name, "timestamp": None, + "hash": hashlib.sha256(outputs[0].read_bytes()).hexdigest()}] + else: + frames = extract_frames(raw, settings.media_frames_per_video, reservation.directory / "frames", + settings.media_decode_timeout, settings.media_ffmpeg_threads, + max_seconds=settings.media_video_max_seconds) + outputs = [f.path for f in frames] + data = [{"file": f.path.relative_to(reservation.directory).as_posix(), "timestamp": f.timestamp_ms, + "hash": f.content_hash} for f in frames] + raw.unlink(missing_ok=True) + manifest = reservation.directory / f"{key}.json" + manifest.write_text(json.dumps(data), encoding="utf-8") + with self.budget.state.lock: + self.budget.commit(reservation.token, outputs + [manifest]) + paths = reservation.committed_paths[:-1] + pins.enter_context(self.budget.pin(paths)) + for row, path in zip(data, paths, strict=True): + identity = hashlib.sha256(f"{row['hash']}:{candidate.offer_key}:{row['timestamp']}".encode()).hexdigest()[:32] + evidence.append(MediaEvidence(identity, "image" if candidate.kind == "image" else "frame", + row["hash"], path, candidate.offer_key, + candidate.key if candidate.kind == "video" else None, row["timestamp"])) + except (ValueError, RuntimeError, OSError, MediaDownloadError): + limitations.append(f"{candidate.kind}_unavailable") + yield evidence, tuple(dict.fromkeys(limitations)) diff --git a/goodprice/media/selection.py b/goodprice/media/selection.py new file mode 100644 index 0000000..f8e7723 --- /dev/null +++ b/goodprice/media/selection.py @@ -0,0 +1,44 @@ +from goodprice.media.types import MediaEvidence, MediaSelection + + +def select_media(images: list[MediaEvidence], frames: list[MediaEvidence], + image_quota: int, frame_quota: int, total_limit: int) -> MediaSelection: + if not (0 <= image_quota <= 16 and 0 <= frame_quota <= 16 and 1 <= total_limit <= 16): + raise ValueError("Invalid media input budget") + seen = set() + groups = [] + for group in (images, frames): + unique = [] + for item in group: + identity = (item.kind, item.content_hash, item.offer_key) + if identity not in seen: + seen.add(identity) + unique.append(item) + groups.append(unique) + quotas = [image_quota, frame_quota] + positions = [0, 0] + selected = [] + # First distribute the requested shares fairly, then lend unused shares. + for borrowing in (False, True): + while len(selected) < total_limit: + progressed = False + for i in (0, 1): + if (quotas[i] and positions[i] < len(groups[i]) + and (borrowing or positions[i] < quotas[i])): + selected.append(groups[i][positions[i]]) + positions[i] += 1 + progressed = True + if len(selected) == total_limit: + break + if not progressed: + break + ids = {x.id for x in selected} + omitted = tuple(x.id for x in images + frames if x.id not in ids) + limits = [] + if frames and not frame_quota: + limits.append("frames_disabled") + if images and not image_quota: + limits.append("images_disabled") + if omitted: + limits.append("partial_media_coverage") + return MediaSelection(tuple(selected), omitted, tuple(limits)) diff --git a/goodprice/media/types.py b/goodprice/media/types.py new file mode 100644 index 0000000..3411b5f --- /dev/null +++ b/goodprice/media/types.py @@ -0,0 +1,46 @@ +from dataclasses import dataclass +from pathlib import Path +from typing import Literal + + +@dataclass(frozen=True) +class MediaCandidate: + key: str + kind: Literal["image", "video"] + url: str + offer_key: str | None = None + source_locator: str = "" + poster_url: str | None = None + status: str = "pending" + + +@dataclass(frozen=True) +class MediaEvidence: + id: str + kind: Literal["image", "frame"] + content_hash: str + local_path: Path + offer_key: str | None = None + video_id: str | None = None + timestamp_ms: int | None = None + + +@dataclass(frozen=True) +class MediaSelection: + items: tuple[MediaEvidence, ...] + omitted_ids: tuple[str, ...] = () + limitations: tuple[str, ...] = () + + +@dataclass(frozen=True) +class VideoInfo: + duration_seconds: float + width: int + height: int + + +@dataclass(frozen=True) +class FrameFile: + path: Path + timestamp_ms: int + content_hash: str diff --git a/goodprice/migrations.py b/goodprice/migrations.py new file mode 100644 index 0000000..b5ed470 --- /dev/null +++ b/goodprice/migrations.py @@ -0,0 +1,30 @@ +"""Additive offer migration; the caller owns the transaction.""" +from sqlalchemy import inspect + +from goodprice.models import ( + AnalysisCache, + ListingOffer, + ManualQuote, + MediaAsset, + NotificationDelivery, + NotificationEvent, + OfferEvaluationRecord, + OfferObservation, + SchemaVersion, + TaskRun, + WorkItem, +) + + +def migrate_offer_schema(session) -> None: + connection = session.connection() + tables = set(inspect(connection).get_table_names()) + legacy = "schema_versions" not in tables and "watch_tasks" in tables + for model in (SchemaVersion, ListingOffer, OfferObservation, ManualQuote, OfferEvaluationRecord, MediaAsset, AnalysisCache, + TaskRun, WorkItem, NotificationEvent, NotificationDelivery): + model.__table__.create(connection, checkfirst=True) + if session.get(SchemaVersion, 1) is None: + session.add(SchemaVersion(id=1, version=3, legacy_install=legacy)) + else: + marker = session.get(SchemaVersion, 1) + marker.version = max(marker.version, 3) diff --git a/goodprice/models.py b/goodprice/models.py index cedc986..d1a22c8 100644 --- a/goodprice/models.py +++ b/goodprice/models.py @@ -36,6 +36,9 @@ class WatchTask(Base): interval_minutes: Mapped[int] = mapped_column(Integer, default=20) enabled: Mapped[bool] = mapped_column(default=True) fetch_detail: Mapped[bool] = mapped_column(default=True) + target_variant: Mapped[str] = mapped_column(Text, default="") + notify_pending_price: Mapped[bool] = mapped_column(default=False) + version: Mapped[int] = mapped_column(Integer, default=1) created_at: Mapped[datetime] = mapped_column(DateTime, default=_now) last_run_at: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True) last_error: Mapped[Optional[str]] = mapped_column(Text, nullable=True) @@ -53,6 +56,9 @@ class Listing(Base): external_id: Mapped[str] = mapped_column(String(200)) title: Mapped[str] = mapped_column(String(500)) price: Mapped[float] = mapped_column(Float) + offer_price_cents: Mapped[Optional[int]] = mapped_column(Integer, nullable=True) + offer_verified_until: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True) + offer_task_version: Mapped[Optional[int]] = mapped_column(Integer, nullable=True) url: Mapped[str] = mapped_column(Text) image_urls: Mapped[list[str]] = mapped_column(JSON, default=list) seller: Mapped[Optional[str]] = mapped_column(String(200), nullable=True) @@ -88,6 +94,8 @@ class Listing(Base): notifications: Mapped[list["Notification"]] = relationship( back_populates="listing", cascade="all, delete-orphan" ) + offers: Mapped[list["ListingOffer"]] = relationship(cascade="all, delete-orphan") + media_assets: Mapped[list["MediaAsset"]] = relationship(cascade="all, delete-orphan") class PriceSnapshot(Base): @@ -105,6 +113,9 @@ class Notification(Base): __tablename__ = "notifications" id: Mapped[int] = mapped_column(primary_key=True) + offer_id: Mapped[Optional[int]] = mapped_column(Integer, nullable=True) + event_key: Mapped[Optional[str]] = mapped_column(String(128), nullable=True, index=True) + event_context: Mapped[Optional[dict]] = mapped_column(JSON, nullable=True) listing_id: Mapped[Optional[int]] = mapped_column( ForeignKey("listings.id", ondelete="SET NULL"), nullable=True ) @@ -128,6 +139,67 @@ class AppSetting(Base): value: Mapped[str] = mapped_column(Text, default="") +class SchemaVersion(Base): + __tablename__ = "schema_versions" + id: Mapped[int] = mapped_column(primary_key=True) + version: Mapped[int] = mapped_column(Integer, default=1) + legacy_install: Mapped[bool] = mapped_column(Boolean, default=False) + + +class ListingOffer(Base): + __tablename__ = "listing_offers" + __table_args__ = (UniqueConstraint("listing_id", "offer_key", name="uq_listing_offer"),) + id: Mapped[int] = mapped_column(primary_key=True) + listing_id: Mapped[int] = mapped_column(ForeignKey("listings.id"), index=True) + offer_key: Mapped[str] = mapped_column(String(200)) + name: Mapped[str] = mapped_column(Text, default="") + attributes: Mapped[dict] = mapped_column(JSON, default=dict) + availability: Mapped[str] = mapped_column(String(20), default="unknown") + seen_in_latest_detail: Mapped[bool] = mapped_column(Boolean, default=True) + last_notified_score: Mapped[Optional[float]] = mapped_column(Float, nullable=True) + last_notified_amount: Mapped[Optional[int]] = mapped_column(Integer, nullable=True) + last_notified_source: Mapped[Optional[str]] = mapped_column(String(30), nullable=True) + last_notified_task_version: Mapped[Optional[int]] = mapped_column(Integer, nullable=True) + effective_price_state: Mapped[str] = mapped_column(String(20), default="unknown") + effective_price_cents: Mapped[Optional[int]] = mapped_column(Integer, nullable=True) + effective_price_source: Mapped[Optional[str]] = mapped_column(String(30), nullable=True) + price_evidence_fingerprint: Mapped[Optional[str]] = mapped_column(String(64), nullable=True) + price_valid_until: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True) + price_task_version: Mapped[Optional[int]] = mapped_column(Integer, nullable=True) + observations: Mapped[list["OfferObservation"]] = relationship(cascade="all, delete-orphan") + quotes: Mapped[list["ManualQuote"]] = relationship(cascade="all, delete-orphan") + evaluations: Mapped[list["OfferEvaluationRecord"]] = relationship(cascade="all, delete-orphan") + + +class OfferObservation(Base): + __tablename__ = "offer_observations" + id: Mapped[int] = mapped_column(primary_key=True) + offer_id: Mapped[int] = mapped_column(ForeignKey("listing_offers.id"), index=True) + fingerprint: Mapped[str] = mapped_column(String(64)) + payload: Mapped[dict] = mapped_column(JSON, default=dict) + observed_at: Mapped[datetime] = mapped_column(DateTime, default=_now) + + +class ManualQuote(Base): + __tablename__ = "manual_quotes" + id: Mapped[int] = mapped_column(primary_key=True) + offer_id: Mapped[int] = mapped_column(ForeignKey("listing_offers.id"), index=True) + amount_cents: Mapped[int] = mapped_column(Integer) + confirmed_at: Mapped[datetime] = mapped_column(DateTime) + expires_at: Mapped[datetime] = mapped_column(DateTime) + evidence_fingerprint: Mapped[str] = mapped_column(String(64)) + revoked: Mapped[bool] = mapped_column(Boolean, default=False) + + +class OfferEvaluationRecord(Base): + __tablename__ = "offer_evaluations" + id: Mapped[int] = mapped_column(primary_key=True) + offer_id: Mapped[int] = mapped_column(ForeignKey("listing_offers.id"), index=True) + task_version: Mapped[int] = mapped_column(Integer) + payload: Mapped[dict] = mapped_column(JSON, default=dict) + evaluated_at: Mapped[datetime] = mapped_column(DateTime, default=_now) + + class Seller(Base): __tablename__ = "sellers" __table_args__ = ( @@ -145,3 +217,88 @@ class Seller(Base): positive_rate: Mapped[Optional[float]] = mapped_column(Float, nullable=True) last_fetched_at: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True) blocked: Mapped[bool] = mapped_column(default=False) + + +class MediaAsset(Base): + __tablename__ = "media_assets" + id: Mapped[int] = mapped_column(primary_key=True) + listing_id: Mapped[int] = mapped_column(ForeignKey("listings.id"), index=True) + key: Mapped[str] = mapped_column(String(64)) + kind: Mapped[str] = mapped_column(String(10)) + offer_key: Mapped[Optional[str]] = mapped_column(String(200), nullable=True) + source_locator: Mapped[str] = mapped_column(Text, default="") + source_url: Mapped[str] = mapped_column(Text, default="") + parent_video_key: Mapped[Optional[str]] = mapped_column(String(64), nullable=True) + timestamp_ms: Mapped[Optional[int]] = mapped_column(Integer, nullable=True) + content_hash: Mapped[Optional[str]] = mapped_column(String(64), nullable=True) + relative_path: Mapped[Optional[str]] = mapped_column(Text, nullable=True) + status: Mapped[str] = mapped_column(String(30), default="pending") + failure: Mapped[Optional[str]] = mapped_column(Text, nullable=True) + observed_at: Mapped[datetime] = mapped_column(DateTime, default=_now) + + +class TaskRun(Base): + __tablename__ = "task_runs" + id: Mapped[int] = mapped_column(primary_key=True) + task_id: Mapped[int] = mapped_column(Integer, index=True) + task_version: Mapped[int] = mapped_column(Integer) + kind: Mapped[str] = mapped_column(String(30), default="crawl") + target_id: Mapped[Optional[int]] = mapped_column(Integer, nullable=True) + state: Mapped[str] = mapped_column(String(30), default="queued", index=True) + followup: Mapped[bool] = mapped_column(default=False) + cursor: Mapped[dict] = mapped_column(JSON, default=dict) + created_at: Mapped[datetime] = mapped_column(DateTime, default=_now) + started_at: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True) + deadline_at: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True) + + +class WorkItem(Base): + __tablename__ = "work_items" + __table_args__ = (UniqueConstraint("run_id", "stage", "input_fingerprint", name="uq_work_input"),) + id: Mapped[int] = mapped_column(primary_key=True) + run_id: Mapped[int] = mapped_column(ForeignKey("task_runs.id"), index=True) + task_id: Mapped[int] = mapped_column(Integer, index=True) + stage: Mapped[str] = mapped_column(String(20), index=True) + input_fingerprint: Mapped[str] = mapped_column(String(64), default="") + payload: Mapped[dict] = mapped_column(JSON, default=dict) + output: Mapped[Optional[dict]] = mapped_column(JSON, nullable=True) + state: Mapped[str] = mapped_column(String(30), default="pending", index=True) + attempt: Mapped[int] = mapped_column(Integer, default=0) + lease_id: Mapped[Optional[str]] = mapped_column(String(64), nullable=True, index=True) + lease_until: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True) + not_before: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True) + started_at: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True) + finished_at: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True) + + +class AnalysisCache(Base): + __tablename__ = "analysis_cache" + key: Mapped[str] = mapped_column(String(64), primary_key=True) + payload: Mapped[dict] = mapped_column(JSON) + created_at: Mapped[datetime] = mapped_column(DateTime, default=_now) + + +class NotificationEvent(Base): + __tablename__ = "notification_events" + id: Mapped[int] = mapped_column(primary_key=True) + business_key: Mapped[str] = mapped_column(String(128), unique=True) + task_id: Mapped[int] = mapped_column(Integer, index=True) + task_version: Mapped[int] = mapped_column(Integer) + offer_id: Mapped[int] = mapped_column(Integer) + listing_id: Mapped[int] = mapped_column(Integer) + payload: Mapped[dict] = mapped_column(JSON, default=dict) + state: Mapped[str] = mapped_column(String(30), default="pending") + created_at: Mapped[datetime] = mapped_column(DateTime, default=_now) + + +class NotificationDelivery(Base): + __tablename__ = "notification_deliveries" + __table_args__ = (UniqueConstraint("event_id", "channel", name="uq_delivery_channel"),) + id: Mapped[int] = mapped_column(primary_key=True) + event_id: Mapped[int] = mapped_column(ForeignKey("notification_events.id"), index=True) + channel: Mapped[str] = mapped_column(String(50)) + state: Mapped[str] = mapped_column(String(30), default="pending", index=True) + attempt: Mapped[int] = mapped_column(Integer, default=0) + lease_id: Mapped[Optional[str]] = mapped_column(String(64), nullable=True) + lease_until: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True) + not_before: Mapped[Optional[datetime]] = mapped_column(DateTime, nullable=True) diff --git a/goodprice/notify/base.py b/goodprice/notify/base.py index 739a226..edfd752 100644 --- a/goodprice/notify/base.py +++ b/goodprice/notify/base.py @@ -1,6 +1,10 @@ from dataclasses import dataclass +class DeliveryRejected(RuntimeError): + """The channel rejected the request or no transmission was attempted.""" + + @dataclass class NotificationMessage: title: str diff --git a/goodprice/notify/feishu.py b/goodprice/notify/feishu.py index 4176398..6f60432 100644 --- a/goodprice/notify/feishu.py +++ b/goodprice/notify/feishu.py @@ -6,7 +6,7 @@ import httpx -from goodprice.notify.base import NotificationMessage, Notifier +from goodprice.notify.base import DeliveryRejected, NotificationMessage, Notifier class FeishuNotifier(Notifier): @@ -32,7 +32,7 @@ def enabled(self) -> bool: def send(self, message: NotificationMessage) -> None: if not self.enabled: - raise RuntimeError("飞书机器人未配置 webhook") + raise DeliveryRejected("飞书机器人未配置 webhook") content = f"{message.title}\n{message.content}" if message.url: content += f"\n{message.url}" @@ -59,4 +59,4 @@ def send(self, message: NotificationMessage) -> None: result = response.json() code = result.get("code", result.get("StatusCode", -1)) if code != 0: - raise RuntimeError(f"飞书机器人发送失败: code={code}, msg={result.get('msg', '')}") + raise DeliveryRejected(f"飞书机器人发送失败: code={code}, msg={result.get('msg', '')}") diff --git a/goodprice/notify/gotify.py b/goodprice/notify/gotify.py index da69ac4..c373823 100644 --- a/goodprice/notify/gotify.py +++ b/goodprice/notify/gotify.py @@ -2,7 +2,7 @@ import httpx -from goodprice.notify.base import NotificationMessage, Notifier +from goodprice.notify.base import DeliveryRejected, NotificationMessage, Notifier class GotifyNotifier(Notifier): @@ -30,7 +30,7 @@ def enabled(self) -> bool: def send(self, message: NotificationMessage) -> None: if not self.enabled: - raise RuntimeError("Gotify 未配置 URL 或应用 Token") + raise DeliveryRejected("Gotify 未配置 URL 或应用 Token") endpoint = self.url if self.url.endswith("/message") else f"{self.url}/message" content = message.content if message.url: diff --git a/goodprice/notify/serverchan.py b/goodprice/notify/serverchan.py index 3c3a3db..ccb44a8 100644 --- a/goodprice/notify/serverchan.py +++ b/goodprice/notify/serverchan.py @@ -2,7 +2,7 @@ import httpx -from goodprice.notify.base import NotificationMessage, Notifier +from goodprice.notify.base import DeliveryRejected, NotificationMessage, Notifier SERVERCHAN_URL = "https://sctapi.ftqq.com/{key}.send" @@ -26,7 +26,7 @@ def enabled(self) -> bool: def send(self, message: NotificationMessage) -> None: if not self.enabled: - raise RuntimeError("Server酱未配置 sendkey") + raise DeliveryRejected("Server酱未配置 sendkey") with httpx.Client(transport=self._transport, timeout=self.timeout) as client: response = client.post( SERVERCHAN_URL.format(key=self.sendkey), @@ -35,4 +35,4 @@ def send(self, message: NotificationMessage) -> None: response.raise_for_status() data = response.json() if data.get("code") != 0: - raise RuntimeError(f"Server酱返回错误: {data}") + raise DeliveryRejected(f"Server酱返回错误: {data}") diff --git a/goodprice/notify/wecom_robot.py b/goodprice/notify/wecom_robot.py index dc88c1f..ed9291e 100644 --- a/goodprice/notify/wecom_robot.py +++ b/goodprice/notify/wecom_robot.py @@ -2,7 +2,7 @@ import httpx -from goodprice.notify.base import NotificationMessage, Notifier +from goodprice.notify.base import DeliveryRejected, NotificationMessage, Notifier class WeComRobotNotifier(Notifier): @@ -24,7 +24,7 @@ def enabled(self) -> bool: def send(self, message: NotificationMessage) -> None: if not self.enabled: - raise RuntimeError("企业微信群机器人未配置 webhook") + raise DeliveryRejected("企业微信群机器人未配置 webhook") content = f"{message.title}\n{message.content}\n{message.url}" with httpx.Client(transport=self._transport, timeout=self.timeout) as client: response = client.post( @@ -34,8 +34,8 @@ def send(self, message: NotificationMessage) -> None: data = response.json() errcode = data.get("errcode", -1) if errcode == 93000: - raise RuntimeError("企业微信群机器人 webhook 无效或机器人已被移除,请重新添加") + raise DeliveryRejected("企业微信群机器人 webhook 无效或机器人已被移除,请重新添加") if errcode == 93004: - raise RuntimeError("企业微信群机器人发送太频繁(20 条/分钟),请稍后重试或降低频率") + raise DeliveryRejected("企业微信群机器人发送太频繁(20 条/分钟),请稍后重试或降低频率") if errcode != 0: - raise RuntimeError(f"企业微信群机器人发送失败: {data}") + raise DeliveryRejected(f"企业微信群机器人发送失败: {data}") diff --git a/goodprice/offers/__init__.py b/goodprice/offers/__init__.py new file mode 100644 index 0000000..a5dfebc --- /dev/null +++ b/goodprice/offers/__init__.py @@ -0,0 +1 @@ +"""Offer identity, observed prices and buyer-confirmed quotes.""" diff --git a/goodprice/offers/evaluation.py b/goodprice/offers/evaluation.py new file mode 100644 index 0000000..d33cd71 --- /dev/null +++ b/goodprice/offers/evaluation.py @@ -0,0 +1,34 @@ +"""Pure offer gates shared by scheduling and final publication.""" +from dataclasses import asdict + +from goodprice.offers.matching import match_variant +from goodprice.offers.pricing import decide_price + + +def assess_offer(task, listing, candidate, quote, now, *, vision_enabled, + condition_score=None, requirement_match=None, visual_valid=False, ready=True): + ttl = task.interval_minutes * 60 if task.fetch_detail else 0 + decision = decide_price(candidate, quote, now, ttl) + match = match_variant(task.target_variant, candidate.attributes, listing.title) + eligible = match == "matched" and decision.state == "exact" and candidate.availability != "unavailable" + reason = decision.reason if not eligible else "" + if decision.amount_cents is not None: + if decision.amount_cents < round(task.min_price * 100) or (task.max_price and decision.amount_cents > round(task.max_price * 100)): + eligible, reason = False, "该规格价格不在预算内" + if match != "matched": + eligible, reason = False, "目标规格不匹配" if match == "mismatched" else "规格待核验" + if not task.enabled or listing.blocked: + eligible, reason = False, "任务已停用或商品已拉黑" + price_eligible = eligible + if not vision_enabled or not visual_valid or type(condition_score) is not int or not 0 <= condition_score <= 10: + condition_score = None + if task.min_condition_score and (not vision_enabled or not visual_valid or condition_score is None + or condition_score < task.min_condition_score): + eligible, reason = False, "品相未核验或低于门槛" + if requirement_match is False: + eligible, reason = False, "需求不匹配" + if not ready: + eligible, reason = False, "分析尚未完成" + return {**asdict(decision), "fresh_until": decision.fresh_until.isoformat() if decision.fresh_until else None, + "match": match, "eligible": eligible, "price_eligible": price_eligible, "reason": reason, + "condition_score": condition_score, "requirement_match": requirement_match} diff --git a/goodprice/offers/matching.py b/goodprice/offers/matching.py new file mode 100644 index 0000000..de07438 --- /dev/null +++ b/goodprice/offers/matching.py @@ -0,0 +1,52 @@ +import hashlib +import json +import re +import unicodedata +from typing import Mapping + +from goodprice.offers.types import MatchState + + +def _normal(value: str) -> str: + return re.sub(r"\s+", "", unicodedata.normalize("NFKC", value).casefold()) + + +def stable_offer_key(platform_sku_id: str | None, attributes: Mapping[str, str]) -> str | None: + if platform_sku_id: + return f"sku:{platform_sku_id.strip()}" + normalized = {_normal(k): _normal(v) for k, v in attributes.items() if k.strip() and v.strip()} + if not normalized: + return None + raw = json.dumps(normalized, sort_keys=True, ensure_ascii=False).encode() + return "attrs:" + hashlib.sha256(raw).hexdigest()[:32] + + +def match_variant(target: str, attributes: Mapping[str, str], title: str) -> MatchState: + """Only attest explicit attributes, not broad listing titles or guessed negatives.""" + if not target.strip(): + return "matched" + values = {_normal(v) for v in attributes.values()} + if not values: + return "uncertain" + uncertain = False + capacities = re.findall(r"\d+(?:\.\d+)?\s*(?:gb|tb)", target, re.I) + remainder = re.sub(r"\d+(?:\.\d+)?\s*(?:gb|tb)", "/", target, flags=re.I) + for requirement in [*re.split(r"[/,,;;]+", remainder), *capacities]: + term = _normal(requirement) + if not term: + continue + if term.startswith(("非", "不要", "不含", "无")): + uncertain = True + elif term in values: + continue + elif re.fullmatch(r"\d+(?:\.\d+)?(?:gb|tb)", term): + if any(re.fullmatch(r"\d+(?:\.\d+)?(?:gb|tb)", v) for v in values): + return "mismatched" + uncertain = True + elif any(k in ("型号", "model") for k in attributes): + model = next((v for k, v in attributes.items() if k in ("型号", "model")), "") + if _normal(model) != term: + return "mismatched" + else: + uncertain = True + return "uncertain" if uncertain else "matched" diff --git a/goodprice/offers/pricing.py b/goodprice/offers/pricing.py new file mode 100644 index 0000000..2d2ae3c --- /dev/null +++ b/goodprice/offers/pricing.py @@ -0,0 +1,85 @@ +import hashlib +import json +import re +from datetime import datetime, timedelta, timezone +from decimal import Decimal, InvalidOperation + +from goodprice.offers.types import OfferCandidate, PriceDecision, QuoteInput + + +def utc(value: datetime) -> datetime: + return value.replace(tzinfo=timezone.utc) if value.tzinfo is None else value.astimezone(timezone.utc) + + +def parse_money_cents(raw: str) -> int: + text = str(raw).strip().removeprefix("¥").removeprefix("¥").removesuffix("元").strip() + if not re.fullmatch(r"(?:\d+|\d{1,3}(?:,\d{3})+)(?:\.\d{1,2})?", text): + raise ValueError("价格必须是非负人民币金额,最多两位小数") + try: + cents = Decimal(text.replace(",", "")) * 100 + if not cents.is_finite() or cents < 0 or cents > 10**12: + raise ValueError("价格超出支持范围") + return int(cents) + except InvalidOperation as exc: + raise ValueError("无效价格") from exc + + +def offer_fingerprint(offer: OfferCandidate) -> str: + # Capture time is deliberately excluded: renewing an unchanged observation + # must not invalidate a quote or cause another paid analysis/notification. + claims = sorted((c.source, c.kind, c.text, c.amount_cents, c.min_cents, c.max_cents, + c.scope_key or "") for c in offer.claims) + raw = {"key": offer.key, "attributes": dict(offer.attributes), + "availability": offer.availability, "claims": claims} + return hashlib.sha256(json.dumps(raw, sort_keys=True, ensure_ascii=False).encode()).hexdigest() + + +def decide_price(offer: OfferCandidate, quote: QuoteInput | None, + now: datetime, ttl_seconds: int) -> PriceDecision: + now = utc(now) + claims = [c for c in offer.claims if c.scope_key in (None, "", offer.key)] + ids = tuple(c.evidence_id for c in claims) + + def result(state, reason, amount=None, source=None, until=None): + return PriceDecision(state, amount, source, ids, reason, until) + + if ttl_seconds <= 0 or not claims: + return result("unknown", "无有效价格证据") + if offer.availability == "unavailable": + return result("unknown", "该规格不可购买") + if any(c.evidence_id == "duplicate" for c in claims): + return result("conflict", "规格身份存在歧义,人工价格不能代替身份核验") + if any(c.kind == "unknown" and c.evidence_id == "truncated" for c in claims): + return result("unknown", "价格或规格证据不完整") + until = min(utc(c.observed_at) for c in claims) + timedelta(seconds=ttl_seconds) + if any(utc(c.observed_at) > now + timedelta(seconds=5) for c in claims) or until <= now: + return result("unknown", "价格或规格证据已过期,等待重新核验") + if quote and not quote.revoked and utc(quote.expires_at) > now: + if quote.offer_key != offer.key or quote.evidence_fingerprint != offer_fingerprint(offer): + return result("conflict", "页面证据已变化,请复核人工报价") + if utc(quote.confirmed_at) <= now and type(quote.amount_cents) is int and quote.amount_cents > 0: + return result("exact", "人工确认报价", quote.amount_cents, "manual", + min(until, utc(quote.expires_at))) + return result("unknown", "无效的人工报价") + for state in ("conflict", "inquiry"): + if any(c.kind == state for c in claims): + return result(state, "价格声明存在冲突" if state == "conflict" else "需联系卖家询价") + if any(c.kind == "unknown" for c in claims): + return result("unknown", "价格或规格证据不完整") + exact = [c for c in claims if c.kind == "exact" and c.source in {"sku", "dom", "description"} + and type(c.amount_cents) is int and 0 < c.amount_cents <= 10**12] + amounts = {c.amount_cents for c in exact} + if len(amounts) > 1: + return result("conflict", "同一规格有多个不一致价格") + if len(amounts) == 1: + return result("exact", "该规格有明确页面价格", exact[0].amount_cents, exact[0].source, until) + if any(c.kind == "range_or_from" for c in claims): + return result("range_or_from", "只有起售价或价格区间") + return result("unknown", "缺少可核验的明确价格") + + +def is_comparable_drop(previous: PriceDecision, current: PriceDecision, same_offer: bool) -> bool: + return bool(same_offer and previous.state == current.state == "exact" + and previous.source == current.source + and previous.amount_cents is not None and current.amount_cents is not None + and 0 < current.amount_cents < previous.amount_cents) diff --git a/goodprice/offers/service.py b/goodprice/offers/service.py new file mode 100644 index 0000000..ccc0013 --- /dev/null +++ b/goodprice/offers/service.py @@ -0,0 +1,119 @@ +from contextlib import nullcontext +from datetime import datetime, timezone + +from goodprice.models import ( + Listing, + ListingOffer, + ManualQuote, + Notification, + OfferObservation, + WatchTask, +) +from goodprice.offers.pricing import decide_price, offer_fingerprint, utc +from goodprice.offers.types import OfferCandidate, PriceClaim, QuoteInput + + +def encode_offer(offer: OfferCandidate) -> dict: + return {"key": offer.key, "attributes": dict(offer.attributes), "name": offer.name, + "availability": offer.availability, + "claims": [{**vars(c), "observed_at": utc(c.observed_at).isoformat()} for c in offer.claims]} + + +def decode_offer(payload: dict) -> OfferCandidate: + return OfferCandidate(payload["key"], payload["attributes"], payload["availability"], + tuple(PriceClaim(**{**c, "observed_at": datetime.fromisoformat(c["observed_at"])}) + for c in payload["claims"]), payload.get("name", "")) + + +class OfferService: + def __init__(self, session_factory): + self._session_factory = session_factory + + def _refresh_price_index(self, session, row, listing, candidate, quote=None, now=None): + task = session.get(WatchTask, listing.task_id) if listing.task_id else None + if quote is None: + quote = session.query(ManualQuote).filter_by(offer_id=row.id, revoked=False).order_by(ManualQuote.id.desc()).first() + value = None if quote is None else QuoteInput(row.offer_key, quote.amount_cents, + utc(quote.confirmed_at), utc(quote.expires_at), quote.evidence_fingerprint, quote.revoked) + ttl = task.interval_minutes * 60 if task and task.fetch_detail else 0 + decision = decide_price(candidate, value, now or datetime.now(timezone.utc), ttl) + row.effective_price_state, row.effective_price_cents = decision.state, decision.amount_cents + row.effective_price_source, row.price_evidence_fingerprint = decision.source, offer_fingerprint(candidate) + row.price_valid_until = utc(decision.fresh_until).replace(tzinfo=None) if decision.fresh_until else None + row.price_task_version = task.version if task else None + + def record_observation(self, listing_id: int, offer: OfferCandidate, *, session=None, now=None) -> int: + if not offer.key or not offer.claims: + raise ValueError("规格或证据不能为空") + own = session is None + with self._session_factory() if own else nullcontext(session) as session: + listing = session.get(Listing, listing_id) + if listing is None: + raise LookupError("商品不存在") + row = session.query(ListingOffer).filter_by(listing_id=listing_id, offer_key=offer.key).first() + if row is None: + row = ListingOffer(listing_id=listing_id, offer_key=offer.key) + legacy_sent = session.query(Notification).filter_by( + listing_id=listing_id, offer_id=None, status="sent").first() + if legacy_sent and offer.key == "default": + row.last_notified_source = "legacy_baseline" + session.add(row) + session.flush() + row.name, row.attributes, row.availability = offer.name, dict(offer.attributes), offer.availability + row.seen_in_latest_detail = True + session.add(OfferObservation(offer_id=row.id, fingerprint=offer_fingerprint(offer), + payload=encode_offer(offer), + observed_at=min(utc(c.observed_at) for c in offer.claims))) + session.flush() + self._refresh_price_index(session, row, listing, offer, now=now) + if own: + session.commit() + return row.id + + def confirm_quote(self, offer_id: int, quote: QuoteInput) -> int: + now = datetime.now(timezone.utc) + if (type(quote.amount_cents) is not int or not 0 < quote.amount_cents <= 10**12 + or utc(quote.expires_at) <= now or utc(quote.confirmed_at) > now + or utc(quote.expires_at) <= utc(quote.confirmed_at) or quote.revoked): + raise ValueError("报价金额、确认时间或有效期无效") + with self._session_factory() as session: + offer = session.get(ListingOffer, offer_id) + if offer is None: + raise LookupError("规格不存在") + latest = session.query(OfferObservation).filter_by(offer_id=offer_id).order_by( + OfferObservation.observed_at.desc(), OfferObservation.id.desc()).first() + listing = session.get(Listing, offer.listing_id) + candidate = decode_offer(latest.payload) if latest else None + if candidate and any(c.evidence_id in ("duplicate", "truncated") for c in candidate.claims): + raise ValueError("规格身份不明确或获取不完整,不能用报价代替核验") + task = session.get(WatchTask, listing.task_id) if listing.task_id else None + ttl = (task.interval_minutes if task else 20) * 60 + if (not offer.seen_in_latest_detail or latest is None or latest.fingerprint != quote.evidence_fingerprint + or offer.offer_key != quote.offer_key or offer.availability == "unavailable" + or (now - utc(latest.observed_at)).total_seconds() >= ttl): + raise ValueError("证据已变化或过期,请刷新后重新确认") + for old in session.query(ManualQuote).filter_by(offer_id=offer_id, revoked=False): + old.revoked = True + row = ManualQuote(offer_id=offer_id, amount_cents=quote.amount_cents, + confirmed_at=utc(quote.confirmed_at), expires_at=utc(quote.expires_at), + evidence_fingerprint=quote.evidence_fingerprint, revoked=False) + session.add(row) + listing.offer_verified_until = None + self._refresh_price_index(session, offer, listing, candidate, quote=row) + session.commit() + return row.id + + def revoke_quote(self, quote_id: int) -> None: + with self._session_factory() as session: + quote = session.get(ManualQuote, quote_id) + if quote is None: + raise LookupError("报价不存在") + quote.revoked = True + offer = session.get(ListingOffer, quote.offer_id) + listing = session.get(Listing, offer.listing_id) + listing.offer_verified_until = None + session.flush() + observation = session.query(OfferObservation).filter_by(offer_id=offer.id).order_by(OfferObservation.id.desc()).first() + if observation: + self._refresh_price_index(session, offer, listing, decode_offer(observation.payload)) + session.commit() diff --git a/goodprice/offers/types.py b/goodprice/offers/types.py new file mode 100644 index 0000000..0ba37c9 --- /dev/null +++ b/goodprice/offers/types.py @@ -0,0 +1,62 @@ +from dataclasses import dataclass, field +from datetime import datetime +from types import MappingProxyType +from typing import Literal, Mapping + +PriceState = Literal["exact", "range_or_from", "inquiry", "conflict", "unknown"] +MatchState = Literal["matched", "mismatched", "uncertain"] + + +@dataclass(frozen=True) +class PriceClaim: + source: str + kind: PriceState + text: str + observed_at: datetime + evidence_id: str + amount_cents: int | None = None + min_cents: int | None = None + max_cents: int | None = None + scope_key: str | None = None + + +@dataclass(frozen=True) +class OfferCandidate: + key: str + attributes: Mapping[str, str] = field(default_factory=dict) + availability: Literal["available", "unavailable", "unknown"] = "unknown" + claims: tuple[PriceClaim, ...] = () + name: str = "" + + def __post_init__(self): + object.__setattr__(self, "attributes", MappingProxyType(dict(self.attributes))) + object.__setattr__(self, "claims", tuple(self.claims)) + + +@dataclass(frozen=True) +class QuoteInput: + offer_key: str + amount_cents: int + confirmed_at: datetime + expires_at: datetime + evidence_fingerprint: str + revoked: bool = False + + +@dataclass(frozen=True) +class PriceDecision: + state: PriceState + amount_cents: int | None = None + source: str | None = None + evidence_ids: tuple[str, ...] = () + reason: str = "" + fresh_until: datetime | None = None + + +@dataclass(frozen=True) +class OfferEvaluation: + offer_key: str + match: MatchState + price: PriceDecision + eligible: bool + reason: str = "" diff --git a/goodprice/resources.py b/goodprice/resources.py new file mode 100644 index 0000000..bb875ad --- /dev/null +++ b/goodprice/resources.py @@ -0,0 +1,144 @@ +"""Local, read-only capability detection. Recommendations never apply themselves.""" +import platform +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path, PurePosixPath + + +@dataclass(frozen=True) +class ResourceSnapshot: + effective_cpu: float | None + memory_limit_bytes: int | None + available_memory_bytes: int | None + source: str + warnings: tuple[str, ...] + detected_at: datetime + + +def resource_profile(name): + if name not in {"low", "balanced"}: + raise ValueError("Unknown resource profile") + return {"execution_active_runs": 2, "execution_crawler_workers": 1, + "execution_model_workers": 2 if name == "balanced" else 1, + "execution_media_workers": 1, "media_ffmpeg_threads": 1} + + +def recommend_profile(snapshot): + balanced = (snapshot.source != "unknown" and (snapshot.effective_cpu or 0) >= 4 + and (snapshot.memory_limit_bytes or 0) >= 6 * 1024**3 + and (snapshot.available_memory_bytes or 0) >= 3 * 1024**3) + name = "balanced" if balanced else "low" + return {"name": name, "values": resource_profile(name), + "reason": "有效CPU及内存余量足够,可选择均衡" if balanced else "设备余量较小或检测不完整,建议低占用"} + + +def _cpus(raw): + values = set() + for part in raw.strip().split(","): + if not part: + continue + bounds = part.split("-") + start, end = int(bounds[0]), int(bounds[-1]) + if start < 0 or end < start or end - start > 65536: + raise ValueError() + values.update(range(start, end + 1)) + return values + + +def _unescape(raw): + for escaped, value in [("\\040", " "), ("\\011", "\t"), ("\\134", "\\")]: + raw = raw.replace(escaped, value) + return raw + + +def detect_resources(*, system=None, read=None, host=None, affinity=None): + system = system or platform.system() + read = read or (lambda p: Path(p).read_text(encoding="utf-8")) + warnings = [] + if host is None: + try: + import psutil + memory = psutil.virtual_memory() + host = (psutil.cpu_count(logical=True), memory.total, memory.available) + try: + affinity = set(psutil.Process().cpu_affinity()) + except (AttributeError, NotImplementedError, psutil.Error): + pass + except (ImportError, OSError): + host = (None, None, None) + warnings.append("系统资源检测不可用") + cpu, memory_limit, available = host + if affinity: + cpu = min(cpu, len(affinity)) if cpu else len(affinity) + source = "host" if all(v is not None for v in host) else "unknown" + def optional(path): + try: + return read(str(path)).strip() + except (OSError, KeyError, ValueError): + return "" + if system == "Linux": + groups = optional("/proc/self/cgroup") + mounts = optional("/proc/self/mountinfo") + if not groups or not mounts: + source = "unknown" + warnings.append("无法读取容器层级,保守推荐") + else: + mappings = [] + for line in groups.splitlines(): + parts = line.split(":", 2) + if len(parts) == 3: + mappings.append((set(parts[1].split(",")), PurePosixPath(parts[2]))) + for line in mounts.splitlines(): + fields = line.split() + try: + sep = fields.index("-") + fs = fields[sep+1] + if fs not in {"cgroup", "cgroup2"}: + continue + mount_root, mount = PurePosixPath(_unescape(fields[3])), PurePosixPath(_unescape(fields[4])) + controllers = set(fields[sep+3].split(",")) + for names, process in mappings: + if fs == "cgroup" and not (controllers & names): + continue + if fs == "cgroup2" and names != {""}: + continue + if not process.is_relative_to(mount_root): + continue + directory = mount / process.relative_to(mount_root) + for ancestor in (directory, *directory.parents): + if not ancestor.is_relative_to(mount): + break + quota = optional(ancestor / "cpu.max") if fs == "cgroup2" else " ".join([ + optional(ancestor / "cpu.cfs_quota_us"), optional(ancestor / "cpu.cfs_period_us")]) + parts = quota.split() + if len(parts) == 2 and parts[0] not in {"max", "-1"}: + limit = int(parts[0]) / int(parts[1]) + if limit > 0: + cpu = min(cpu, limit) if cpu else limit + source = "cgroup" + cpuset = optional(ancestor / ("cpuset.cpus.effective" if fs == "cgroup2" else "cpuset.cpus")) + if cpuset: + ids = _cpus(cpuset) + if affinity: + ids &= affinity + if ids: + cpu = min(cpu, len(ids)) if cpu else len(ids) + source = "cgroup" + raw_limit = optional(ancestor / ("memory.max" if fs == "cgroup2" else "memory.limit_in_bytes")) + raw_current = optional(ancestor / ("memory.current" if fs == "cgroup2" else "memory.usage_in_bytes")) + if raw_limit and raw_limit != "max": + limit = int(raw_limit) + if 0 < limit < 2**60: + memory_limit = min(memory_limit, limit) if memory_limit else limit + source = "cgroup" + if raw_current: + remaining = max(0, limit - int(raw_current)) + available = min(available, remaining) if available is not None else remaining + else: + available = 0 + warnings.append("容器内存占用不可读") + except (ValueError, IndexError, ZeroDivisionError): + source = "unknown" + warnings.append("容器资源格式无法解析") + return ResourceSnapshot(float(cpu) if cpu is not None else None, memory_limit, available, + source, tuple(dict.fromkeys(warnings)), datetime.now(timezone.utc)) diff --git a/goodprice/scheduler.py b/goodprice/scheduler.py index 8f5ea79..3d1247d 100644 --- a/goodprice/scheduler.py +++ b/goodprice/scheduler.py @@ -4,31 +4,35 @@ from apscheduler.triggers.interval import IntervalTrigger -def build_scheduler(session_factory, submit_fn, task_service) -> BackgroundScheduler: +def build_scheduler(session_factory, submit_fn, task_service, jitter_seconds=0) -> BackgroundScheduler: scheduler = BackgroundScheduler(timezone="Asia/Shanghai") scheduler.add_job( _sync_tasks, trigger=IntervalTrigger(minutes=5), - args=[session_factory, submit_fn, task_service, scheduler], + args=[session_factory, submit_fn, task_service, scheduler, jitter_seconds], id="sync_tasks", replace_existing=True, max_instances=1, ) - _sync_tasks(session_factory, submit_fn, task_service, scheduler) + _sync_tasks(session_factory, submit_fn, task_service, scheduler, jitter_seconds) return scheduler -def _sync_tasks(session_factory, submit_fn, task_service, scheduler) -> None: +def _sync_tasks(session_factory, submit_fn, task_service, scheduler, jitter_seconds=0) -> None: + jitter = max(0, float(jitter_seconds() if callable(jitter_seconds) else jitter_seconds)) enabled_ids = sorted(task.id for task in task_service.enabled_tasks()) job_ids = {job.id for job in scheduler.get_jobs()} for task_id in enabled_ids: job_id = f"crawl_{task_id}" + task = task_service.get_task(task_id) if job_id in job_ids: + job = scheduler.get_job(job_id) + if job.trigger.interval.total_seconds() != max(1, task.interval_minutes) * 60 or job.trigger.jitter != jitter: + scheduler.reschedule_job(job_id, trigger=IntervalTrigger(minutes=max(1, task.interval_minutes), jitter=jitter)) continue - task = task_service.get_task(task_id) scheduler.add_job( submit_fn, - trigger=IntervalTrigger(minutes=max(1, task.interval_minutes)), + trigger=IntervalTrigger(minutes=max(1, task.interval_minutes), jitter=jitter), args=[task_id], id=job_id, next_run_time=datetime.now(), diff --git a/goodprice/services/crawl_service.py b/goodprice/services/crawl_service.py index a305eab..eef041b 100644 --- a/goodprice/services/crawl_service.py +++ b/goodprice/services/crawl_service.py @@ -1,19 +1,11 @@ import logging -import random import threading -import time from datetime import datetime -from typing import Any, Optional +from typing import Any -from goodprice.crawler.base import ListingData +from goodprice.crawler.base import ListingData, ListingDetail from goodprice.crawler.parser import is_product_image -from goodprice.models import Listing, Notification, PriceSnapshot, WatchTask -from goodprice.notify.base import NotificationMessage -from goodprice.services.satisfaction import ( - DROP_PCT_DISPLAY_THRESHOLD, - compute_satisfaction, - drop_pct_from_snapshots, -) +from goodprice.models import Listing, PriceSnapshot, WatchTask logger = logging.getLogger(__name__) @@ -27,6 +19,7 @@ class TaskRunGuard: def __init__(self): self._running: set[int] = set() self._lock = threading.Lock() + self.provider = None def try_start(self, task_id: int) -> bool: with self._lock: @@ -41,7 +34,8 @@ def finish(self, task_id: int) -> None: def running_ids(self) -> set[int]: with self._lock: - return set(self._running) + running = set(self._running) + return running | (set(self.provider()) if self.provider else set()) class CrawlService: @@ -55,6 +49,7 @@ def __init__( settings_service, guard=None, seller_service=None, + media_service=None, ): self._session_factory = session_factory self.adapter = adapter @@ -64,6 +59,7 @@ def __init__( self.settings_service = settings_service self.guard = guard or TaskRunGuard() self.seller_service = seller_service + self.media_service = media_service self._judger = None def run_task(self, task_id: int) -> dict[str, Any]: @@ -71,164 +67,18 @@ def run_task(self, task_id: int) -> dict[str, Any]: logger.info("任务 %s 已在运行,忽略重复触发", task_id) return {"found": 0, "new": 0, "notified": 0, "skipped": "already_running"} try: - return self._run_impl(task_id) - finally: - self.guard.finish(task_id) + from goodprice.services.offer_crawl import OfferCrawlRunner + from goodprice.services.offer_pipeline import StaleEvaluation - def _run_impl(self, task_id: int) -> dict[str, Any]: - stats = { - "found": 0, - "new": 0, - "notified": 0, - "backfilled": 0, - "reevaluated": 0, - "gone": 0, - } - settings = self.settings_service.get() - self._judger = self._build_judger(settings) - jitter = int(settings.default_crawl_jitter_minutes) - if jitter: - time.sleep(random.uniform(0, jitter * 60)) - with self._session_factory() as session: - task = session.get(WatchTask, task_id) - if task is None: - raise RuntimeError(f"任务 {task_id} 不存在") - task.last_run_at = datetime.now() - task.last_error = None - task.last_run_count = (task.last_run_count or 0) + 1 - session.commit() - try: - items = self.adapter.search(task.keyword) + try: + return OfferCrawlRunner(self).run(task_id) + except StaleEvaluation: + return {"found": 0, "new": 0, "notified": 0, "skipped": "superseded"} except Exception as exc: - self._record_error(task_id, f"抓取失败: {exc}") + self._record_error(task_id, f"处理商品时出错: {exc}"[:1000]) raise - stats["found"] = len(items) - logger.info("任务 %s 搜索命中 %s 条", task_id, len(items)) - batch_rows: list[dict] = [] - pending: list[tuple] = [] # (task, listing, old_price, is_renotify) - seen_ids: set[int] = set() - with self._session_factory() as session: - task = session.get(WatchTask, task_id) - try: - for data in items: - if task.max_price and data.price > task.max_price: - logger.info("任务 %s 跳过(超最高价 %s): %s ¥%s", task_id, task.max_price, data.title[:30], data.price) - continue - if task.min_price and data.price < task.min_price: - logger.info("任务 %s 跳过(低于价格下限 %s): %s ¥%s", task_id, task.min_price, data.title[:30], data.price) - continue - if self._excluded(task, data.title): - logger.info("任务 %s 跳过(命中排除词): %s", task_id, data.title[:40]) - continue - old_price: Optional[float] = None - existing = ( - session.query(Listing) - .filter( - Listing.platform == task.platform, - Listing.external_id == data.external_id, - Listing.task_id == task.id, - ) - .first() - ) - if existing is not None and abs(existing.price - data.price) > 0.001: - old_price = existing.price - listing, is_new = self._upsert_listing(session, task, data) - seen_ids.add(listing.id) - if self._is_blocked(session, listing): - session.commit() - continue - listing.status = "active" - listing.missed_count = 0 - if is_new: - stats["new"] += 1 - logger.info("任务 %s 新品 %s:%s ¥%s", task_id, data.external_id, data.title[:30], data.price) - if task.fetch_detail: - self._fetch_detail(session, listing) - reason = self._evaluate_and_queue( - session, task, listing, old_price, batch_rows, pending, False - ) - if reason: - logger.info("任务 %s %s,不收录:%s", task_id, reason, listing.title[:30]) - session.commit() - continue - else: - changed = old_price is not None or listing.status == "gone" - if changed: - stats["reevaluated"] += 1 - logger.info("任务 %s 重评 %s:%s(价格变化或重新上架)", task_id, data.external_id, data.title[:30]) - reason = self._evaluate_and_queue( - session, task, listing, old_price, batch_rows, pending, True - ) - if reason: - session.commit() - continue - else: - if self._backfill(session, listing, task): - stats["backfilled"] += 1 - session.commit() - # 下架软标记:本轮未出现的该任务商品 - for other in ( - session.query(Listing) - .filter(Listing.task_id == task.id) - .filter(~Listing.id.in_(seen_ids)) - ): - other.missed_count = (other.missed_count or 0) + 1 - if other.missed_count >= GONE_THRESHOLD and other.status != "gone": - other.status = "gone" - stats["gone"] += 1 - logger.info("任务 %s 商品 %s 标记为已下架", task_id, other.title[:30]) - session.commit() - # 批性价比:本批通过筛选的商品统一横向对比 - if batch_rows and self._value_client() is not None: - self._batch_value(session, batch_rows) - session.commit() - # 通知:新品在批性价比后统一发出;重评仅满意度提高时发出 - for task, listing, old_price, is_renotify in pending: - satisfaction = self._satisfaction(listing) - listing.satisfaction = satisfaction - if ( - is_renotify - and listing.last_notified_satisfaction is not None - and satisfaction <= listing.last_notified_satisfaction - ): - continue - self._notify( - session, - task, - listing, - satisfaction, - old_price=old_price, - is_renotify=is_renotify, - ) - stats["notified"] += 1 - session.commit() - except Exception as exc: - task.last_error = f"处理商品时出错: {exc}"[:1000] - session.commit() - raise - session.commit() - return stats - - def _evaluate_and_queue( - self, - session, - task: WatchTask, - listing: Listing, - old_price: Optional[float], - batch_rows: list[dict], - pending: list[tuple], - is_renotify: bool, - ) -> Optional[str]: - """需求→品相→门槛→卖家评估并入队;不通过时返回原因(调用方需 commit 后跳过)。""" - if not self._requirement_pass(session, listing, task): - return "需求不匹配" - self._condition_analysis(session, listing, task) - if self._condition_gate_fails(task, listing): - return "品相分低于门槛" - self._seller_check(session, listing, task) - batch_rows.append(self._batch_row(listing, task.condition_requirement or "")) - pending.append((task, listing, old_price, is_renotify)) - return None + finally: + self.guard.finish(task_id) def _upsert_listing(self, session, task: WatchTask, data: ListingData): listing = ( @@ -261,8 +111,44 @@ def _upsert_listing(self, session, task: WatchTask, data: ListingData): listing.price = data.price session.add(PriceSnapshot(listing_id=listing.id, price=data.price)) listing.last_seen_at = datetime.now() + if listing.title != data.title: + from goodprice.models import ( + ListingOffer, + NotificationDelivery, + NotificationEvent, + OfferEvaluationRecord, + ) + listing.requirement_match, listing.requirement_reason = None, None + listing.condition_score, listing.condition_detail, listing.value_score = None, None, None + listing.satisfaction, listing.best_of_batch, listing.offer_verified_until = 0, False, None + for offer in session.query(ListingOffer).filter_by(listing_id=listing.id): + latest = session.query(OfferEvaluationRecord).filter_by(offer_id=offer.id).order_by(OfferEvaluationRecord.id.desc()).first() + if latest: + old = {k: v for k, v in latest.payload.items() if k != "run_id"} + session.add(OfferEvaluationRecord(offer_id=offer.id, task_version=task.version, + payload={**old, "status": "source_changed", "eligible": False, "value_score": None, + "condition_score": None, "requirement_match": None, "media": None, + "reason": "标题已变化,等待重新核验"})) + events = session.query(NotificationEvent.id).filter_by(listing_id=listing.id) + session.query(NotificationDelivery).filter(NotificationDelivery.event_id.in_(events), + NotificationDelivery.state.in_(("pending", "failed", "retry_requested", "awaiting_revalidation"))).update( + {"state": "superseded"}, synchronize_session=False) + listing.title, listing.url = data.title, data.url return listing, False + def _commit_if_current(self, session, task) -> bool: + """Acquire the short write lock and gate all pending result mutations.""" + from sqlalchemy import update + + result = session.execute(update(WatchTask).where(WatchTask.id == task.id, + WatchTask.version == task.version, WatchTask.enabled.is_(True)).values( + version=WatchTask.version).execution_options(synchronize_session=False)) + if result.rowcount != 1: + session.rollback() + return False + session.commit() + return True + def _is_blocked(self, session, listing: Listing) -> bool: if listing.blocked: return True @@ -280,7 +166,7 @@ def _is_blocked(self, session, listing: Listing) -> bool: return True return False - def _fetch_detail(self, session, listing: Listing, force: bool = False) -> None: + def _fetch_detail(self, session, listing: Listing, force: bool = False) -> ListingDetail | None: if not listing.url: return if not force and listing.description: @@ -290,50 +176,35 @@ def _fetch_detail(self, session, listing: Listing, force: bool = False) -> None: except Exception as exc: logger.warning("详情抓取失败,退回标题判断: %s", exc) return - if detail.description: - listing.description = detail.description - merged = [u for u in (listing.image_urls or []) if is_product_image(u)] + listing.description = detail.description + if self.media_service: + self.media_service.record(listing.id, detail.media) + merged = [] for url in detail.image_urls: if url not in merged and is_product_image(url): merged.append(url) listing.image_urls = merged[:8] - if detail.variants: - listing.variants = detail.variants[:6] + listing.variants = detail.variants[:6] if detail.seller_uid: listing.seller_uid = detail.seller_uid listing.seller_name = detail.seller_name or listing.seller_name listing.seller_risk = { + **(listing.seller_risk or {}), "credit_label": detail.credit_label, "positive_rate": detail.positive_rate, "sold_count": detail.sold_count, } + return detail def reanalyze_listing(self, listing_id: int) -> dict[str, Any]: - """手动重新分析单个商品:重抓详情并重跑需求/品相/性价比/卖家,不发送通知。""" - stats = {"updated": 0} - self._judger = self._build_judger(self.settings_service.get()) - with self._session_factory() as session: - listing = session.get(Listing, listing_id) - if listing is None: - raise RuntimeError(f"商品 {listing_id} 不存在") - task = ( - session.get(WatchTask, listing.task_id) - if listing.task_id is not None - else WatchTask(keyword="", condition_requirement="", min_condition_score=0) - ) - if listing.url: - self._fetch_detail(session, listing, force=True) - self._requirement_pass(session, listing, task) - self._condition_analysis(session, listing, task) - self._seller_check(session, listing, task) - if self._value_client() is not None: - self._batch_value( - session, [self._batch_row(listing, task.condition_requirement or "")] - ) - listing.satisfaction = self._satisfaction(listing) - session.commit() - stats["updated"] = 1 - return stats + """Revalidate offer evidence before analysis, without sending notifications.""" + from goodprice.services.offer_crawl import OfferCrawlRunner + from goodprice.services.offer_pipeline import StaleEvaluation + + try: + return OfferCrawlRunner(self).reanalyze(listing_id) + except StaleEvaluation: + return {"updated": 0, "skipped": "superseded"} def _seller_check(self, session, listing: Listing, task: WatchTask) -> None: if not listing.seller_uid or self.seller_service is None: @@ -430,9 +301,22 @@ def _build_judger(self, settings): ) return judger - def _condition_analysis(self, session, listing: Listing, task: WatchTask) -> None: + def _condition_analysis(self, session, listing: Listing, task: WatchTask, + price: float | None = None, offer=None, force=False) -> None: if not self.vision.enabled: return + listing.condition_score = None + if self.media_service: + from goodprice.offers.types import OfferCandidate + + try: + verdict = self.media_service.analyze(listing.id, offer or OfferCandidate("default", {}), + self.vision, task.condition_requirement or "", self.settings_service.get(), force=force) + listing.condition_score = verdict.get("condition_score") + listing.condition_detail = verdict + except Exception: + listing.condition_detail = {"error": "媒体分析失败或超出预算,品相待核验"} + return valid = [u for u in (listing.image_urls or []) if is_product_image(u)] if not valid: listing.condition_detail = {"error": "无有效商品图"} @@ -442,7 +326,7 @@ def _condition_analysis(self, session, listing: Listing, task: WatchTask) -> Non try: verdict = self.vision.analyze_condition( title=listing.title, - price=listing.price, + price=price if price is not None else listing.price, description=listing.description or "", requirement=task.condition_requirement or "", image_urls=valid, @@ -455,13 +339,6 @@ def _condition_analysis(self, session, listing: Listing, task: WatchTask) -> Non return listing.condition_detail = {"error": str(last_exc)[:200]} - def _condition_gate_fails(self, task: WatchTask, listing: Listing) -> bool: - return bool( - task.min_condition_score - and listing.condition_score is not None - and listing.condition_score < task.min_condition_score - ) - def _excluded(self, task: WatchTask, title: str) -> bool: """排除词按空白/中英文逗号拆分,标题命中任一即跳过。""" words = ( @@ -497,190 +374,6 @@ def _value_client(self): return client return None - def _drop_pct(self, listing: Listing) -> float: - return drop_pct_from_snapshots(listing) - - def _satisfaction(self, listing: Listing) -> float: - return compute_satisfaction( - listing, - vision_enabled=self.vision.enabled, - price_drop_pct=self._drop_pct(listing), - ) - - def _batch_value(self, session, rows: list[dict]) -> None: - now = datetime.now() - judger = self._judger - if judger is not None and judger.enabled: - try: - result = judger.analyze_batch_value(rows[:MAX_BATCH_VALUE_ITEMS]) - except Exception as exc: - logger.warning("Jev 批量性价比失败,回落原路径: %s", exc) - result = None - if result is not None: - scores = result.get("scores") or {} - best = result.get("best") - logger.info( - "Jev 批量性价比完成:%s 个商品,本批最优 %s", len(rows), best - ) - for row in rows: - listing = session.get(Listing, row["listing_id"]) - if listing is None: - continue - score = scores.get(row["external_id"]) - if score is not None: - listing.value_score = score - listing.value_batch_at = now - listing.best_of_batch = best == row["external_id"] - return - client = self._value_client() - if client is None: - return - try: - result = client.analyze_batch_value(rows[:MAX_BATCH_VALUE_ITEMS]) - except Exception as exc: - logger.warning("批量性价比分析失败,跳过: %s", exc) - return - scores = result.get("scores") or {} - best = result.get("best") - logger.info("批量性价比完成:%s 个商品,本批最优 %s", len(rows), best) - for row in rows: - listing = session.get(Listing, row["listing_id"]) - if listing is None: - continue - score = scores.get(row["external_id"]) - if score is not None: - listing.value_score = score - listing.value_batch_at = now - listing.best_of_batch = best == row["external_id"] - - def _backfill(self, session, listing: Listing, task: WatchTask) -> bool: - changed = False - requirement = (task.condition_requirement or "").strip() - if requirement and self.llm.enabled and listing.requirement_match is None: - verdict = self._requirement_verdict(listing, requirement) - if verdict is not None: - listing.requirement_match, listing.requirement_reason = verdict - changed = True - else: - try: - result = self.llm.analyze_requirement( - title=listing.title, - description=listing.description or "", - requirement=requirement, - ) - listing.requirement_match = result["matched"] - listing.requirement_reason = result["reason"] - changed = True - except Exception as exc: - logger.warning("回填需求分析失败: %s", exc) - if self.vision.enabled and listing.condition_score is None: - try: - verdict = self.vision.analyze_condition( - title=listing.title, - price=listing.price, - description=listing.description or "", - requirement=requirement, - image_urls=listing.image_urls, - ) - listing.condition_score = verdict["condition_score"] - listing.condition_detail = verdict - changed = True - except Exception as exc: - logger.warning("回填品相分析失败: %s", exc) - return changed - - def _notify( - self, - session, - task: WatchTask, - listing: Listing, - satisfaction: float, - old_price: Optional[float] = None, - is_renotify: bool = False, - ) -> None: - def _fmt(value: float) -> str: - return format(value, "g") - - lines = [f"价格:{_fmt(listing.price)} 元"] - drop_pct = self._drop_pct(listing) - if drop_pct >= DROP_PCT_DISPLAY_THRESHOLD: - lines.append(f"较首见降价 {drop_pct:.0%}") - if is_renotify and old_price is not None: - lines.append(f"价格更新重推:{_fmt(old_price)} → {_fmt(listing.price)} 元") - if listing.requirement_match is not None: - status = "是" if listing.requirement_match else "否" - reason = listing.requirement_reason or "" - line = f"需求匹配:{status}" - if reason: - line += f"({reason})" - lines.append(line) - if listing.condition_score is not None: - lines.append(f"品相分:{listing.condition_score}") - elif self.vision.enabled: - err = "" - if isinstance(listing.condition_detail, dict): - err = listing.condition_detail.get("error") or "" - lines.append(f"品相分:未评估({err or '分析失败'})") - else: - lines.append("品相分:视觉模型未启用,未评估") - if listing.value_score is not None: - lines.append(f"性价比:{listing.value_score}/10") - else: - lines.append("性价比:未评估") - if listing.best_of_batch: - lines.append("本批最优") - if listing.variants: - parts = " · ".join( - f"{v.get('name')} {_fmt(v.get('price'))} 元" for v in listing.variants[:6] - ) - lines.append(f"规格:{parts}") - if listing.seller_risk: - risk = listing.seller_risk - name = risk.get("nickname") or "卖家" - level = risk.get("risk_level") - reason = risk.get("risk_reason") or "" - rate = risk.get("positive_rate") - rate_txt = f"好评率 {rate * 100:.0f}%" if isinstance(rate, (int, float)) else "" - lines.append(f"卖家:{name} {rate_txt} · 风险{level}({reason})") - if isinstance(listing.condition_detail, dict): - extra = listing.condition_detail.get("reason", "") - if extra: - lines.append(extra) - message = NotificationMessage( - title=f"[{task.keyword}] {listing.title}", - content="\n".join(lines), - url=listing.url, - ) - for channel, notifier in self.notifiers: - try: - notifier.send(message) - logger.info("通知[%s] 已发送:%s", channel, listing.title[:30]) - session.add( - Notification( - listing_id=listing.id, - task_id=task.id, - channel=channel, - status="sent", - title=message.title, - content=message.content, - ) - ) - except Exception as exc: - logger.warning("通知[%s]失败: %s", channel, exc) - session.add( - Notification( - listing_id=listing.id, - task_id=task.id, - channel=channel, - status="failed", - detail=str(exc), - title=message.title, - content=message.content, - ) - ) - listing.notified_at = datetime.now() - listing.last_notified_satisfaction = satisfaction - def _record_error(self, task_id: int, message: str) -> None: with self._session_factory() as session: task = session.get(WatchTask, task_id) diff --git a/goodprice/services/media_service.py b/goodprice/services/media_service.py new file mode 100644 index 0000000..efd0541 --- /dev/null +++ b/goodprice/services/media_service.py @@ -0,0 +1,103 @@ +from datetime import datetime + +from goodprice.analysis.media_analysis import analysis_key, analyze_media +from goodprice.media.selection import select_media +from goodprice.media.types import MediaCandidate, MediaSelection +from goodprice.models import Listing, MediaAsset + + +class MediaService: + def __init__(self, factory, processor): + self.factory = factory + self.processor = processor + + def record(self, listing_id, candidates): + with self.factory() as session: + rows = session.query(MediaAsset).filter_by(listing_id=listing_id).all() + sources = {row.key: row for row in rows if row.kind != "frame" and row.source_url} + for row in sources.values(): + row.status = "superseded" + for candidate in candidates: + row = sources.get(candidate.key) + if row is None: + row = MediaAsset(listing_id=listing_id, key=candidate.key, kind=candidate.kind) + session.add(row) + row.source_url, row.source_locator = candidate.url, candidate.source_locator + row.relative_path, row.content_hash = None, None + row.offer_key, row.status = candidate.offer_key, candidate.status + row.observed_at = datetime.now() + session.commit() + + def _candidates(self, listing_id): + with self.factory() as session: + rows = session.query(MediaAsset).filter(MediaAsset.listing_id == listing_id, + MediaAsset.source_url != "", MediaAsset.status != "superseded").order_by(MediaAsset.id).all() + return [MediaCandidate(r.key, r.kind, r.source_url, r.offer_key, r.source_locator, + status="unsupported" if r.status == "unsupported" else "pending") for r in rows] + + def analyze(self, listing_id, offer, client, requirement, settings, *, force=False): + candidates = self._candidates(listing_id) + with self.factory() as session: + listing = session.get(Listing, listing_id) + seller_text = f"{listing.title}\n{listing.description or ''}" if listing else "" + with self.processor.session(candidates, settings) as (evidence, failures): + # A media explicitly bound to a different SKU is not this offer's evidence. + relevant = [e for e in evidence if e.offer_key is None or e.offer_key == offer.key] + selection = select_media([e for e in relevant if e.kind == "image"], + [e for e in relevant if e.kind == "frame"], settings.media_image_quota, + settings.media_frame_quota, settings.media_total_limit) + selection = MediaSelection(selection.items, selection.omitted_ids, + tuple(dict.fromkeys(selection.limitations + failures))) + # Price is excluded: this is a visual judgement, not a price verdict. + key = analysis_key(offer, selection, client, requirement, settings, seller_text) + with self.factory() as session: + listing = session.get(Listing, listing_id) + previous = listing.condition_detail or {} if listing else {} + for e in evidence: + row = session.query(MediaAsset).filter_by(listing_id=listing_id, + key=f"evidence:{e.id}").first() + if row is None: + row = MediaAsset(listing_id=listing_id, key=f"evidence:{e.id}", kind=e.kind) + session.add(row) + row.content_hash, row.offer_key = e.content_hash, e.offer_key + row.relative_path = e.local_path.relative_to(self.processor.budget.root).as_posix() + row.timestamp_ms, row.parent_video_key = e.timestamp_ms, e.video_id + row.status = "ready" + session.commit() + if not force and previous.get("media_cache_key") == key and selection.items: + return previous + result = analyze_media(offer, selection, client, requirement, settings.media_request_max_mib * 1024**2, seller_text) + result["media_cache_key"] = key + result["omitted_evidence_ids"] = list(selection.omitted_ids) + result["defects"] = [o["text"] for o in result.get("observations", [])] + return result + + def save_result(self, listing_id, result): + with self.factory() as session: + listing = session.get(Listing, listing_id) + if listing: + listing.condition_score = result.get("condition_score") + listing.condition_detail = result + session.commit() + + def public_assets(self, listing_id): + with self.factory() as session: + rows = session.query(MediaAsset).filter_by(listing_id=listing_id).order_by(MediaAsset.id).all() + result = [] + for row in rows: + if row.status == "superseded": + continue + available = False + if row.relative_path: + try: + path = self.processor.budget.checked(self.processor.budget.root / row.relative_path) + available = path.is_file() + except ValueError: + pass + viewable = available or (row.kind == "image" and bool(row.source_url) and row.status != "unsupported") + result.append({"id": row.id, "evidence_id": row.key.removeprefix("evidence:"), + "kind": row.kind, "offer_key": row.offer_key, "timestamp_ms": row.timestamp_ms, + "content_hash": row.content_hash, "status": row.status, + "available": available, "viewable": viewable, + "url": f"/media/{row.id}" if viewable else None}) + return result diff --git a/goodprice/services/offer_crawl.py b/goodprice/services/offer_crawl.py new file mode 100644 index 0000000..276a701 --- /dev/null +++ b/goodprice/services/offer_crawl.py @@ -0,0 +1,406 @@ +"""Serial adapter for offer stages; the coordinator will call the same stages. + +Keep page prices unchanged. Only explicit, fresh offer evidence can enter value +analysis and notifications. External fetches happen before observation writes. +""" +import hashlib +import json +import logging +from datetime import datetime, timezone +from types import SimpleNamespace + +import httpx + +from goodprice.models import ( + Listing, + ListingOffer, + Notification, + OfferEvaluationRecord, + OfferObservation, + WatchTask, +) +from goodprice.notify.base import NotificationMessage +from goodprice.offers.pricing import decide_price, offer_fingerprint, utc +from goodprice.offers.service import decode_offer +from goodprice.offers.types import OfferCandidate, PriceClaim +from goodprice.services.offer_pipeline import OfferPipeline +from goodprice.services.satisfaction import compute_satisfaction + +logger = logging.getLogger(__name__) + + +class OfferCrawlRunner: + def __init__(self, crawl): + self.crawl = crawl + self.factory = crawl._session_factory + self.pipeline = OfferPipeline(self.factory, vision_enabled=crawl.vision.enabled) + + def run(self, task_id: int) -> dict: + now = datetime.now(timezone.utc) + stats = dict.fromkeys(("found", "new", "notified", "backfilled", "reevaluated", "gone"), 0) + with self.factory() as session: + task = session.get(WatchTask, task_id) + if task is None: + raise RuntimeError(f"任务 {task_id} 不存在") + if not task.enabled: + return {**stats, "skipped": "disabled"} + task.last_run_at = now + task.last_run_count = (task.last_run_count or 0) + 1 + task.last_error = None + session.commit() + self.crawl._judger = self.crawl._build_judger(self.crawl.settings_service.get()) + self._retry_failed(task, now) + try: + items = self.crawl.adapter.search(task.keyword) + except Exception as exc: + self.crawl._record_error(task_id, f"抓取失败: {exc}") + raise + stats["found"] = len(items) + pending, seen = [], set() + for data in items[:30]: + if self.crawl._excluded(task, data.title): + continue + with self.factory() as session: + listing, is_new = self.crawl._upsert_listing(session, task, data) + was_missing = listing.status != "active" + listing.status, listing.missed_count = "active", 0 + seen.add(listing.id) + blocked = self.crawl._is_blocked(session, listing) + session.commit() + listing_id = listing.id + if blocked: + continue + stats["new"] += int(is_new) + with self.factory() as session: + previous = session.query(OfferObservation).join(ListingOffer).filter( + ListingOffer.listing_id == listing_id).order_by(OfferObservation.id.desc()).first() + old_fingerprint = previous.fingerprint if previous else None + previous_evaluations = {o["key"]: o["evaluation"] for o in self.pipeline.latest(listing_id)} + candidates = self._fetch(listing_id, task) + now = datetime.now(timezone.utc) + changed = is_new or was_missing or len(candidates) > 1 or any( + offer_fingerprint(c) != old_fingerprint for c in candidates) + # Freshness renews each observation; content identity does not. + ids = self.pipeline.evaluate_listing(listing_id, task_id, candidates, now, expected_version=task.version) + with self.factory() as session: + records = [session.get(OfferEvaluationRecord, i) for i in ids] + potential = any(r.payload["state"] == "exact" and r.payload["match"] == "matched" + and r.payload["amount_cents"] is not None + and r.payload["amount_cents"] >= round(task.min_price * 100) + and (not task.max_price or r.payload["amount_cents"] <= round(task.max_price * 100)) + for r in records) + analysis_price = next((r.payload["amount_cents"] / 100 for r in records + if r.payload["state"] == "exact" and r.payload["match"] == "matched"), None) + for candidate, record in zip(candidates, records, strict=False): + old = previous_evaluations.get(candidate.key) + if old is None or any(old.get(k) != record.payload.get(k) for k in ("state", "source", "amount_cents", "task_version")): + changed = True + listing = session.get(Listing, listing_id) + settings = self.crawl.settings_service.get() + inputs = {"title": listing.title, "description": listing.description, "images": listing.image_urls, + "requirement": task.condition_requirement, "target": task.target_variant, + "vision": self.crawl.vision.enabled, "model": settings.llm_model, + "vision_model": settings.vision_model, "prompt": "offer-v1", + "llm_format": settings.llm_api_format, "vision_format": settings.vision_api_format, + "llm_url": settings.llm_base_url, "vision_url": settings.vision_base_url} + analysis_fp = hashlib.sha256(json.dumps(inputs, sort_keys=True).encode()).hexdigest() + content_fp = hashlib.sha256(json.dumps( + {"title": listing.title, "description": listing.description, "images": listing.image_urls}, sort_keys=True).encode()).hexdigest() + unchanged_content = all((previous_evaluations.get(c.key) or {}).get("analysis_content_fingerprint") == content_fp + for c in candidates) + needs_backfill = (not changed and unchanged_content and self.crawl.vision.enabled + and listing.condition_score is None) + if any((previous_evaluations.get(c.key) or {}).get("analysis_fingerprint") != analysis_fp + for c in candidates): + changed = True + if needs_backfill: + changed = False + if (potential or self.crawl.media_service) and (changed or needs_backfill or self.crawl.media_service): + with self.factory() as session: + listing = session.get(Listing, listing_id) + previous_media = (listing.condition_detail or {}).get("media_cache_key") + previous_score = listing.condition_score + if (not changed and not needs_backfill) or self.crawl._requirement_pass(session, listing, task): + if len(candidates) == 1: + self.crawl._condition_analysis(session, listing, task, price=analysis_price, offer=candidates[0]) + else: + listing.condition_score = None + listing.condition_detail = {"error": "多规格共享媒体,品相待核验"} + self.crawl._seller_check(session, listing, task) + if self.crawl.media_service and (previous_media != (listing.condition_detail or {}).get("media_cache_key") + or previous_score != listing.condition_score): + changed = True + if not self.crawl._commit_if_current(session, task): + return {**stats, "skipped": "superseded"} + ids = self.pipeline.evaluate_listing(listing_id, task_id, candidates, now, expected_version=task.version) + stats["reevaluated"] += int(not is_new and changed) + stats["backfilled"] += int(needs_backfill) + for record_id in ids: + with self.factory() as session: + row = session.get(OfferEvaluationRecord, record_id) + row.payload = {**row.payload, "analysis_fingerprint": analysis_fp, + "analysis_content_fingerprint": content_fp} + offer = session.get(ListingOffer, row.offer_id) + listing = session.get(Listing, listing_id) + if row.payload["eligible"] and not changed: + old = previous_evaluations.get(offer.offer_key) or {} + row.payload = {**row.payload, **{key: old[key] for key in + ("value_score", "best_of_batch", "satisfaction") if key in old}} + listing.offer_price_cents = row.payload["amount_cents"] + listing.offer_verified_until = datetime.fromisoformat(row.payload["fresh_until"]) + listing.offer_task_version = task.version + session.commit() + if row.payload["eligible"] and changed: + key = listing.external_id if len(candidates) == 1 else f"{listing.external_id}:{offer.offer_key}" + pending.append((record_id, { + **self.crawl._batch_row(listing, task.condition_requirement), + "external_id": key, "offer_id": offer.id, + "price": row.payload["amount_cents"] / 100, + "title": f"{listing.title} {offer.name} {dict(offer.attributes)}", + })) + elif not row.payload["eligible"]: + listing.value_score, listing.best_of_batch = None, False + listing.offer_verified_until = None + if task.notify_pending_price and row.payload["state"] != "exact" and row.payload["match"] != "mismatched": + self._notify_pending(session, task, listing, offer, row.payload) + session.commit() + session.commit() + self._finish(task, pending, now, stats) + with self.factory() as session: + for listing in session.query(Listing).filter_by(task_id=task.id): + if listing.id not in seen: + listing.missed_count += 1 + if listing.missed_count >= 3: + listing.status = "unseen" # A limited search is not proof of delisting. + session.commit() + stats["deferred"] = max(0, len(items) - 30) + return stats + + def _retry_failed(self, task, now): + notifiers = dict(self.crawl.notifiers) + with self.factory() as session: + rows = session.query(Notification).filter_by(task_id=task.id, status="failed").order_by( + Notification.id.desc()).limit(100).all() + seen = set() + for row in rows: + pair = (row.event_key, row.channel) + ctx = row.event_context or {} + if pair in seen or not row.event_key or row.channel not in notifiers or ctx.get("task_version") != task.version: + continue + seen.add(pair) + with self.factory() as session: + already_sent = session.query(Notification.id).filter_by( + event_key=row.event_key, channel=row.channel, status="sent").first() + attempts = session.query(Notification).filter_by(event_key=row.event_key, channel=row.channel).count() + if already_sent or attempts >= 3: + continue + snapshots = self.pipeline.snapshot(row.listing_id, now) + current = next((s for s in snapshots if s["id"] == row.offer_id), None) + if current is None or current["fingerprint"] != ctx.get("fingerprint"): + continue + if not ctx.get("pending") and (current["price_state"] != "exact" + or current["source"] != ctx.get("source") + or current["effective_price_cents"] != ctx.get("amount_cents")): + continue + status, detail = "sent", None + try: + with self.factory() as session: + listing = session.get(Listing, row.listing_id) + url = listing.url + notifiers[row.channel].send(NotificationMessage(row.title, row.content, url)) + except Exception as exc: + status = "uncertain" if isinstance(exc, (TimeoutError, httpx.TimeoutException)) else "failed" + detail = str(exc)[:500] + with self.factory() as session: + session.add(Notification(listing_id=row.listing_id, task_id=task.id, offer_id=row.offer_id, + event_key=row.event_key, event_context=ctx, channel=row.channel, status=status, + title=row.title, content=row.content, detail=detail)) + session.commit() + + def _notify_pending(self, session, task, listing, offer, payload): + event_key = "pending:" + hashlib.sha256( + f"{task.id}:{task.version}:{offer.id}:{payload['fingerprint']}".encode()).hexdigest() + message = NotificationMessage(f"[待询价] {listing.title}", + f"规格:{offer.name or dict(offer.attributes) or '待核验'}\n{payload['reason']}\n" + "价格尚未确认,不参与正常性价比排名;请自行联系卖家核实。", listing.url) + for channel, notifier in self.crawl.notifiers: + if session.query(Notification).filter_by(event_key=event_key, channel=channel, status="sent").first(): + continue + status, detail = "sent", None + try: + notifier.send(message) + except Exception as exc: + status, detail = "failed", str(exc)[:500] + session.add(Notification(listing_id=listing.id, task_id=task.id, offer_id=offer.id, + event_key=event_key, channel=channel, status=status, detail=detail, + event_context={"task_version": task.version, "fingerprint": payload["fingerprint"], "pending": True}, + title=message.title, content=message.content)) + + def _fetch(self, listing_id, task): + detail = None + if not task.fetch_detail: + return [OfferCandidate("default", claims=(PriceClaim("dom", "unknown", + "详情抓取关闭,价格和规格待核验", datetime.now(timezone.utc), "detail:disabled"),))] + if task.fetch_detail: + with self.factory() as session: + listing = session.get(Listing, listing_id) + detail = self.crawl._fetch_detail(session, listing, force=True) + session.commit() + if detail and detail.offers: + return detail.offers + if detail is None: + # Failure is not a new observation. Reuse original timestamps so a + # transient fault neither revokes a fresh quote nor renews stale data. + with self.factory() as session: + rows = session.query(OfferObservation).join(ListingOffer).filter( + ListingOffer.listing_id == listing_id).order_by(OfferObservation.id.desc()).all() + cached = {} + for row in rows: + cached.setdefault(row.offer_id, decode_offer(row.payload)) + if cached: + return list(cached.values()) + return [OfferCandidate("default", claims=(PriceClaim( + "dom", "unknown", "详情价格未核验", datetime.now(timezone.utc), "detail:missing"),))] + + def reanalyze(self, listing_id): + with self.factory() as session: + listing = session.get(Listing, listing_id) + if listing is None: + raise RuntimeError(f"商品 {listing_id} 不存在") + task = session.get(WatchTask, listing.task_id) if listing.task_id else None + if task is None: + raise RuntimeError("原监控任务已删除,无法按旧条件重新分析") + candidates = self._fetch(listing_id, task) + now = datetime.now(timezone.utc) + ids = self.pipeline.evaluate_listing(listing_id, task.id, candidates, now, expected_version=task.version) + with self.factory() as session: + listing = session.get(Listing, listing_id) + potential = any(session.get(OfferEvaluationRecord, i).payload["state"] == "exact" for i in ids) + analysis_price = next((session.get(OfferEvaluationRecord, i).payload["amount_cents"] / 100 + for i in ids if session.get(OfferEvaluationRecord, i).payload["state"] == "exact"), None) + if (potential or self.crawl.media_service) and self.crawl._requirement_pass(session, listing, task): + if len(candidates) == 1: + self.crawl._condition_analysis(session, listing, task, price=analysis_price, offer=candidates[0], force=True) + else: + listing.condition_score = None + listing.condition_detail = {"error": "多规格共享媒体,品相待核验"} + self.crawl._seller_check(session, listing, task) + if not potential: + listing.value_score, listing.best_of_batch = None, False + if not self.crawl._commit_if_current(session, task): + return {"updated": 0, "skipped": "superseded"} + ids = self.pipeline.evaluate_listing(listing_id, task.id, candidates, datetime.now(timezone.utc), expected_version=task.version) + pending = [] + with self.factory() as session: + listing = session.get(Listing, listing_id) + for record_id in ids: + row = session.get(OfferEvaluationRecord, record_id) + if row.payload["eligible"]: + offer = session.get(ListingOffer, row.offer_id) + pending.append((record_id, {**self.crawl._batch_row(listing, task.condition_requirement), + "external_id": listing.external_id if len(candidates) == 1 else f"{listing.external_id}:{offer.offer_key}", + "offer_id": offer.id, "price": row.payload["amount_cents"] / 100})) + self.crawl._judger = self.crawl._build_judger(self.crawl.settings_service.get()) + self._finish(task, pending, now, {"notified": 0}, notify=False) + return {"updated": 1} + + def _finish(self, task, pending, now, stats, notify=True): + result = {"scores": {}, "best": None, "reasons": {}} + client = self.crawl._value_client() + judger = self.crawl._judger + typed = None + if pending and judger is not None and judger.enabled: + try: + typed = judger.analyze_batch_value([r for _, r in pending]) + except Exception as exc: + logger.warning("规格 Jev 性价比失败,回落原模型: %s", exc) + if typed is not None: + result = typed + elif client and pending: + try: + result = client.analyze_batch_value([r for _, r in pending]) + except Exception as exc: + logger.warning("规格性价比分析失败: %s", exc) + for record_id, item in pending: + with self.factory() as session: + current_task = session.get(WatchTask, task.id) + if current_task is None or not current_task.enabled or current_task.version != task.version: + continue + record = session.get(OfferEvaluationRecord, record_id) + listing = session.get(Listing, item["listing_id"]) + offer = session.get(ListingOffer, item["offer_id"]) + if record is None or listing is None or offer is None: + continue + payload = dict(record.payload) + if payload["fresh_until"] and datetime.fromisoformat(payload["fresh_until"]) <= datetime.now(timezone.utc): + continue + value = result.get("scores", {}).get(item["external_id"]) + best = result.get("best") == item["external_id"] + first = session.query(OfferObservation).filter_by(offer_id=offer.id).order_by(OfferObservation.id).first() + first_price = decide_price(decode_offer(first.payload), None, utc(first.observed_at), task.interval_minutes * 60) + drop = 0.0 + if first_price.source == payload["source"] and first_price.amount_cents: + drop = max(0.0, (first_price.amount_cents - payload["amount_cents"]) / first_price.amount_cents) + score = compute_satisfaction(SimpleNamespace(requirement_match=listing.requirement_match, + condition_score=listing.condition_score, value_score=value, seller_risk=listing.seller_risk), + vision_enabled=self.crawl.vision.enabled, price_drop_pct=drop) + payload.update(value_score=value, best_of_batch=best, satisfaction=score) + record.payload = payload + listing.value_score, listing.best_of_batch, listing.satisfaction = value, best, score + listing.offer_price_cents = payload["amount_cents"] + listing.offer_verified_until = datetime.fromisoformat(payload["fresh_until"]) + listing.offer_task_version = task.version + if not notify: + session.commit() + continue + if offer.last_notified_source == "legacy_baseline": + offer.last_notified_score, offer.last_notified_amount = score, payload["amount_cents"] + offer.last_notified_source = payload["source"] + session.commit() + continue + if offer.last_notified_score is not None and score <= offer.last_notified_score: + session.commit() + continue + event_key = hashlib.sha256(f"{task.id}:{task.version}:{offer.id}:{payload['fingerprint']}:{score}".encode()).hexdigest() + previous_amount = offer.last_notified_amount + renotify = (previous_amount is not None and offer.last_notified_source == payload["source"] + and previous_amount != payload["amount_cents"]) + lines = [f"规格:{offer.name or dict(offer.attributes) or '默认商品'}", + f"价格:{payload['amount_cents'] / 100:g} 元({'人工报价' if payload['source'] == 'manual' else '页面确价'})", + f"需求匹配:{'否' if listing.requirement_match is False else '是' if listing.requirement_match else '未核验'}", + f"品相分:{listing.condition_score if listing.condition_score is not None else '未评估'}" + + ("(视觉模型未启用)" if not self.crawl.vision.enabled else ""), + f"性价比:{str(value) + '/10' if value is not None else '未评估'}"] + if renotify: + lines.append(f"价格更新重推:{previous_amount / 100:g} → {payload['amount_cents'] / 100:g} 元") + if best: + lines.append(f"本批最优(本次已就绪 {len(pending)} 个规格)") + if listing.seller_risk: + risk = listing.seller_risk + rate = risk.get("positive_rate") + lines.append(f"卖家:{risk.get('nickname', '卖家')} · 风险{risk.get('risk_level', '未知')} " + + (f"好评率 {rate * 100:.0f}%" if isinstance(rate, (int, float)) else "")) + message = NotificationMessage(f"[{task.keyword}] {listing.title}", "\n".join(lines), listing.url) + success = False + for channel, notifier in self.crawl.notifiers: + if session.query(Notification).filter_by(event_key=event_key, channel=channel, status="sent").first(): + success = True + continue + status, error = "sent", None + try: + notifier.send(message) + success = True + except Exception as exc: + status = "uncertain" if isinstance(exc, (TimeoutError, httpx.TimeoutException)) else "failed" + error = str(exc)[:500] + session.add(Notification(listing_id=listing.id, task_id=task.id, offer_id=offer.id, + event_key=event_key, channel=channel, status=status, detail=error, + event_context={"task_version": task.version, "fingerprint": payload["fingerprint"], + "source": payload["source"], "amount_cents": payload["amount_cents"]}, + title=message.title, content=message.content)) + if success: + offer.last_notified_score, offer.last_notified_amount = score, payload["amount_cents"] + offer.last_notified_source = payload["source"] + listing.notified_at, listing.last_notified_satisfaction = now, score + stats["notified"] += 1 + session.commit() diff --git a/goodprice/services/offer_pipeline.py b/goodprice/services/offer_pipeline.py new file mode 100644 index 0000000..406ac1b --- /dev/null +++ b/goodprice/services/offer_pipeline.py @@ -0,0 +1,208 @@ +"""Offer decisions shared by serial execution and the future stage coordinator.""" +from contextlib import nullcontext +from datetime import datetime, timedelta + +from goodprice.models import ( + Listing, + ListingOffer, + ManualQuote, + OfferEvaluationRecord, + OfferObservation, + WatchTask, +) +from goodprice.offers.evaluation import assess_offer +from goodprice.offers.matching import match_variant +from goodprice.offers.pricing import decide_price, utc +from goodprice.offers.service import OfferService +from goodprice.offers.types import OfferCandidate, PriceDecision, QuoteInput + + +class StaleEvaluation(RuntimeError): + pass + + +class OfferPipeline: + def __init__(self, session_factory, vision_enabled: bool = True): + self._session_factory = session_factory + self.vision_enabled = vision_enabled + self.offers = OfferService(session_factory) + + def evaluate_listing(self, listing_id: int, task_id: int, + candidates: list[OfferCandidate], now: datetime, *, expected_version=None, + session=None, per_offer_results=None, ready=True, + record_observations=True) -> list[int]: + # Network/AI callers must finish their work before this short write phase. + result = [] + own = session is None + with self._session_factory() if own else nullcontext(session) as session: + listing, task = session.get(Listing, listing_id), session.get(WatchTask, task_id) + if listing is None or task is None or listing.task_id != task.id: + raise LookupError("任务或商品不存在/关联不一致") + if expected_version is not None: + from sqlalchemy import update + gate = session.execute(update(WatchTask).where(WatchTask.id == task_id, + WatchTask.version == expected_version, WatchTask.enabled.is_(True)).values( + version=WatchTask.version).execution_options(synchronize_session=False)) + if gate.rowcount != 1: + raise StaleEvaluation("任务已暂停或条件已变化") + if record_observations: + ids = [self.offers.record_observation(listing_id, o, session=session, now=now) for o in candidates[:20]] + else: + existing = {o.offer_key: o.id for o in session.query(ListingOffer).filter_by(listing_id=listing_id)} + ids = [existing[o.key] for o in candidates[:20]] + for candidate, offer_id in zip(candidates, ids, strict=False): + quote = session.query(ManualQuote).filter_by(offer_id=offer_id, revoked=False).order_by( + ManualQuote.id.desc()).first() + value = None if quote is None else QuoteInput(candidate.key, quote.amount_cents, + utc(quote.confirmed_at), utc(quote.expires_at), quote.evidence_fingerprint, quote.revoked) + scoped = (per_offer_results or {}).get(candidate.key, {}) + condition_score = scoped.get("condition_score") if per_offer_results is not None else listing.condition_score + visual_valid = (scoped.get("visual_scope") == candidate.key if per_offer_results is not None else len(candidates) == 1) + requirement_match = scoped.get("requirement_match") if per_offer_results is not None else listing.requirement_match + payload = assess_offer(task, listing, candidate, value, now, vision_enabled=self.vision_enabled, + condition_score=condition_score, requirement_match=requirement_match, + visual_valid=visual_valid, ready=ready) + latest = session.query(OfferObservation).filter_by(offer_id=offer_id).order_by( + OfferObservation.id.desc()).first() + payload["fingerprint"] = latest.fingerprint + payload["observation_id"] = latest.id + payload["task_version"] = task.version + row = OfferEvaluationRecord(offer_id=offer_id, task_version=task.version, + payload=payload, evaluated_at=now) + session.add(row) + session.flush() + result.append(row.id) + if own: + session.commit() + else: + session.flush() + return result + + def latest(self, listing_id: int) -> list[dict]: + with self._session_factory() as session: + result = [] + for offer in session.query(ListingOffer).filter_by(listing_id=listing_id).order_by(ListingOffer.id): + row = session.query(OfferEvaluationRecord).filter_by(offer_id=offer.id).order_by( + OfferEvaluationRecord.id.desc()).first() + result.append({"id": offer.id, "key": offer.offer_key, "name": offer.name, + "attributes": offer.attributes, "availability": offer.availability, + "evaluation": row.payload if row else None}) + return result + + def snapshot(self, listing_id: int, now: datetime) -> list[dict]: + result = self.snapshot_many([listing_id], now) + if listing_id not in result: + raise LookupError("商品不存在") + return result[listing_id] + + def snapshot_many(self, listing_ids, now: datetime) -> dict[int, list[dict]]: + """One bounded set of latest-row queries for the whole visible page.""" + from sqlalchemy import func + + from goodprice.offers.service import decode_offer + + if not listing_ids: + return {} + with self._session_factory() as session: + listings = {row.id: row for row in session.query(Listing).filter(Listing.id.in_(listing_ids))} + task_ids = {row.task_id for row in listings.values() if row.task_id} + tasks = {row.id: row for row in session.query(WatchTask).filter(WatchTask.id.in_(task_ids))} if task_ids else {} + offers = session.query(ListingOffer).filter(ListingOffer.listing_id.in_(listing_ids)).order_by(ListingOffer.id).all() + offer_ids = [row.id for row in offers] + result = {key: [] for key in listings} + if not offer_ids: + return result + def latest(model, active=False): + ids = session.query(func.max(model.id)).filter(model.offer_id.in_(offer_ids)) + if active: + ids = ids.filter(model.revoked.is_(False)) + ids = ids.group_by(model.offer_id) + return {row.offer_id: row for row in session.query(model).filter(model.id.in_(ids))} + observations = latest(OfferObservation) + evaluations = latest(OfferEvaluationRecord) + quotes = latest(ManualQuote, active=True) + for offer in offers: + listing = listings[offer.listing_id] + task = tasks.get(listing.task_id) + observation, quote, record = observations.get(offer.id), quotes.get(offer.id), evaluations.get(offer.id) + item = {"id": offer.id, "key": offer.offer_key, "name": offer.name, + "attributes": offer.attributes, "availability": offer.availability} + if observation is None: + item.update(price_state="unknown", effective_price_cents=None, source=None, + fingerprint="", reason="缺少价格证据", quote_id=None, + evaluation={"eligible": False}) + result[listing.id].append(item) + continue + candidate = decode_offer(observation.payload) + scoped_claims = [c for c in candidate.claims if c.scope_key in (None, "", candidate.key)] + expiry = min((utc(c.observed_at) for c in scoped_claims), default=utc(now)) + timedelta(minutes=task.interval_minutes if task else 20) + item["observation_fresh"] = bool(scoped_claims and expiry > utc(now) and + all(utc(c.observed_at) <= utc(now) + timedelta(seconds=5) for c in scoped_claims)) + item["target_match"] = match_variant(task.target_variant, candidate.attributes, listing.title) if task else "uncertain" + value = None if quote is None else QuoteInput(candidate.key, quote.amount_cents, + utc(quote.confirmed_at), utc(quote.expires_at), quote.evidence_fingerprint, quote.revoked) + ttl = (task.interval_minutes if task else 20) * 60 + decision = decide_price(candidate, value, now, 0 if task and not task.fetch_detail else ttl) + item["scope_current"] = offer.seen_in_latest_detail + item["observed_price_cents"] = decision.amount_cents + if not offer.seen_in_latest_detail: + decision = PriceDecision("unknown", None, None, (), "本次详情未核验此规格,旧观察仅供参考", None) + item.update(fingerprint=observation.fingerprint, price_state=decision.state, + effective_price_cents=decision.amount_cents, source=decision.source, + fresh_until=decision.fresh_until.isoformat() if decision.fresh_until else None, + reason=decision.reason, quote_id=quote.id if quote else None) + item.update(quote_confirmed_at=utc(quote.confirmed_at).isoformat() if quote else None, + quote_expires_at=utc(quote.expires_at).isoformat() if quote else None) + evaluation = dict(record.payload or {}) if record else {} + item["evaluation_current"] = (offer.seen_in_latest_detail and task is not None and evaluation.get("task_version") == task.version + and evaluation.get("fingerprint") == observation.fingerprint) + valid = (offer.seen_in_latest_detail and task is not None and task.enabled and not listing.blocked + and decision.state == "exact" and evaluation.get("task_version") == task.version + and evaluation.get("fingerprint") == observation.fingerprint + and evaluation.get("source") == decision.source + and evaluation.get("amount_cents") == decision.amount_cents + and (not task.min_condition_score or self.vision_enabled)) + if not valid: + evaluation.update(eligible=False, value_score=None, best_of_batch=False, satisfaction=None) + item["evaluation"] = evaluation + result[listing.id].append(item) + return result + + +def present_listings(listings, factory, vision_enabled=True): + from types import SimpleNamespace + + pipeline = OfferPipeline(factory, vision_enabled=vision_enabled) + result = [] + snapshots = pipeline.snapshot_many([listing.id for listing in listings], datetime.now().astimezone()) + for listing in listings: + view = SimpleNamespace(**{col.name: getattr(listing, col.name) for col in Listing.__table__.columns}) + view.image_urls = [f"/media/legacy-listing/{listing.id}/image/{i}" for i, _ in enumerate((listing.image_urls or [])[:32])] + offers = snapshots.get(listing.id, []) + eligible = [o for o in offers if (o.get("evaluation") or {}).get("eligible")] + exact = [o for o in offers if o["price_state"] == "exact"] + matched_exact = [o for o in exact if o.get("target_match") == "matched"] + cheapest = min(matched_exact, key=lambda o: o["effective_price_cents"], default=None) + view.price_confirmed = bool(eligible) + view.price_state = "exact" if exact else "pending" + view.exact_offer_count = len(matched_exact) + view.effective_price_cents = cheapest["effective_price_cents"] if cheapest else None + view.effective_price_source = cheapest["source"] if cheapest else None + current_media = [o for o in offers if o.get("evaluation_current") and o.get("target_match") == "matched"] + if len(current_media) == 1: + assessment = current_media[0].get("evaluation") or {} + view.condition_score = assessment.get("condition_score") + view.requirement_match = assessment.get("requirement_match") + if assessment.get("media") is not None: + view.condition_detail = assessment["media"] + elif offers: + view.condition_score = None + view.condition_detail = {"reason": "请查看当前规格的媒体结果,历史分析不代表新条件"} + if not eligible: + view.value_score, view.best_of_batch, view.satisfaction = None, False, 0.0 + elif any((o.get("evaluation") or {}).get("status") == "ready" for o in eligible): + chosen = max((o["evaluation"] for o in eligible), key=lambda e: e.get("satisfaction") or 0) + view.value_score, view.satisfaction = chosen.get("value_score"), chosen.get("satisfaction") or 0 + view.best_of_batch = any(o["evaluation"].get("best_of_batch", False) for o in eligible) + result.append(view) + return result diff --git a/goodprice/services/settings_service.py b/goodprice/services/settings_service.py index 028f040..d1693b5 100644 --- a/goodprice/services/settings_service.py +++ b/goodprice/services/settings_service.py @@ -1,7 +1,7 @@ from typing import Optional -from goodprice.config import Settings -from goodprice.models import AppSetting +from goodprice.config import SECRET_FIELDS, Settings +from goodprice.models import AppSetting, SchemaVersion # 设置字段只在 goodprice.config.Settings 定义一次。运行时值与数据库覆盖都经同一个 # pydantic 模型校验与类型转换(int/float/bool 由字段类型自动完成),不再维护第二份 @@ -17,22 +17,36 @@ def __init__(self, session_factory, base: Optional[Settings] = None): def _overrides(self, session) -> dict[str, str]: return {row.key: row.value for row in session.query(AppSetting).all()} - def _merge(self, overrides: dict[str, str]) -> RuntimeSettings: + def _merge(self, overrides: dict[str, str], legacy: bool = False) -> RuntimeSettings: values = self._base.model_dump() + if legacy and "media_total_limit" not in self._base.model_fields_set: + values["media_total_limit"] = 4 for key, value in overrides.items(): - if key in values and value != "": + if key in values and (value != "" or key in SECRET_FIELDS): values[key] = value return Settings(_env_file=None, **values) def get(self) -> RuntimeSettings: with self._session_factory() as session: - return self._merge(self._overrides(session)) + marker = session.get(SchemaVersion, 1) + return self._merge(self._overrides(session), bool(marker and marker.legacy_install)) - def set_many(self, values: dict[str, str]) -> RuntimeSettings: + def set_many(self, values: dict[str, str], *, clear_keys=frozenset()) -> RuntimeSettings: + if set(clear_keys) - SECRET_FIELDS: + raise ValueError("Only credentials can be explicitly cleared") + values = {**values, **dict.fromkeys(clear_keys, "")} with self._session_factory() as session: + overrides = self._overrides(session) + for key, value in values.items(): + if value == "" and key not in clear_keys: + overrides.pop(key, None) + else: + overrides[key] = value + marker = session.get(SchemaVersion, 1) + validated = self._merge(overrides, bool(marker and marker.legacy_install)) for key, value in values.items(): row = session.get(AppSetting, key) - if value == "": + if value == "" and key not in clear_keys: if row: session.delete(row) elif row: @@ -40,4 +54,4 @@ def set_many(self, values: dict[str, str]) -> RuntimeSettings: else: session.add(AppSetting(key=key, value=value)) session.commit() - return self._merge(self._overrides(session)) + return validated diff --git a/goodprice/services/task_service.py b/goodprice/services/task_service.py index 84c3369..9133264 100644 --- a/goodprice/services/task_service.py +++ b/goodprice/services/task_service.py @@ -4,7 +4,7 @@ from typing import Optional from goodprice.constants import SCORE_MAX -from goodprice.models import WatchTask +from goodprice.models import Listing, Notification, WatchTask def normalize_task_data(data: dict) -> dict: @@ -48,6 +48,8 @@ def number(name: str) -> float: "platform": str(data.get("platform") or "xianyu").strip() or "xianyu", "interval_minutes": interval_minutes, "fetch_detail": bool(data.get("fetch_detail", True)), + "target_variant": str(data.get("target_variant") or "").strip(), + "notify_pending_price": bool(data.get("notify_pending_price", False)), "enabled": bool(data.get("enabled", True)), } @@ -64,6 +66,8 @@ def number(name: str) -> float: "platform", "interval_minutes", "fetch_detail", + "target_variant", + "notify_pending_price", "enabled", ) @@ -104,19 +108,26 @@ def create_task(self, data: dict) -> WatchTask: return task def toggle_task(self, task_id: int) -> Optional[WatchTask]: + from goodprice.execution.store import invalidate_in_session with self._session_factory() as session: task = session.get(WatchTask, task_id) if task: task.enabled = not task.enabled + task.version += 1 + invalidate_in_session(session, task_id) session.commit() session.refresh(task) return task def delete_task(self, task_id: int) -> bool: + from goodprice.execution.store import invalidate_in_session with self._session_factory() as session: task = session.get(WatchTask, task_id) if not task: return False + invalidate_in_session(session, task_id) + session.query(Listing).filter_by(task_id=task_id).update({"task_id": None}) + session.query(Notification).filter_by(task_id=task_id).update({"task_id": None}) session.delete(task) session.commit() return True @@ -126,6 +137,7 @@ def enabled_tasks(self) -> list[WatchTask]: return session.query(WatchTask).filter(WatchTask.enabled.is_(True)).all() def update_task(self, task_id: int, data: dict) -> Optional[WatchTask]: + from goodprice.execution.store import invalidate_in_session with self._session_factory() as session: task = session.get(WatchTask, task_id) if not task: @@ -133,6 +145,11 @@ def update_task(self, task_id: int, data: dict) -> Optional[WatchTask]: values = normalize_task_data( {**{field: getattr(task, field) for field in TASK_FIELDS}, **data} ) + for field in TASK_FIELDS: + if getattr(task, field) != values[field]: + task.version += 1 + invalidate_in_session(session, task_id) + break for field in TASK_FIELDS: setattr(task, field, values[field]) session.commit() diff --git a/goodprice/web/list_context.py b/goodprice/web/list_context.py new file mode 100644 index 0000000..7f31cfc --- /dev/null +++ b/goodprice/web/list_context.py @@ -0,0 +1,64 @@ +from dataclasses import dataclass +from urllib.parse import parse_qsl, unquote, urlencode, urlsplit + +FIELDS = {"task_id", "show", "sort", "price_state", "loaded"} + + +@dataclass(frozen=True) +class ListContext: + task_id: int | None = None + show: str = "active" + sort: str = "satisfaction" + price_state: str = "all" + loaded: int = 100 + + @classmethod + def from_query(cls, query): + def single(name, default): + values = query.getlist(name) if hasattr(query, "getlist") else [query.get(name, default)] + return str(values[0])[:100] if len(values) == 1 else default + raw_id = single("task_id", "") + task_id = int(raw_id) if raw_id.isascii() and raw_id.isdigit() and 0 < int(raw_id) <= 2**31-1 else None + raw_loaded = single("loaded", "100") + loaded = min(1000, max(100, int(raw_loaded))) if raw_loaded.isascii() and raw_loaded.isdigit() else 100 + def choice(name, options, default): + raw = single(name, default) + return raw if raw in options else default + return cls(task_id, choice("show", {"active", "gone", "blocked", "all", "unseen"}, "active"), + choice("sort", {"satisfaction", "price_asc", "price_desc", "newest"}, "satisfaction"), + choice("price_state", {"all", "exact", "pending"}, "all"), loaded) + + def values(self): + values = {"show": self.show, "sort": self.sort} + if self.task_id: + values = {"task_id": str(self.task_id), **values} + if self.price_state != "all": + values["price_state"] = self.price_state + if self.loaded != 100: + values["loaded"] = str(self.loaded) + return values + + def url(self, path="/listings"): + return path + "?" + urlencode(self.values()) + + +def safe_return_to(raw, fallback="/listings"): + if not raw or len(raw) > 2048: + return fallback + decoded = raw + for _ in range(3): + if any(ord(c) < 32 for c in decoded) or "\\" in decoded: + return fallback + decoded = unquote(decoded) + try: + parts = urlsplit(raw) + except ValueError: + return fallback + if parts.scheme or parts.netloc or parts.path != "/listings" or parts.fragment: + return fallback + pairs = parse_qsl(parts.query, keep_blank_values=True) + if any(k not in FIELDS for k, v in pairs) or len({k for k, v in pairs}) != len(pairs): + return fallback + context = ListContext.from_query(dict(pairs)) + canonical = context.values() + return "/listings" + ("?" + urlencode({k: canonical[k] for k, v in pairs if k in canonical}) if pairs else "") diff --git a/goodprice/web/media_routes.py b/goodprice/web/media_routes.py new file mode 100644 index 0000000..cd38232 --- /dev/null +++ b/goodprice/web/media_routes.py @@ -0,0 +1,159 @@ +import hashlib +import shutil + +from fastapi import APIRouter, HTTPException, Request +from fastapi.responses import FileResponse, RedirectResponse +from pydantic import ValidationError + +from goodprice.config import Settings +from goodprice.media.budget import MediaBudget +from goodprice.models import Listing, MediaAsset + +router = APIRouter() +MEDIA_FIELDS = {name for name in Settings.model_fields if name.startswith("media_")} | {"video_analysis_enabled"} +MEDIA_LABELS = dict(zip([ + "media_image_count", "media_video_count", "media_frames_per_video", "media_image_quota", + "media_frame_quota", "media_total_limit", "media_model_limit", "media_video_max_mib", + "media_video_max_seconds", "media_cache_mib", "media_retention_days", "media_download_timeout", + "media_decode_timeout", "media_request_max_mib", "media_ffmpeg_threads"], [ + "每商品原图保留数(1–32)", "每商品视频数(1–3)", "每视频采样帧数(1–24)", + "每请求原图配额(0–16)", "每请求帧配额(0–16)", "每请求总输入上限(1–16)", + "已确认模型上限(未知为8,最高16)", "视频大小上限(10–200 MiB)", + "视频时长上限(15–300秒)", "缓存容量(256–10240 MiB)", "保留天数(1–30)", + "下载超时(10–120秒)", "解码超时(10–120秒)", "请求体上限(1–32 MiB)", + "每进程解码线程(1–4)"], strict=True)) + + +def media_budget(settings): + return MediaBudget(settings.data_root / "media-cache", settings.media_cache_mib * 1024**2, + settings.media_retention_days * 86400) + + +def capabilities(settings): + return {"ffmpeg_available": bool(shutil.which("ffmpeg") and shutil.which("ffprobe")), + "video_enabled": settings.video_analysis_enabled, + "total_limit": settings.media_total_limit, "model_limit": settings.media_model_limit, + "image_quota": settings.media_image_quota, "frame_quota": settings.media_frame_quota, + "request_max_mib": settings.media_request_max_mib} + + +@router.get("/api/media/capabilities") +def media_capabilities(request: Request): + return capabilities(request.app.state.settings_service.get()) + + +class PinnedFileResponse(FileResponse): + def __init__(self, path, pin): + super().__init__(path, media_type="image/jpeg", headers={"X-Content-Type-Options": "nosniff"}) + self.pin = pin + + async def __call__(self, scope, receive, send): + try: + await super().__call__(scope, receive, send) + finally: + self.pin.__exit__(None, None, None) + + +def _pinned_response(budget, path): + with budget.state.lock: + path = budget.checked(path) + if not path.is_file() or path.suffix != ".jpg": + raise ValueError() + pin = budget.pin([path]) + pin.__enter__() + return PinnedFileResponse(path, pin) + + +@router.get("/media/{asset_id}") +def media_file(asset_id: int, request: Request): + settings = request.app.state.settings_service.get() + budget = media_budget(settings) + with request.app.state.session_factory() as session: + asset = session.get(MediaAsset, asset_id) + if asset is None or asset.kind not in {"image", "frame"} or asset.status == "superseded": + raise HTTPException(404, "证据文件不可用或已清理") + if asset.relative_path: + try: + return _pinned_response(budget, budget.root / asset.relative_path) + except ValueError: + raise HTTPException(404, "证据文件不可用或已清理") from None + # Viewing an image does not require vision to be enabled or configured. + # Source requests use the SAME pinned public downloader and disk budget. + if asset.kind == "image" and asset.source_url and asset.status != "unsupported": + from goodprice.media.processing import MediaProcessor + from goodprice.media.types import MediaCandidate + + guard = getattr(request.app.state, "resource_guard", None) + if guard: + settings = guard.media_settings(settings) + processor = MediaProcessor(budget) + with processor.session([MediaCandidate(asset.key, "image", asset.source_url, asset.offer_key)], settings) as (evidence, _): + if evidence: + image = evidence[0] + response = _pinned_response(budget, image.local_path) + with request.app.state.session_factory() as session: + row = session.get(MediaAsset, asset_id) + if row: + row.relative_path = image.local_path.relative_to(budget.root).as_posix() + row.content_hash = image.content_hash + row.status = "ready" + session.commit() + return response + raise HTTPException(404, "证据文件不可用或已清理") + + +@router.get("/media/legacy-listing/{listing_id}/image/{index}") +def legacy_listing_image(listing_id: int, index: int, request: Request): + # An internal listing/index reference migrates a legacy image lazily on view; + # neither pages nor APIs publish expiring source URLs or take caller URLs. + with request.app.state.session_factory() as session: + listing = session.get(Listing, listing_id) + if listing is None or not 0 <= index < min(32, len(listing.image_urls or [])): + raise HTTPException(404, "图片不存在") + source = listing.image_urls[index] + key = "legacy:" + hashlib.sha256(source.encode()).hexdigest()[:32] + row = session.query(MediaAsset).filter(MediaAsset.listing_id == listing_id, + MediaAsset.source_url == source, MediaAsset.kind == "image", + MediaAsset.status != "superseded").order_by(MediaAsset.id.desc()).first() + if row is None: + row = session.query(MediaAsset).filter_by(listing_id=listing_id, key=key).first() + if row: + row.status, row.relative_path, row.content_hash = "pending", None, None + session.commit() + if row is None: + row = MediaAsset(listing_id=listing_id, key=key, kind="image", source_url=source, + source_locator=f"legacy-image:{index}") + session.add(row) + session.commit() + asset_id = row.id + return media_file(asset_id, request) + + +@router.post("/settings/media") +async def save_media(request: Request): + form = await request.form() + if set(form) - MEDIA_FIELDS - {"confirm_cost", "media_form"}: + raise HTTPException(422, "媒体分类含不可修改的字段") + values = {k: str(v) for k, v in form.items() if k in MEDIA_FIELDS} + if "media_form" in form: + values["video_analysis_enabled"] = "1" if "video_analysis_enabled" in form else "0" + service = request.app.state.settings_service + before = service.get() + try: + proposed = Settings(_env_file=None, **(before.model_dump() | values)) + increases = any(getattr(proposed, k) > getattr(before, k) for k in MEDIA_FIELDS + if k != "media_retention_days") + if increases and form.get("confirm_cost") != "1": + raise HTTPException(422, "增加媒体预算需确认费用、响应延迟、内存、存储与模型限制") + service.set_many(values) + except ValidationError: + raise HTTPException(422, "媒体设置无效:请检查范围、模型上限和缓存容量") from None + return RedirectResponse("/settings#media", status_code=303) + + +def public_assets(request, listing_id): + from goodprice.media.processing import MediaProcessor + from goodprice.services.media_service import MediaService + + return MediaService(request.app.state.session_factory, + MediaProcessor(media_budget(request.app.state.settings_service.get()))).public_assets(listing_id) diff --git a/goodprice/web/offer_routes.py b/goodprice/web/offer_routes.py new file mode 100644 index 0000000..970853b --- /dev/null +++ b/goodprice/web/offer_routes.py @@ -0,0 +1,66 @@ +from datetime import datetime, timedelta, timezone +from urllib.parse import urlencode + +from fastapi import APIRouter, HTTPException, Request +from fastapi.responses import RedirectResponse +from pydantic import BaseModel, Field, ValidationError + +from goodprice.models import ListingOffer +from goodprice.offers.pricing import parse_money_cents +from goodprice.offers.service import OfferService +from goodprice.offers.types import QuoteInput +from goodprice.services.offer_pipeline import OfferPipeline +from goodprice.web.list_context import safe_return_to + +router = APIRouter() + + +class QuoteForm(BaseModel): + amount: str = Field(min_length=1, max_length=40) + fingerprint: str = Field(min_length=1, max_length=64) + valid_days: int = Field(default=7, ge=1, le=30) + + +@router.get("/api/listings/{listing_id}/offers") +def list_offers(request: Request, listing_id: int): + try: + return OfferPipeline(request.app.state.session_factory).snapshot(listing_id, datetime.now(timezone.utc)) + except LookupError as exc: + raise HTTPException(404, str(exc)) from exc + + +@router.post("/offers/{offer_id}/quotes") +async def confirm_quote(request: Request, offer_id: int): + factory = request.app.state.session_factory + with factory() as session: + offer = session.get(ListingOffer, offer_id) + if offer is None: + raise HTTPException(404, "规格不存在") + key, listing_id = offer.offer_key, offer.listing_id + json_request = "application/json" in request.headers.get("content-type", "") + try: + form = QuoteForm.model_validate(await request.json() if json_request else dict(await request.form())) + now = datetime.now(timezone.utc) + row_id = OfferService(factory).confirm_quote(offer_id, QuoteInput( + key, parse_money_cents(form.amount), now, now + timedelta(days=form.valid_days), form.fingerprint)) + except (ValidationError, ValueError, TypeError) as exc: + raise HTTPException(422, "报价无效或价格证据已变化,请刷新后确认") from exc + if json_request: + return {"id": row_id} + return RedirectResponse(f"/listings/{listing_id}?" + urlencode({"return_to": safe_return_to(request.query_params.get("return_to"))}), status_code=303) + + +@router.post("/quotes/{quote_id}/revoke") +async def revoke_quote(request: Request, quote_id: int): + from goodprice.models import ManualQuote + + factory = request.app.state.session_factory + with factory() as session: + quote = session.get(ManualQuote, quote_id) + if quote is None: + raise HTTPException(404, "报价不存在") + listing_id = session.get(ListingOffer, quote.offer_id).listing_id + OfferService(factory).revoke_quote(quote_id) + if "application/json" in request.headers.get("content-type", ""): + return {"revoked": True} + return RedirectResponse(f"/listings/{listing_id}?" + urlencode({"return_to": safe_return_to(request.query_params.get("return_to"))}), status_code=303) diff --git a/goodprice/web/resource_routes.py b/goodprice/web/resource_routes.py new file mode 100644 index 0000000..734bb94 --- /dev/null +++ b/goodprice/web/resource_routes.py @@ -0,0 +1,54 @@ +from dataclasses import asdict + +from fastapi import APIRouter, HTTPException, Request +from fastapi.responses import RedirectResponse + +from goodprice.execution.limits import ResourceLimits +from goodprice.resources import detect_resources, recommend_profile, resource_profile + +router = APIRouter() +RESOURCE_LABELS = { + "execution_active_runs": "同时推进的任务数(1–4)", + "execution_crawler_workers": "抓取工作线程(1–2)", + "execution_model_workers": "模型请求并发(1–8)", + "execution_media_workers": "媒体处理并发(1–2)", + "execution_platform_interval_seconds": "平台导航最小间隔(5–300秒)", + "execution_model_rpm": "模型 HTTP 请求上限(1–60次/分钟)", + "execution_global_items": "全局未完成步骤(1–200)", + "execution_task_items": "单任务未完成步骤(1–60)", + "execution_run_details": "每轮详情预算(1–100)", + "execution_run_http_attempts": "每轮模型 HTTP 尝试(1–200)", + "execution_run_minutes": "每轮墙钟预算(5–60分钟)", +} + + +@router.get("/api/resources/recommendation") +def resource_recommendation(request: Request): + snapshot = detect_resources() + return {"snapshot": asdict(snapshot), "recommendation": recommend_profile(snapshot), **effective_resources(request)} + + +def effective_resources(request): + configured = ResourceLimits.from_settings(request.app.state.settings_service.get()) + guard = getattr(request.app.state, "resource_guard", None) + current = guard.evaluate(configured) if guard else None + controller = request.app.state.coordinator + return {"configured": asdict(configured), "effective": asdict(current.limits if current else configured), + "paused": sorted(current.paused) if current else [], "reasons": list(current.reasons) if current else [], + "running_tasks": len(controller.running_ids()), "queued_tasks": len(controller.queued_ids())} + + +@router.post("/settings/resources/apply-profile") +async def apply_profile(request: Request): + form = await request.form() + if form.get("confirm") != "1" or set(form) - {"name", "confirm"}: + raise HTTPException(422, "请明确确认应用资源配置") + try: + values = resource_profile(form.get("name", "")) + except ValueError: + raise HTTPException(422, "未知资源配置") from None + runtime = request.app.state.settings_service.set_many({k: str(v) for k, v in values.items()}) + coordinator = getattr(request.app.state, "coordinator", None) + if coordinator: + coordinator.update_limits(ResourceLimits.from_settings(runtime)) + return RedirectResponse("/settings#resources", status_code=303) diff --git a/goodprice/web/routes.py b/goodprice/web/routes.py index 942e9d2..f988eee 100644 --- a/goodprice/web/routes.py +++ b/goodprice/web/routes.py @@ -1,16 +1,17 @@ import logging -import threading from pathlib import Path from typing import Optional from urllib.parse import parse_qsl, urlencode -from fastapi import APIRouter, Form, Query, Request +from fastapi import APIRouter, Form, HTTPException, Query, Request from fastapi.responses import HTMLResponse, RedirectResponse from fastapi.templating import Jinja2Templates from pydantic import BaseModel, Field, field_validator, model_validator -from sqlalchemy import func, text +from sqlalchemy import and_, case, func, select, text from goodprice.constants import SCORE_MAX +from goodprice.web.list_context import ListContext, safe_return_to +from goodprice.web.media_routes import MEDIA_LABELS, capabilities, public_assets logger = logging.getLogger(__name__) @@ -22,15 +23,17 @@ def _optional_int(value: str) -> Optional[int]: """把查询参数安全转成 int;非数字返回 None,避免非法输入导致 500。""" - return int(value) if str(value).isdigit() else None + return int(value) if str(value).isascii() and str(value).isdigit() else None -def _apply_listing_filters(query, *, task_id, show, sort): +def _apply_listing_filters(query, *, task_id, show, sort, price_state="all"): """列表页 / 加载更多 / API 共用的过滤与排序,避免三处 if/elif 语义分叉。 show 的取值由模板下拉决定;未知值(含 "all")不追加过滤条件。 """ - from goodprice.models import Listing + from datetime import datetime, timezone + + from goodprice.models import Listing, ListingOffer, WatchTask if task_id: query = query.filter(Listing.task_id == task_id) @@ -40,13 +43,25 @@ def _apply_listing_filters(query, *, task_id, show, sort): query = query.filter(Listing.status == "gone") elif show == "blocked": query = query.filter(Listing.blocked.is_(True)) + elif show == "unseen": + query = query.filter(Listing.status == "unseen") + version = select(WatchTask.version).where(WatchTask.id == Listing.task_id).correlate(Listing).scalar_subquery() + if price_state in {"exact", "pending"}: + exact = select(ListingOffer.id).where(ListingOffer.listing_id == Listing.id, + ListingOffer.effective_price_state == "exact", ListingOffer.price_valid_until > datetime.now(timezone.utc), + ListingOffer.price_task_version == version).correlate(Listing).exists() + query = query.filter(exact if price_state == "exact" else ~exact) + valid = and_(Listing.offer_verified_until > datetime.now(timezone.utc), + Listing.offer_price_cents.is_not(None), Listing.offer_task_version == version) + query = query.order_by(case((valid, 0), else_=1)) + price = case((valid, Listing.offer_price_cents), else_=None) if sort == "price_asc": - return query.order_by(Listing.price.asc()) + return query.order_by(price.asc(), Listing.id.desc()) if sort == "price_desc": - return query.order_by(Listing.price.desc()) + return query.order_by(price.desc(), Listing.id.desc()) if sort == "newest": return query.order_by(Listing.first_seen_at.desc()) - return query.order_by(Listing.satisfaction.desc(), Listing.first_seen_at.desc()) + return query.order_by(case((valid, Listing.satisfaction), else_=0).desc(), Listing.first_seen_at.desc(), Listing.id.desc()) @router.get("/healthz", include_in_schema=False) @@ -58,6 +73,8 @@ def healthz(request: Request): class TaskCreate(BaseModel): + target_variant: str = "" + notify_pending_price: bool = False name: str = "" keyword: str = Field(min_length=1) max_price: float = Field(default=0, ge=0, allow_inf_nan=False) @@ -136,10 +153,16 @@ def dashboard(request: Request): return templates.TemplateResponse( request, "dashboard.html", - {"stats": stats, "recent": recent, "active": "dashboard"}, + {"stats": stats, "recent": _present_recent(request, recent), "active": "dashboard"}, ) +def _present_recent(request, rows): + from goodprice.services.offer_pipeline import present_listings + return present_listings(rows, request.app.state.session_factory, + request.app.state.settings_service.get().vision_enabled) + + @router.get("/tasks", response_class=HTMLResponse) def tasks_page(request: Request): task_service, _ = _services(request) @@ -232,25 +255,52 @@ def notifications_page(request: Request): @router.post("/notifications/delete-batch") def delete_notifications_batch(request: Request, ids: list[int] = Form(...)): - with request.app.state.session_factory() as session: - from goodprice.models import Notification - - session.query(Notification).filter(Notification.id.in_(ids)).delete( - synchronize_session=False - ) - session.commit() + _delete_notification_history(request, ids) return RedirectResponse("/notifications", status_code=303) @router.post("/notifications/{notification_id}/delete") def delete_notification(request: Request, notification_id: int): - with request.app.state.session_factory() as session: - from goodprice.models import Notification + _delete_notification_history(request, [notification_id]) + return RedirectResponse("/notifications", status_code=303) - row = session.get(Notification, notification_id) - if row: - session.delete(row) - session.commit() + +def _delete_notification_history(request, ids): + from goodprice.models import Notification, NotificationDelivery + delivery = request.app.state.coordinator.pipeline.delivery + with delivery.lock, request.app.state.session_factory() as session: + rows = session.query(Notification).filter(Notification.id.in_(ids[:200])).all() + pending = [] + for history in rows: + delivery_id = (history.event_context or {}).get("delivery_id") + row = session.get(NotificationDelivery, delivery_id) if delivery_id else None + if row and row.state == "sending": + raise HTTPException(409, "该通知正在投递,无法撤回;请等待结果后操作") + if row and row.state != "sent": + pending.append(row) + for row in pending: + row.state = "cancelled_manual" + for history in rows: + session.delete(history) + session.commit() + + +@router.post("/notifications/deliveries/{delivery_id}/retry") +def retry_notification(request: Request, delivery_id: int, confirm: str = Form("")): + from datetime import datetime, timezone + + from goodprice.models import NotificationDelivery + with request.app.state.session_factory() as session: + row = session.get(NotificationDelivery, delivery_id) + if row is None: + raise HTTPException(404, "通知投递不存在") + if row.state == "uncertain" and confirm != "1": + raise HTTPException(422, "投递结果未知,核验后重试可能重复,请明确确认") + if row.state in {"sent", "sending", "cancelled_manual"}: + raise HTTPException(409, "当前状态不能重试") + controller = request.app.state.coordinator + controller.pipeline.delivery.queue_retry(delivery_id, datetime.now(timezone.utc)) + controller.wake() return RedirectResponse("/notifications", status_code=303) @@ -276,6 +326,8 @@ def api_list_notifications(request: Request): @router.post("/tasks") def create_task_form( request: Request, + target_variant: str = Form(""), + notify_pending_price: Optional[int] = Form(None), keyword: str = Form(...), name: str = Form(""), max_price: float = Form(0), @@ -291,6 +343,8 @@ def create_task_form( try: task_service.create_task( { + "target_variant": target_variant, + "notify_pending_price": bool(notify_pending_price), "keyword": keyword, "name": name, "max_price": max_price, @@ -358,6 +412,8 @@ def task_detail_page(request: Request, task_id: int): def edit_task_form( request: Request, task_id: int, + target_variant: str = Form(""), + notify_pending_price: Optional[int] = Form(None), keyword: str = Form(...), name: str = Form(""), max_price: float = Form(0), @@ -374,6 +430,8 @@ def edit_task_form( task_service.update_task( task_id, { + "target_variant": target_variant, + "notify_pending_price": bool(notify_pending_price), "keyword": keyword, "name": name, "max_price": max_price, @@ -404,9 +462,15 @@ def toggle_task(request: Request, task_id: int): @router.post("/tasks/{task_id}/run") def run_task(request: Request, task_id: int): - threading.Thread( - target=request.app.state.run_job, args=(task_id,), daemon=True - ).start() + from fastapi import HTTPException + task = request.app.state.task_service.get_task(task_id) + if task is None: + raise HTTPException(404, "任务不存在") + if not task.enabled: + raise HTTPException(409, "任务已停用,请先启用后执行") + queued = request.app.state.run_job(task_id) + if queued is False: + return RedirectResponse("/tasks?" + urlencode({"toast": "任务已在队列中,无需重复提交"}), status_code=303) return RedirectResponse(f"/tasks?run={task_id}", status_code=303) @@ -427,23 +491,31 @@ def listings_page( sort: str = Query("satisfaction"), show: str = Query("active"), ): + context = ListContext.from_query(request.query_params) + task_id, sort, show = context.task_id, context.sort, context.show + offset = min(10000, max(0, offset)) with request.app.state.session_factory() as session: from goodprice.models import Listing, WatchTask query = session.query(Listing) tasks = session.query(WatchTask).order_by(WatchTask.id).all() - task_id_int = _optional_int(task_id) - query = _apply_listing_filters(query, task_id=task_id_int, show=show, sort=sort) + task_id_int = task_id + query = _apply_listing_filters(query, task_id=task_id_int, show=show, sort=sort, price_state=context.price_state) query = query.offset(offset) - listings = query.limit(PAGE_SIZE).all() + listings = query.limit(context.loaded).all() + from goodprice.services.offer_pipeline import present_listings + + listings = present_listings(listings, request.app.state.session_factory, + request.app.state.settings_service.get().vision_enabled) notify_counts = _notify_counts(session, listings) - more = len(listings) >= PAGE_SIZE - partial_url = f"/listings?partial=1&task_id={task_id or ''}&sort={sort}&show={show}" + more = len(listings) >= context.loaded and context.loaded < 1000 + partial_url = context.url() + "&partial=1" + return_to = context.url() if partial: return templates.TemplateResponse( request, "listings_grid.html", - {"listings": listings, "notify_counts": notify_counts}, + {"listings": listings, "notify_counts": notify_counts, "return_to": return_to}, ) running_ids = request.app.state.guard.running_ids() return templates.TemplateResponse( @@ -456,13 +528,13 @@ def listings_page( "task_id": task_id, "sort": sort, "show": show, + "price_state": context.price_state, "partial_url": partial_url, "notify_counts": notify_counts, "more": more, - "load_more_url": ( - f"/listings/more?offset={offset + len(listings)}" - f"&task_id={task_id or ''}&sort={sort}&show={show}" - ), + "return_to": return_to, + "load_more_url": context.url("/listings/more") + f"&offset={offset + len(listings)}", + "load_more_fallback": ListContext(context.task_id, show, sort, context.price_state, min(1000, context.loaded + PAGE_SIZE)).url(), "active": "listings", }, ) @@ -493,21 +565,39 @@ def listings_more( sort: str = Query("satisfaction"), show: str = Query("active"), ): + context = ListContext.from_query(request.query_params) + task_id, sort, show = context.task_id, context.sort, context.show + offset = min(1000, max(0, offset)) with request.app.state.session_factory() as session: from goodprice.models import Listing query = session.query(Listing) - task_id_int = _optional_int(task_id) - query = _apply_listing_filters(query, task_id=task_id_int, show=show, sort=sort) - listings = query.offset(offset).limit(PAGE_SIZE).all() + task_id_int = task_id + query = _apply_listing_filters(query, task_id=task_id_int, show=show, sort=sort, price_state=context.price_state) + listings = query.offset(offset).limit(min(PAGE_SIZE, 1000-offset)).all() + from goodprice.services.offer_pipeline import present_listings + + listings = present_listings(listings, request.app.state.session_factory, + request.app.state.settings_service.get().vision_enabled) notify_counts = _notify_counts(session, listings) - return templates.TemplateResponse( - request, "listings_cards.html", {"listings": listings, "notify_counts": notify_counts} - ) + loaded = min(1000, max(100, offset + len(listings))) + current = ListContext(task_id, show, sort, context.price_state, loaded) + response = templates.TemplateResponse(request, "listings_more.html", { + "listings": listings, "notify_counts": notify_counts, "return_to": current.url(), + "more": len(listings) == PAGE_SIZE and loaded < 1000, + "load_more_url": current.url("/listings/more") + f"&offset={loaded}", + "load_more_fallback": ListContext(task_id, show, sort, context.price_state, min(1000, loaded + PAGE_SIZE)).url()}) + response.headers["HX-Replace-Url"] = current.url() + return response @router.get("/listings/{listing_id}", response_class=HTMLResponse) def listing_detail_page(request: Request, listing_id: int): + from datetime import datetime as dt + from datetime import timezone + + from goodprice.services.offer_pipeline import OfferPipeline, present_listings + with request.app.state.session_factory() as session: from goodprice.models import Listing, Notification, PriceSnapshot @@ -538,13 +628,17 @@ def listing_detail_page(request: Request, listing_id: int): request, "listings_detail.html", { - "listing": listing, + "listing": present_listings([listing], request.app.state.session_factory, + request.app.state.settings_service.get().vision_enabled)[0], "snapshots": snapshots, "notifications": notifications, "notify_count": notify_count, "first_price": first_price, "drop_pct": drop_pct, "drop_pct_threshold": DROP_PCT_DISPLAY_THRESHOLD, + "offers": OfferPipeline(request.app.state.session_factory).snapshot(listing_id, dt.now(timezone.utc)), + "media_assets": public_assets(request, listing_id), + "return_to": safe_return_to(request.query_params.get("return_to"), "/listings"), "active": "listings", }, ) @@ -552,10 +646,20 @@ def listing_detail_page(request: Request, listing_id: int): @router.post("/listings/{listing_id}/reanalyze") def reanalyze_listing(request: Request, listing_id: int): - threading.Thread( - target=request.app.state.run_reanalyze, args=(listing_id,), daemon=True - ).start() - return RedirectResponse(f"/listings/{listing_id}?toast=已提交重新分析", status_code=303) + from fastapi import HTTPException + with request.app.state.session_factory() as session: + from goodprice.models import Listing, WatchTask + listing = session.get(Listing, listing_id) + if listing is None: + raise HTTPException(404, "商品不存在") + task = session.get(WatchTask, listing.task_id) if listing.task_id else None + if task is None or not task.enabled: + raise HTTPException(409, "原任务已删除或停用,请先检查任务状态") + accepted = request.app.state.run_reanalyze(listing_id) + if accepted is False: + raise HTTPException(429, "重评队列暂满,请稍后重试") + return RedirectResponse(f"/listings/{listing_id}?" + urlencode({"toast": "已提交重新分析", + "return_to": safe_return_to(request.query_params.get("return_to"), "/listings")}), status_code=303) @router.post("/listings/delete-batch") @@ -606,12 +710,18 @@ def unblock_seller(request: Request, listing_id: int): def _listings_back(request: Request, toast: str) -> str: - params = {} - ref = request.headers.get("referer", "") - if "?" in ref: - params = { - k: v for k, v in parse_qsl(ref.split("?", 1)[1]) if k in ("task_id", "sort", "show") - } + raw = request.query_params.get("return_to") + if raw is None: + # Legacy same-origin forms: reduce Referer to a whitelisted relative path. + from urllib.parse import urlsplit + try: + ref = urlsplit(request.headers.get("referer", "")) + except ValueError: + ref = urlsplit("") + if (not ref.netloc and not ref.scheme or ref.netloc == request.url.netloc and ref.scheme == request.url.scheme) and ref.path == "/listings": + raw = ref.path + ("?" + ref.query if ref.query else "") + target = safe_return_to(raw, "/listings") + params = dict(parse_qsl(target.split("?", 1)[1])) if "?" in target else {} params["toast"] = toast return "/listings?" + urlencode(params) @@ -644,6 +754,13 @@ def _set_blocked(request: Request, listing_id: int, blocked: bool, seller: bool) @router.get("/settings", response_class=HTMLResponse) def settings_page(request: Request): + return render_settings(request) + + +def render_settings(request, overrides=None, error=None, status_code=200): + from goodprice.resources import detect_resources, recommend_profile + from goodprice.web.resource_routes import RESOURCE_LABELS, effective_resources + resource_snapshot = detect_resources() _, settings_service = _services(request) settings = settings_service.get() login_session = getattr(request.app.state, "login_session", None) @@ -654,12 +771,20 @@ def settings_page(request: Request): request, "settings.html", { - "settings": settings, + "settings": settings.model_dump() | (overrides or {}), + "settings_error": error, + "resource_snapshot": resource_snapshot, + "resource_recommendation": recommend_profile(resource_snapshot), + "resource_labels": RESOURCE_LABELS, + "resource_effective": effective_resources(request), + "media_capabilities": capabilities(settings), + "media_fields": [name for name in type(settings).model_fields if name.startswith("media_")], + "media_labels": MEDIA_LABELS, "login_status": login_status, "login_message": login_message, **_login_context(login_session, settings.novnc_port), "active": "settings", - }, + }, status_code=status_code, ) @@ -701,12 +826,21 @@ def settings_login_status(request: Request): @router.post("/settings") async def save_settings(request: Request): + from fastapi import HTTPException + from pydantic import ValidationError + from goodprice.config import SECRET_FIELDS, Settings + from goodprice.web.media_routes import MEDIA_FIELDS _, settings_service = _services(request) form = await request.form() + forbidden = MEDIA_FIELDS | {"database_url", "data_root", "app_name", "runtime_mode", "enable_novnc", "novnc_password_file", "novnc_port"} + if set(form) & forbidden: + raise HTTPException(422, "请使用媒体分类修改预算;部署参数仅可通过环境配置") values: dict[str, str] = {} for field in Settings.model_fields: + if field in forbidden: + continue if field.endswith("_enabled"): # 复选框未勾选时不会出现在表单里,统一记为“关闭”。 raw = form.get(field) @@ -718,7 +852,10 @@ async def save_settings(request: Request): for field in SECRET_FIELDS: if values.get(field) == "": values.pop(field) # 留空 = 保持原值 - runtime = settings_service.set_many(values) + try: + runtime = settings_service.set_many(values) + except ValidationError: + raise HTTPException(422, "设置值无效,未保存任何更改") from None from goodprice.services.satisfaction import backfill_satisfaction backfill_satisfaction(request.app.state.session_factory, vision_enabled=runtime.vision_enabled) @@ -971,14 +1108,19 @@ def api_list_listings( task_id: str = Query(""), sort: str = Query("satisfaction"), show: str = Query("active"), + price_state: str = Query("all"), ): with request.app.state.session_factory() as session: from goodprice.models import Listing query = session.query(Listing) task_id_int = _optional_int(task_id) - query = _apply_listing_filters(query, task_id=task_id_int, show=show, sort=sort) + query = _apply_listing_filters(query, task_id=task_id_int, show=show, sort=sort, price_state=price_state) rows = query.offset(offset).limit(PAGE_SIZE).all() + from goodprice.services.offer_pipeline import present_listings + + rows = present_listings(rows, request.app.state.session_factory, + request.app.state.settings_service.get().vision_enabled) return [ { "id": row.id, @@ -986,6 +1128,10 @@ def api_list_listings( "external_id": row.external_id, "title": row.title, "price": row.price, + "effective_price_cents": row.effective_price_cents, + "price_state": row.price_state, + "price_confirmed": row.price_confirmed, + "value_score": row.value_score, "url": row.url, "image_urls": row.image_urls, "condition_score": row.condition_score, diff --git a/goodprice/web/settings_sections.py b/goodprice/web/settings_sections.py new file mode 100644 index 0000000..28f71d8 --- /dev/null +++ b/goodprice/web/settings_sections.py @@ -0,0 +1,54 @@ +from fastapi import APIRouter, Request +from fastapi.responses import RedirectResponse +from pydantic import ValidationError + +from goodprice.config import SECRET_FIELDS, Settings + +router = APIRouter() +SECTIONS = { + "account": frozenset({"xianyu_cookie", "proxy", "default_crawl_interval_minutes", "default_crawl_jitter_minutes"}), + "models": frozenset({"llm_base_url", "llm_api_key", "llm_model", "llm_api_format", "vision_base_url", "vision_api_key", + "vision_model", "vision_api_format", "vision_enabled", "jev_enabled", "jev_auto_threshold", "jev_backend", "jev_api_key"}), + "notifications": frozenset({"serverchan_sendkey", "serverchan_enabled", "wecom_webhook", "wecom_robot_enabled", + "feishu_webhook", "feishu_secret", "feishu_enabled", "gotify_url", "gotify_token", "gotify_priority", "gotify_enabled"}), + "deployment_readonly": frozenset(), + "resources": frozenset({name for name in Settings.model_fields if name.startswith("execution_")}), +} + + +def save_section(service, section, form): + fields = SECTIONS.get(section) + if not fields: + raise ValueError("设置分类不可修改") + extras = {f"clear_{name}" for name in fields & SECRET_FIELDS} | {"confirm_clear", "confirm_resources"} + if set(form) - fields - extras: + raise ValueError("分类包含不允许修改的字段") + clear = {name for name in fields & SECRET_FIELDS if form.get(f"clear_{name}") == "1"} + if clear and form.get("confirm_clear") != "1": + raise ValueError("清除凭据需要明确确认") + values = {k: str(v) for k, v in form.items() if k in fields and (k not in SECRET_FIELDS or str(v) != "")} + if section == "resources": + before = service.get() + proposed = Settings(_env_file=None, **(before.model_dump() | values)) + increase = any(getattr(proposed, k) > getattr(before, k) for k in fields + if k != "execution_platform_interval_seconds") + increase |= proposed.execution_platform_interval_seconds < before.execution_platform_interval_seconds + if increase and form.get("confirm_resources") != "1": + raise ValueError("提高资源或访问频率需确认代价") + for name in fields: + if name.endswith("_enabled"): + values[name] = "1" if str(form.get(name, "")).lower() in {"1", "true", "on", "yes"} else "0" + return service.set_many(values, clear_keys=clear) + + +@router.post("/settings/sections/{section}") +async def save_settings_section(request: Request, section: str): + from goodprice.web.routes import render_settings + + form = await request.form() + try: + save_section(request.app.state.settings_service, section, form) + except (ValueError, ValidationError): + public = {k: str(v) for k, v in form.items() if k in SECTIONS.get(section, ()) and k not in SECRET_FIELDS} + return render_settings(request, overrides=public, error="设置值无效,未保存;请检查范围和分类字段。", status_code=422) + return RedirectResponse(f"/settings#{section}", status_code=303) diff --git a/goodprice/web/static/THIRD_PARTY.md b/goodprice/web/static/THIRD_PARTY.md new file mode 100644 index 0000000..8fe680e --- /dev/null +++ b/goodprice/web/static/THIRD_PARTY.md @@ -0,0 +1,14 @@ +# Bundled browser assets + +- HTMX 1.9.12: https://github.com/bigskysoftware/htmx/tree/v1.9.12 + `htmx.min.js` SHA-256: `449317ade7881e949510db614991e195c3a099c4c791c24dacec55f9f4a2a452`. + License: `htmx.LICENSE` (Zero-Clause BSD). +- Tailwind CSS 3.4.17: https://github.com/tailwindlabs/tailwindcss/tree/v3.4.17 + License: `tailwind.LICENSE` (MIT). `app.css` is generated from the existing templates. + +Normal application startup needs neither Node.js nor a CDN connection. Maintainers +changing utility classes can regenerate the committed stylesheet with: + +```sh +npx --yes tailwindcss@3.4.17 -i ./goodprice/web/static/tailwind.input.css -o ./goodprice/web/static/app.css --content './goodprice/web/templates/**/*.html' --minify +``` diff --git a/goodprice/web/static/app.css b/goodprice/web/static/app.css new file mode 100644 index 0000000..bc141e7 --- /dev/null +++ b/goodprice/web/static/app.css @@ -0,0 +1 @@ +*,:after,:before{--tw-border-spacing-x:0;--tw-border-spacing-y:0;--tw-translate-x:0;--tw-translate-y:0;--tw-rotate:0;--tw-skew-x:0;--tw-skew-y:0;--tw-scale-x:1;--tw-scale-y:1;--tw-pan-x: ;--tw-pan-y: ;--tw-pinch-zoom: ;--tw-scroll-snap-strictness:proximity;--tw-gradient-from-position: ;--tw-gradient-via-position: ;--tw-gradient-to-position: ;--tw-ordinal: ;--tw-slashed-zero: ;--tw-numeric-figure: ;--tw-numeric-spacing: ;--tw-numeric-fraction: ;--tw-ring-inset: ;--tw-ring-offset-width:0px;--tw-ring-offset-color:#fff;--tw-ring-color:rgba(59,130,246,.5);--tw-ring-offset-shadow:0 0 #0000;--tw-ring-shadow:0 0 #0000;--tw-shadow:0 0 #0000;--tw-shadow-colored:0 0 #0000;--tw-blur: ;--tw-brightness: ;--tw-contrast: ;--tw-grayscale: ;--tw-hue-rotate: ;--tw-invert: ;--tw-saturate: ;--tw-sepia: ;--tw-drop-shadow: ;--tw-backdrop-blur: ;--tw-backdrop-brightness: ;--tw-backdrop-contrast: ;--tw-backdrop-grayscale: ;--tw-backdrop-hue-rotate: ;--tw-backdrop-invert: ;--tw-backdrop-opacity: ;--tw-backdrop-saturate: ;--tw-backdrop-sepia: ;--tw-contain-size: ;--tw-contain-layout: ;--tw-contain-paint: ;--tw-contain-style: }::backdrop{--tw-border-spacing-x:0;--tw-border-spacing-y:0;--tw-translate-x:0;--tw-translate-y:0;--tw-rotate:0;--tw-skew-x:0;--tw-skew-y:0;--tw-scale-x:1;--tw-scale-y:1;--tw-pan-x: ;--tw-pan-y: ;--tw-pinch-zoom: ;--tw-scroll-snap-strictness:proximity;--tw-gradient-from-position: ;--tw-gradient-via-position: ;--tw-gradient-to-position: ;--tw-ordinal: ;--tw-slashed-zero: ;--tw-numeric-figure: ;--tw-numeric-spacing: ;--tw-numeric-fraction: ;--tw-ring-inset: ;--tw-ring-offset-width:0px;--tw-ring-offset-color:#fff;--tw-ring-color:rgba(59,130,246,.5);--tw-ring-offset-shadow:0 0 #0000;--tw-ring-shadow:0 0 #0000;--tw-shadow:0 0 #0000;--tw-shadow-colored:0 0 #0000;--tw-blur: ;--tw-brightness: ;--tw-contrast: ;--tw-grayscale: ;--tw-hue-rotate: ;--tw-invert: ;--tw-saturate: ;--tw-sepia: ;--tw-drop-shadow: ;--tw-backdrop-blur: ;--tw-backdrop-brightness: ;--tw-backdrop-contrast: ;--tw-backdrop-grayscale: ;--tw-backdrop-hue-rotate: ;--tw-backdrop-invert: ;--tw-backdrop-opacity: ;--tw-backdrop-saturate: ;--tw-backdrop-sepia: ;--tw-contain-size: ;--tw-contain-layout: ;--tw-contain-paint: ;--tw-contain-style: }/*! tailwindcss v3.4.17 | MIT License | https://tailwindcss.com*/*,:after,:before{box-sizing:border-box;border:0 solid #e5e7eb}:after,:before{--tw-content:""}:host,html{line-height:1.5;-webkit-text-size-adjust:100%;-moz-tab-size:4;-o-tab-size:4;tab-size:4;font-family:ui-sans-serif,system-ui,sans-serif,Apple Color Emoji,Segoe UI Emoji,Segoe UI Symbol,Noto Color Emoji;font-feature-settings:normal;font-variation-settings:normal;-webkit-tap-highlight-color:transparent}body{margin:0;line-height:inherit}hr{height:0;color:inherit;border-top-width:1px}abbr:where([title]){-webkit-text-decoration:underline dotted;text-decoration:underline dotted}h1,h2,h3,h4,h5,h6{font-size:inherit;font-weight:inherit}a{color:inherit;text-decoration:inherit}b,strong{font-weight:bolder}code,kbd,pre,samp{font-family:ui-monospace,SFMono-Regular,Menlo,Monaco,Consolas,Liberation Mono,Courier New,monospace;font-feature-settings:normal;font-variation-settings:normal;font-size:1em}small{font-size:80%}sub,sup{font-size:75%;line-height:0;position:relative;vertical-align:baseline}sub{bottom:-.25em}sup{top:-.5em}table{text-indent:0;border-color:inherit;border-collapse:collapse}button,input,optgroup,select,textarea{font-family:inherit;font-feature-settings:inherit;font-variation-settings:inherit;font-size:100%;font-weight:inherit;line-height:inherit;letter-spacing:inherit;color:inherit;margin:0;padding:0}button,select{text-transform:none}button,input:where([type=button]),input:where([type=reset]),input:where([type=submit]){-webkit-appearance:button;background-color:transparent;background-image:none}:-moz-focusring{outline:auto}:-moz-ui-invalid{box-shadow:none}progress{vertical-align:baseline}::-webkit-inner-spin-button,::-webkit-outer-spin-button{height:auto}[type=search]{-webkit-appearance:textfield;outline-offset:-2px}::-webkit-search-decoration{-webkit-appearance:none}::-webkit-file-upload-button{-webkit-appearance:button;font:inherit}summary{display:list-item}blockquote,dd,dl,figure,h1,h2,h3,h4,h5,h6,hr,p,pre{margin:0}fieldset{margin:0}fieldset,legend{padding:0}menu,ol,ul{list-style:none;margin:0;padding:0}dialog{padding:0}textarea{resize:vertical}input::-moz-placeholder,textarea::-moz-placeholder{opacity:1;color:#9ca3af}input::placeholder,textarea::placeholder{opacity:1;color:#9ca3af}[role=button],button{cursor:pointer}:disabled{cursor:default}audio,canvas,embed,iframe,img,object,svg,video{display:block;vertical-align:middle}img,video{max-width:100%;height:auto}[hidden]:where(:not([hidden=until-found])){display:none}.fixed{position:fixed}.sticky{position:sticky}.inset-0{inset:0}.bottom-4{bottom:1rem}.right-4{right:1rem}.top-4{top:1rem}.z-10{z-index:10}.z-50{z-index:50}.col-span-full{grid-column:1/-1}.mx-auto{margin-left:auto;margin-right:auto}.mb-1{margin-bottom:.25rem}.mb-1\.5{margin-bottom:.375rem}.mb-2{margin-bottom:.5rem}.mb-3{margin-bottom:.75rem}.mb-4{margin-bottom:1rem}.mb-5{margin-bottom:1.25rem}.mb-6{margin-bottom:1.5rem}.mt-0\.5{margin-top:.125rem}.mt-1{margin-top:.25rem}.mt-1\.5{margin-top:.375rem}.mt-2{margin-top:.5rem}.mt-3{margin-top:.75rem}.mt-4{margin-top:1rem}.mt-5{margin-top:1.25rem}.mt-6{margin-top:1.5rem}.block{display:block}.inline{display:inline}.flex{display:flex}.inline-flex{display:inline-flex}.table{display:table}.grid{display:grid}.hidden{display:none}.aspect-\[3\/4\]{aspect-ratio:3/4}.aspect-square{aspect-ratio:1/1}.h-14{height:3.5rem}.h-full{height:100%}.max-h-full{max-height:100%}.w-14{width:3.5rem}.w-16{width:4rem}.w-20{width:5rem}.w-full{width:100%}.min-w-0{min-width:0}.max-w-4xl{max-width:56rem}.max-w-6xl{max-width:72rem}.max-w-full{max-width:100%}.flex-1{flex:1 1 0%}.basis-full{flex-basis:100%}.transform{transform:translate(var(--tw-translate-x),var(--tw-translate-y)) rotate(var(--tw-rotate)) skewX(var(--tw-skew-x)) skewY(var(--tw-skew-y)) scaleX(var(--tw-scale-x)) scaleY(var(--tw-scale-y))}.cursor-pointer{cursor:pointer}.cursor-zoom-in{cursor:zoom-in}.cursor-zoom-out{cursor:zoom-out}.select-none{-webkit-user-select:none;-moz-user-select:none;user-select:none}.select-all{-webkit-user-select:all;-moz-user-select:all;user-select:all}.resize-y{resize:vertical}.resize{resize:both}.grid-cols-1{grid-template-columns:repeat(1,minmax(0,1fr))}.grid-cols-2{grid-template-columns:repeat(2,minmax(0,1fr))}.flex-col{flex-direction:column}.flex-wrap{flex-wrap:wrap}.items-center{align-items:center}.justify-center{justify-content:center}.justify-between{justify-content:space-between}.gap-1{gap:.25rem}.gap-1\.5{gap:.375rem}.gap-2{gap:.5rem}.gap-3{gap:.75rem}.gap-4{gap:1rem}.gap-5{gap:1.25rem}.gap-6{gap:1.5rem}.space-y-2>:not([hidden])~:not([hidden]){--tw-space-y-reverse:0;margin-top:calc(.5rem*(1 - var(--tw-space-y-reverse)));margin-bottom:calc(.5rem*var(--tw-space-y-reverse))}.space-y-3>:not([hidden])~:not([hidden]){--tw-space-y-reverse:0;margin-top:calc(.75rem*(1 - var(--tw-space-y-reverse)));margin-bottom:calc(.75rem*var(--tw-space-y-reverse))}.space-y-4>:not([hidden])~:not([hidden]){--tw-space-y-reverse:0;margin-top:calc(1rem*(1 - var(--tw-space-y-reverse)));margin-bottom:calc(1rem*var(--tw-space-y-reverse))}.space-y-6>:not([hidden])~:not([hidden]){--tw-space-y-reverse:0;margin-top:calc(1.5rem*(1 - var(--tw-space-y-reverse)));margin-bottom:calc(1.5rem*var(--tw-space-y-reverse))}.divide-y>:not([hidden])~:not([hidden]){--tw-divide-y-reverse:0;border-top-width:calc(1px*(1 - var(--tw-divide-y-reverse)));border-bottom-width:calc(1px*var(--tw-divide-y-reverse))}.overflow-hidden{overflow:hidden}.overflow-x-auto{overflow-x:auto}.truncate{overflow:hidden;text-overflow:ellipsis;white-space:nowrap}.whitespace-pre-wrap{white-space:pre-wrap}.break-words{overflow-wrap:break-word}.break-all{word-break:break-all}.rounded{border-radius:.25rem}.rounded-full{border-radius:9999px}.rounded-lg{border-radius:.5rem}.rounded-md{border-radius:.375rem}.border{border-width:1px}.border-b{border-bottom-width:1px}.border-t{border-top-width:1px}.border-amber-200{--tw-border-opacity:1;border-color:rgb(253 230 138/var(--tw-border-opacity,1))}.border-blue-200{--tw-border-opacity:1;border-color:rgb(191 219 254/var(--tw-border-opacity,1))}.border-gray-100{--tw-border-opacity:1;border-color:rgb(243 244 246/var(--tw-border-opacity,1))}.border-gray-200{--tw-border-opacity:1;border-color:rgb(229 231 235/var(--tw-border-opacity,1))}.border-green-200{--tw-border-opacity:1;border-color:rgb(187 247 208/var(--tw-border-opacity,1))}.border-purple-100{--tw-border-opacity:1;border-color:rgb(243 232 255/var(--tw-border-opacity,1))}.border-red-300{--tw-border-opacity:1;border-color:rgb(252 165 165/var(--tw-border-opacity,1))}.border-yellow-200{--tw-border-opacity:1;border-color:rgb(254 240 138/var(--tw-border-opacity,1))}.bg-amber-100{--tw-bg-opacity:1;background-color:rgb(254 243 199/var(--tw-bg-opacity,1))}.bg-black\/80{background-color:rgba(0,0,0,.8)}.bg-blue-100{--tw-bg-opacity:1;background-color:rgb(219 234 254/var(--tw-bg-opacity,1))}.bg-blue-50{--tw-bg-opacity:1;background-color:rgb(239 246 255/var(--tw-bg-opacity,1))}.bg-gray-100{--tw-bg-opacity:1;background-color:rgb(243 244 246/var(--tw-bg-opacity,1))}.bg-gray-200{--tw-bg-opacity:1;background-color:rgb(229 231 235/var(--tw-bg-opacity,1))}.bg-gray-300{--tw-bg-opacity:1;background-color:rgb(209 213 219/var(--tw-bg-opacity,1))}.bg-gray-50{--tw-bg-opacity:1;background-color:rgb(249 250 251/var(--tw-bg-opacity,1))}.bg-gray-50\/80{background-color:rgba(249,250,251,.8)}.bg-green-100{--tw-bg-opacity:1;background-color:rgb(220 252 231/var(--tw-bg-opacity,1))}.bg-green-50{--tw-bg-opacity:1;background-color:rgb(240 253 244/var(--tw-bg-opacity,1))}.bg-green-700{--tw-bg-opacity:1;background-color:rgb(21 128 61/var(--tw-bg-opacity,1))}.bg-indigo-100{--tw-bg-opacity:1;background-color:rgb(224 231 255/var(--tw-bg-opacity,1))}.bg-purple-100{--tw-bg-opacity:1;background-color:rgb(243 232 255/var(--tw-bg-opacity,1))}.bg-purple-50\/40{background-color:rgba(250,245,255,.4)}.bg-red-100{--tw-bg-opacity:1;background-color:rgb(254 226 226/var(--tw-bg-opacity,1))}.bg-red-50{--tw-bg-opacity:1;background-color:rgb(254 242 242/var(--tw-bg-opacity,1))}.bg-white{--tw-bg-opacity:1;background-color:rgb(255 255 255/var(--tw-bg-opacity,1))}.bg-white\/95{background-color:hsla(0,0%,100%,.95)}.bg-yellow-100{--tw-bg-opacity:1;background-color:rgb(254 249 195/var(--tw-bg-opacity,1))}.bg-yellow-50{--tw-bg-opacity:1;background-color:rgb(254 252 232/var(--tw-bg-opacity,1))}.object-contain{-o-object-fit:contain;object-fit:contain}.object-cover{-o-object-fit:cover;object-fit:cover}.p-2{padding:.5rem}.p-3{padding:.75rem}.p-3\.5{padding:.875rem}.p-4{padding:1rem}.p-5{padding:1.25rem}.px-1\.5{padding-left:.375rem;padding-right:.375rem}.px-2{padding-left:.5rem;padding-right:.5rem}.px-2\.5{padding-left:.625rem;padding-right:.625rem}.px-3{padding-left:.75rem;padding-right:.75rem}.px-4{padding-left:1rem;padding-right:1rem}.px-6{padding-left:1.5rem;padding-right:1.5rem}.py-0\.5{padding-top:.125rem;padding-bottom:.125rem}.py-1{padding-top:.25rem;padding-bottom:.25rem}.py-1\.5{padding-top:.375rem;padding-bottom:.375rem}.py-2{padding-top:.5rem;padding-bottom:.5rem}.py-2\.5{padding-top:.625rem;padding-bottom:.625rem}.py-3{padding-top:.75rem;padding-bottom:.75rem}.py-6{padding-top:1.5rem;padding-bottom:1.5rem}.pb-3{padding-bottom:.75rem}.pt-3{padding-top:.75rem}.pt-4{padding-top:1rem}.text-left{text-align:left}.text-center{text-align:center}.font-mono{font-family:ui-monospace,SFMono-Regular,Menlo,Monaco,Consolas,Liberation Mono,Courier New,monospace}.text-2xl{font-size:1.5rem;line-height:2rem}.text-base{font-size:1rem;line-height:1.5rem}.text-lg{font-size:1.125rem;line-height:1.75rem}.text-sm{font-size:.875rem;line-height:1.25rem}.text-xl{font-size:1.25rem;line-height:1.75rem}.text-xs{font-size:.75rem;line-height:1rem}.font-bold{font-weight:700}.font-medium{font-weight:500}.font-semibold{font-weight:600}.leading-snug{line-height:1.375}.text-amber-700{--tw-text-opacity:1;color:rgb(180 83 9/var(--tw-text-opacity,1))}.text-amber-800{--tw-text-opacity:1;color:rgb(146 64 14/var(--tw-text-opacity,1))}.text-blue-600{--tw-text-opacity:1;color:rgb(37 99 235/var(--tw-text-opacity,1))}.text-blue-700{--tw-text-opacity:1;color:rgb(29 78 216/var(--tw-text-opacity,1))}.text-blue-800{--tw-text-opacity:1;color:rgb(30 64 175/var(--tw-text-opacity,1))}.text-gray-400{--tw-text-opacity:1;color:rgb(156 163 175/var(--tw-text-opacity,1))}.text-gray-500{--tw-text-opacity:1;color:rgb(107 114 128/var(--tw-text-opacity,1))}.text-gray-600{--tw-text-opacity:1;color:rgb(75 85 99/var(--tw-text-opacity,1))}.text-gray-700{--tw-text-opacity:1;color:rgb(55 65 81/var(--tw-text-opacity,1))}.text-gray-800{--tw-text-opacity:1;color:rgb(31 41 55/var(--tw-text-opacity,1))}.text-gray-900{--tw-text-opacity:1;color:rgb(17 24 39/var(--tw-text-opacity,1))}.text-green-600{--tw-text-opacity:1;color:rgb(22 163 74/var(--tw-text-opacity,1))}.text-green-700{--tw-text-opacity:1;color:rgb(21 128 61/var(--tw-text-opacity,1))}.text-green-800{--tw-text-opacity:1;color:rgb(22 101 52/var(--tw-text-opacity,1))}.text-indigo-700{--tw-text-opacity:1;color:rgb(67 56 202/var(--tw-text-opacity,1))}.text-purple-600{--tw-text-opacity:1;color:rgb(147 51 234/var(--tw-text-opacity,1))}.text-purple-700{--tw-text-opacity:1;color:rgb(126 34 206/var(--tw-text-opacity,1))}.text-red-600{--tw-text-opacity:1;color:rgb(220 38 38/var(--tw-text-opacity,1))}.text-red-700{--tw-text-opacity:1;color:rgb(185 28 28/var(--tw-text-opacity,1))}.text-white{--tw-text-opacity:1;color:rgb(255 255 255/var(--tw-text-opacity,1))}.text-yellow-600{--tw-text-opacity:1;color:rgb(202 138 4/var(--tw-text-opacity,1))}.text-yellow-700{--tw-text-opacity:1;color:rgb(161 98 7/var(--tw-text-opacity,1))}.text-yellow-800{--tw-text-opacity:1;color:rgb(133 77 14/var(--tw-text-opacity,1))}.underline{text-decoration-line:underline}.accent-purple-600{accent-color:#9333ea}.opacity-50{opacity:.5}.shadow{--tw-shadow:0 1px 3px 0 rgba(0,0,0,.1),0 1px 2px -1px rgba(0,0,0,.1);--tw-shadow-colored:0 1px 3px 0 var(--tw-shadow-color),0 1px 2px -1px var(--tw-shadow-color)}.shadow,.shadow-md{box-shadow:var(--tw-ring-offset-shadow,0 0 #0000),var(--tw-ring-shadow,0 0 #0000),var(--tw-shadow)}.shadow-md{--tw-shadow:0 4px 6px -1px rgba(0,0,0,.1),0 2px 4px -2px rgba(0,0,0,.1);--tw-shadow-colored:0 4px 6px -1px var(--tw-shadow-color),0 2px 4px -2px var(--tw-shadow-color)}.shadow-sm{--tw-shadow:0 1px 2px 0 rgba(0,0,0,.05);--tw-shadow-colored:0 1px 2px 0 var(--tw-shadow-color);box-shadow:var(--tw-ring-offset-shadow,0 0 #0000),var(--tw-ring-shadow,0 0 #0000),var(--tw-shadow)}.backdrop-blur{--tw-backdrop-blur:blur(8px);-webkit-backdrop-filter:var(--tw-backdrop-blur) var(--tw-backdrop-brightness) var(--tw-backdrop-contrast) var(--tw-backdrop-grayscale) var(--tw-backdrop-hue-rotate) var(--tw-backdrop-invert) var(--tw-backdrop-opacity) var(--tw-backdrop-saturate) var(--tw-backdrop-sepia);backdrop-filter:var(--tw-backdrop-blur) var(--tw-backdrop-brightness) var(--tw-backdrop-contrast) var(--tw-backdrop-grayscale) var(--tw-backdrop-hue-rotate) var(--tw-backdrop-invert) var(--tw-backdrop-opacity) var(--tw-backdrop-saturate) var(--tw-backdrop-sepia)}.transition{transition-property:color,background-color,border-color,text-decoration-color,fill,stroke,opacity,box-shadow,transform,filter,-webkit-backdrop-filter;transition-property:color,background-color,border-color,text-decoration-color,fill,stroke,opacity,box-shadow,transform,filter,backdrop-filter;transition-property:color,background-color,border-color,text-decoration-color,fill,stroke,opacity,box-shadow,transform,filter,backdrop-filter,-webkit-backdrop-filter;transition-timing-function:cubic-bezier(.4,0,.2,1);transition-duration:.15s}.transition-all{transition-property:all;transition-timing-function:cubic-bezier(.4,0,.2,1);transition-duration:.15s}.transition-colors{transition-property:color,background-color,border-color,text-decoration-color,fill,stroke;transition-timing-function:cubic-bezier(.4,0,.2,1);transition-duration:.15s}.transition-transform{transition-property:transform;transition-timing-function:cubic-bezier(.4,0,.2,1);transition-duration:.15s}.hover\:bg-blue-100:hover{--tw-bg-opacity:1;background-color:rgb(219 234 254/var(--tw-bg-opacity,1))}.hover\:bg-gray-100:hover{--tw-bg-opacity:1;background-color:rgb(243 244 246/var(--tw-bg-opacity,1))}.hover\:bg-gray-100\/70:hover{background-color:rgba(243,244,246,.7)}.hover\:bg-gray-200:hover{--tw-bg-opacity:1;background-color:rgb(229 231 235/var(--tw-bg-opacity,1))}.hover\:text-blue-600:hover{--tw-text-opacity:1;color:rgb(37 99 235/var(--tw-text-opacity,1))}.hover\:opacity-90:hover{opacity:.9}.hover\:shadow-md:hover{--tw-shadow:0 4px 6px -1px rgba(0,0,0,.1),0 2px 4px -2px rgba(0,0,0,.1);--tw-shadow-colored:0 4px 6px -1px var(--tw-shadow-color),0 2px 4px -2px var(--tw-shadow-color);box-shadow:var(--tw-ring-offset-shadow,0 0 #0000),var(--tw-ring-shadow,0 0 #0000),var(--tw-shadow)}.focus\:ring-1:focus{--tw-ring-offset-shadow:var(--tw-ring-inset) 0 0 0 var(--tw-ring-offset-width) var(--tw-ring-offset-color);--tw-ring-shadow:var(--tw-ring-inset) 0 0 0 calc(1px + var(--tw-ring-offset-width)) var(--tw-ring-color);box-shadow:var(--tw-ring-offset-shadow),var(--tw-ring-shadow),var(--tw-shadow,0 0 #0000)}.focus\:ring-blue-500:focus{--tw-ring-opacity:1;--tw-ring-color:rgb(59 130 246/var(--tw-ring-opacity,1))}.group[open] .group-open\:rotate-180{--tw-rotate:180deg;transform:translate(var(--tw-translate-x),var(--tw-translate-y)) rotate(var(--tw-rotate)) skewX(var(--tw-skew-x)) skewY(var(--tw-skew-y)) scaleX(var(--tw-scale-x)) scaleY(var(--tw-scale-y))}@media (min-width:640px){.sm\:grid-cols-2{grid-template-columns:repeat(2,minmax(0,1fr))}}@media (min-width:768px){.md\:col-span-2{grid-column:span 2/span 2}.md\:col-span-3{grid-column:span 3/span 3}.md\:basis-auto{flex-basis:auto}.md\:grid-cols-2{grid-template-columns:repeat(2,minmax(0,1fr))}.md\:grid-cols-3{grid-template-columns:repeat(3,minmax(0,1fr))}.md\:grid-cols-4{grid-template-columns:repeat(4,minmax(0,1fr))}}@media (min-width:1024px){.lg\:col-span-2{grid-column:span 2/span 2}.lg\:grid-cols-3{grid-template-columns:repeat(3,minmax(0,1fr))}.lg\:grid-cols-4{grid-template-columns:repeat(4,minmax(0,1fr))}}@media (min-width:1280px){.xl\:grid-cols-3{grid-template-columns:repeat(3,minmax(0,1fr))}} \ No newline at end of file diff --git a/goodprice/web/static/htmx.LICENSE b/goodprice/web/static/htmx.LICENSE new file mode 100644 index 0000000..d3061a3 --- /dev/null +++ b/goodprice/web/static/htmx.LICENSE @@ -0,0 +1,13 @@ +Zero-Clause BSD +============= + +Permission to use, copy, modify, and/or distribute this software for +any purpose with or without fee is hereby granted. + +THE SOFTWARE IS PROVIDED “AS IS” AND THE AUTHOR DISCLAIMS ALL +WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES +OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE +FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY +DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN +AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT +OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/goodprice/web/static/htmx.min.js b/goodprice/web/static/htmx.min.js new file mode 100644 index 0000000..de5f0f1 --- /dev/null +++ b/goodprice/web/static/htmx.min.js @@ -0,0 +1 @@ +(function(e,t){if(typeof define==="function"&&define.amd){define([],t)}else if(typeof module==="object"&&module.exports){module.exports=t()}else{e.htmx=e.htmx||t()}})(typeof self!=="undefined"?self:this,function(){return function(){"use strict";var Q={onLoad:F,process:zt,on:de,off:ge,trigger:ce,ajax:Nr,find:C,findAll:f,closest:v,values:function(e,t){var r=dr(e,t||"post");return r.values},remove:_,addClass:z,removeClass:n,toggleClass:$,takeClass:W,defineExtension:Ur,removeExtension:Br,logAll:V,logNone:j,logger:null,config:{historyEnabled:true,historyCacheSize:10,refreshOnHistoryMiss:false,defaultSwapStyle:"innerHTML",defaultSwapDelay:0,defaultSettleDelay:20,includeIndicatorStyles:true,indicatorClass:"htmx-indicator",requestClass:"htmx-request",addedClass:"htmx-added",settlingClass:"htmx-settling",swappingClass:"htmx-swapping",allowEval:true,allowScriptTags:true,inlineScriptNonce:"",attributesToSettle:["class","style","width","height"],withCredentials:false,timeout:0,wsReconnectDelay:"full-jitter",wsBinaryType:"blob",disableSelector:"[hx-disable], [data-hx-disable]",useTemplateFragments:false,scrollBehavior:"smooth",defaultFocusScroll:false,getCacheBusterParam:false,globalViewTransitions:false,methodsThatUseUrlParams:["get"],selfRequestsOnly:false,ignoreTitle:false,scrollIntoViewOnBoost:true,triggerSpecsCache:null},parseInterval:d,_:t,createEventSource:function(e){return new EventSource(e,{withCredentials:true})},createWebSocket:function(e){var t=new WebSocket(e,[]);t.binaryType=Q.config.wsBinaryType;return t},version:"1.9.12"};var r={addTriggerHandler:Lt,bodyContains:se,canAccessLocalStorage:U,findThisElement:xe,filterValues:yr,hasAttribute:o,getAttributeValue:te,getClosestAttributeValue:ne,getClosestMatch:c,getExpressionVars:Hr,getHeaders:xr,getInputValues:dr,getInternalData:ae,getSwapSpecification:wr,getTriggerSpecs:it,getTarget:ye,makeFragment:l,mergeObjects:le,makeSettleInfo:T,oobSwap:Ee,querySelectorExt:ue,selectAndSwap:je,settleImmediately:nr,shouldCancel:ut,triggerEvent:ce,triggerErrorEvent:fe,withExtensions:R};var w=["get","post","put","delete","patch"];var i=w.map(function(e){return"[hx-"+e+"], [data-hx-"+e+"]"}).join(", ");var S=e("head"),q=e("title"),H=e("svg",true);function e(e,t){return new RegExp("<"+e+"(\\s[^>]*>|>)([\\s\\S]*?)<\\/"+e+">",!!t?"gim":"im")}function d(e){if(e==undefined){return undefined}let t=NaN;if(e.slice(-2)=="ms"){t=parseFloat(e.slice(0,-2))}else if(e.slice(-1)=="s"){t=parseFloat(e.slice(0,-1))*1e3}else if(e.slice(-1)=="m"){t=parseFloat(e.slice(0,-1))*1e3*60}else{t=parseFloat(e)}return isNaN(t)?undefined:t}function ee(e,t){return e.getAttribute&&e.getAttribute(t)}function o(e,t){return e.hasAttribute&&(e.hasAttribute(t)||e.hasAttribute("data-"+t))}function te(e,t){return ee(e,t)||ee(e,"data-"+t)}function u(e){return e.parentElement}function re(){return document}function c(e,t){while(e&&!t(e)){e=u(e)}return e?e:null}function L(e,t,r){var n=te(t,r);var i=te(t,"hx-disinherit");if(e!==t&&i&&(i==="*"||i.split(" ").indexOf(r)>=0)){return"unset"}else{return n}}function ne(t,r){var n=null;c(t,function(e){return n=L(t,e,r)});if(n!=="unset"){return n}}function h(e,t){var r=e.matches||e.matchesSelector||e.msMatchesSelector||e.mozMatchesSelector||e.webkitMatchesSelector||e.oMatchesSelector;return r&&r.call(e,t)}function A(e){var t=/<([a-z][^\/\0>\x20\t\r\n\f]*)/i;var r=t.exec(e);if(r){return r[1].toLowerCase()}else{return""}}function s(e,t){var r=new DOMParser;var n=r.parseFromString(e,"text/html");var i=n.body;while(t>0){t--;i=i.firstChild}if(i==null){i=re().createDocumentFragment()}return i}function N(e){return/",0);var a=i.querySelector("template").content;if(Q.config.allowScriptTags){oe(a.querySelectorAll("script"),function(e){if(Q.config.inlineScriptNonce){e.nonce=Q.config.inlineScriptNonce}e.htmxExecuted=navigator.userAgent.indexOf("Firefox")===-1})}else{oe(a.querySelectorAll("script"),function(e){_(e)})}return a}switch(r){case"thead":case"tbody":case"tfoot":case"colgroup":case"caption":return s(""+n+"
",1);case"col":return s(""+n+"
",2);case"tr":return s(""+n+"
",2);case"td":case"th":return s(""+n+"
",3);case"script":case"style":return s("
"+n+"
",1);default:return s(n,0)}}function ie(e){if(e){e()}}function I(e,t){return Object.prototype.toString.call(e)==="[object "+t+"]"}function k(e){return I(e,"Function")}function P(e){return I(e,"Object")}function ae(e){var t="htmx-internal-data";var r=e[t];if(!r){r=e[t]={}}return r}function M(e){var t=[];if(e){for(var r=0;r=0}function se(e){if(e.getRootNode&&e.getRootNode()instanceof window.ShadowRoot){return re().body.contains(e.getRootNode().host)}else{return re().body.contains(e)}}function D(e){return e.trim().split(/\s+/)}function le(e,t){for(var r in t){if(t.hasOwnProperty(r)){e[r]=t[r]}}return e}function E(e){try{return JSON.parse(e)}catch(e){b(e);return null}}function U(){var e="htmx:localStorageTest";try{localStorage.setItem(e,e);localStorage.removeItem(e);return true}catch(e){return false}}function B(t){try{var e=new URL(t);if(e){t=e.pathname+e.search}if(!/^\/$/.test(t)){t=t.replace(/\/+$/,"")}return t}catch(e){return t}}function t(e){return Tr(re().body,function(){return eval(e)})}function F(t){var e=Q.on("htmx:load",function(e){t(e.detail.elt)});return e}function V(){Q.logger=function(e,t,r){if(console){console.log(t,e,r)}}}function j(){Q.logger=null}function C(e,t){if(t){return e.querySelector(t)}else{return C(re(),e)}}function f(e,t){if(t){return e.querySelectorAll(t)}else{return f(re(),e)}}function _(e,t){e=p(e);if(t){setTimeout(function(){_(e);e=null},t)}else{e.parentElement.removeChild(e)}}function z(e,t,r){e=p(e);if(r){setTimeout(function(){z(e,t);e=null},r)}else{e.classList&&e.classList.add(t)}}function n(e,t,r){e=p(e);if(r){setTimeout(function(){n(e,t);e=null},r)}else{if(e.classList){e.classList.remove(t);if(e.classList.length===0){e.removeAttribute("class")}}}}function $(e,t){e=p(e);e.classList.toggle(t)}function W(e,t){e=p(e);oe(e.parentElement.children,function(e){n(e,t)});z(e,t)}function v(e,t){e=p(e);if(e.closest){return e.closest(t)}else{do{if(e==null||h(e,t)){return e}}while(e=e&&u(e));return null}}function g(e,t){return e.substring(0,t.length)===t}function G(e,t){return e.substring(e.length-t.length)===t}function J(e){var t=e.trim();if(g(t,"<")&&G(t,"/>")){return t.substring(1,t.length-2)}else{return t}}function Z(e,t){if(t.indexOf("closest ")===0){return[v(e,J(t.substr(8)))]}else if(t.indexOf("find ")===0){return[C(e,J(t.substr(5)))]}else if(t==="next"){return[e.nextElementSibling]}else if(t.indexOf("next ")===0){return[K(e,J(t.substr(5)))]}else if(t==="previous"){return[e.previousElementSibling]}else if(t.indexOf("previous ")===0){return[Y(e,J(t.substr(9)))]}else if(t==="document"){return[document]}else if(t==="window"){return[window]}else if(t==="body"){return[document.body]}else{return re().querySelectorAll(J(t))}}var K=function(e,t){var r=re().querySelectorAll(t);for(var n=0;n=0;n--){var i=r[n];if(i.compareDocumentPosition(e)===Node.DOCUMENT_POSITION_FOLLOWING){return i}}};function ue(e,t){if(t){return Z(e,t)[0]}else{return Z(re().body,e)[0]}}function p(e){if(I(e,"String")){return C(e)}else{return e}}function ve(e,t,r){if(k(t)){return{target:re().body,event:e,listener:t}}else{return{target:p(e),event:t,listener:r}}}function de(t,r,n){jr(function(){var e=ve(t,r,n);e.target.addEventListener(e.event,e.listener)});var e=k(r);return e?r:n}function ge(t,r,n){jr(function(){var e=ve(t,r,n);e.target.removeEventListener(e.event,e.listener)});return k(r)?r:n}var pe=re().createElement("output");function me(e,t){var r=ne(e,t);if(r){if(r==="this"){return[xe(e,t)]}else{var n=Z(e,r);if(n.length===0){b('The selector "'+r+'" on '+t+" returned no matches!");return[pe]}else{return n}}}}function xe(e,t){return c(e,function(e){return te(e,t)!=null})}function ye(e){var t=ne(e,"hx-target");if(t){if(t==="this"){return xe(e,"hx-target")}else{return ue(e,t)}}else{var r=ae(e);if(r.boosted){return re().body}else{return e}}}function be(e){var t=Q.config.attributesToSettle;for(var r=0;r0){o=e.substr(0,e.indexOf(":"));t=e.substr(e.indexOf(":")+1,e.length)}else{o=e}var r=re().querySelectorAll(t);if(r){oe(r,function(e){var t;var r=i.cloneNode(true);t=re().createDocumentFragment();t.appendChild(r);if(!Se(o,e)){t=r}var n={shouldSwap:true,target:e,fragment:t};if(!ce(e,"htmx:oobBeforeSwap",n))return;e=n.target;if(n["shouldSwap"]){Fe(o,e,e,t,a)}oe(a.elts,function(e){ce(e,"htmx:oobAfterSwap",n)})});i.parentNode.removeChild(i)}else{i.parentNode.removeChild(i);fe(re().body,"htmx:oobErrorNoTarget",{content:i})}return e}function Ce(e,t,r){var n=ne(e,"hx-select-oob");if(n){var i=n.split(",");for(var a=0;a0){var r=t.replace("'","\\'");var n=e.tagName.replace(":","\\:");var i=o.querySelector(n+"[id='"+r+"']");if(i&&i!==o){var a=e.cloneNode();we(e,i);s.tasks.push(function(){we(e,a)})}}})}function Oe(e){return function(){n(e,Q.config.addedClass);zt(e);Nt(e);qe(e);ce(e,"htmx:load")}}function qe(e){var t="[autofocus]";var r=h(e,t)?e:e.querySelector(t);if(r!=null){r.focus()}}function a(e,t,r,n){Te(e,r,n);while(r.childNodes.length>0){var i=r.firstChild;z(i,Q.config.addedClass);e.insertBefore(i,t);if(i.nodeType!==Node.TEXT_NODE&&i.nodeType!==Node.COMMENT_NODE){n.tasks.push(Oe(i))}}}function He(e,t){var r=0;while(r-1){var t=e.replace(H,"");var r=t.match(q);if(r){return r[2]}}}function je(e,t,r,n,i,a){i.title=Ve(n);var o=l(n);if(o){Ce(r,o,i);o=Be(r,o,a);Re(o);return Fe(e,r,t,o,i)}}function _e(e,t,r){var n=e.getResponseHeader(t);if(n.indexOf("{")===0){var i=E(n);for(var a in i){if(i.hasOwnProperty(a)){var o=i[a];if(!P(o)){o={value:o}}ce(r,a,o)}}}else{var s=n.split(",");for(var l=0;l0){var o=t[0];if(o==="]"){n--;if(n===0){if(a===null){i=i+"true"}t.shift();i+=")})";try{var s=Tr(e,function(){return Function(i)()},function(){return true});s.source=i;return s}catch(e){fe(re().body,"htmx:syntax:error",{error:e,source:i});return null}}}else if(o==="["){n++}if(Qe(o,a,r)){i+="(("+r+"."+o+") ? ("+r+"."+o+") : (window."+o+"))"}else{i=i+o}a=t.shift()}}}function y(e,t){var r="";while(e.length>0&&!t.test(e[0])){r+=e.shift()}return r}function tt(e){var t;if(e.length>0&&Ze.test(e[0])){e.shift();t=y(e,Ke).trim();e.shift()}else{t=y(e,x)}return t}var rt="input, textarea, select";function nt(e,t,r){var n=[];var i=Ye(t);do{y(i,Je);var a=i.length;var o=y(i,/[,\[\s]/);if(o!==""){if(o==="every"){var s={trigger:"every"};y(i,Je);s.pollInterval=d(y(i,/[,\[\s]/));y(i,Je);var l=et(e,i,"event");if(l){s.eventFilter=l}n.push(s)}else if(o.indexOf("sse:")===0){n.push({trigger:"sse",sseEvent:o.substr(4)})}else{var u={trigger:o};var l=et(e,i,"event");if(l){u.eventFilter=l}while(i.length>0&&i[0]!==","){y(i,Je);var f=i.shift();if(f==="changed"){u.changed=true}else if(f==="once"){u.once=true}else if(f==="consume"){u.consume=true}else if(f==="delay"&&i[0]===":"){i.shift();u.delay=d(y(i,x))}else if(f==="from"&&i[0]===":"){i.shift();if(Ze.test(i[0])){var c=tt(i)}else{var c=y(i,x);if(c==="closest"||c==="find"||c==="next"||c==="previous"){i.shift();var h=tt(i);if(h.length>0){c+=" "+h}}}u.from=c}else if(f==="target"&&i[0]===":"){i.shift();u.target=tt(i)}else if(f==="throttle"&&i[0]===":"){i.shift();u.throttle=d(y(i,x))}else if(f==="queue"&&i[0]===":"){i.shift();u.queue=y(i,x)}else if(f==="root"&&i[0]===":"){i.shift();u[f]=tt(i)}else if(f==="threshold"&&i[0]===":"){i.shift();u[f]=y(i,x)}else{fe(e,"htmx:syntax:error",{token:i.shift()})}}n.push(u)}}if(i.length===a){fe(e,"htmx:syntax:error",{token:i.shift()})}y(i,Je)}while(i[0]===","&&i.shift());if(r){r[t]=n}return n}function it(e){var t=te(e,"hx-trigger");var r=[];if(t){var n=Q.config.triggerSpecsCache;r=n&&n[t]||nt(e,t,n)}if(r.length>0){return r}else if(h(e,"form")){return[{trigger:"submit"}]}else if(h(e,'input[type="button"], input[type="submit"]')){return[{trigger:"click"}]}else if(h(e,rt)){return[{trigger:"change"}]}else{return[{trigger:"click"}]}}function at(e){ae(e).cancelled=true}function ot(e,t,r){var n=ae(e);n.timeout=setTimeout(function(){if(se(e)&&n.cancelled!==true){if(!ct(r,e,Wt("hx:poll:trigger",{triggerSpec:r,target:e}))){t(e)}ot(e,t,r)}},r.pollInterval)}function st(e){return location.hostname===e.hostname&&ee(e,"href")&&ee(e,"href").indexOf("#")!==0}function lt(t,r,e){if(t.tagName==="A"&&st(t)&&(t.target===""||t.target==="_self")||t.tagName==="FORM"){r.boosted=true;var n,i;if(t.tagName==="A"){n="get";i=ee(t,"href")}else{var a=ee(t,"method");n=a?a.toLowerCase():"get";if(n==="get"){}i=ee(t,"action")}e.forEach(function(e){ht(t,function(e,t){if(v(e,Q.config.disableSelector)){m(e);return}he(n,i,e,t)},r,e,true)})}}function ut(e,t){if(e.type==="submit"||e.type==="click"){if(t.tagName==="FORM"){return true}if(h(t,'input[type="submit"], button')&&v(t,"form")!==null){return true}if(t.tagName==="A"&&t.href&&(t.getAttribute("href")==="#"||t.getAttribute("href").indexOf("#")!==0)){return true}}return false}function ft(e,t){return ae(e).boosted&&e.tagName==="A"&&t.type==="click"&&(t.ctrlKey||t.metaKey)}function ct(e,t,r){var n=e.eventFilter;if(n){try{return n.call(t,r)!==true}catch(e){fe(re().body,"htmx:eventFilter:error",{error:e,source:n.source});return true}}return false}function ht(a,o,e,s,l){var u=ae(a);var t;if(s.from){t=Z(a,s.from)}else{t=[a]}if(s.changed){t.forEach(function(e){var t=ae(e);t.lastValue=e.value})}oe(t,function(n){var i=function(e){if(!se(a)){n.removeEventListener(s.trigger,i);return}if(ft(a,e)){return}if(l||ut(e,a)){e.preventDefault()}if(ct(s,a,e)){return}var t=ae(e);t.triggerSpec=s;if(t.handledFor==null){t.handledFor=[]}if(t.handledFor.indexOf(a)<0){t.handledFor.push(a);if(s.consume){e.stopPropagation()}if(s.target&&e.target){if(!h(e.target,s.target)){return}}if(s.once){if(u.triggeredOnce){return}else{u.triggeredOnce=true}}if(s.changed){var r=ae(n);if(r.lastValue===n.value){return}r.lastValue=n.value}if(u.delayed){clearTimeout(u.delayed)}if(u.throttle){return}if(s.throttle>0){if(!u.throttle){o(a,e);u.throttle=setTimeout(function(){u.throttle=null},s.throttle)}}else if(s.delay>0){u.delayed=setTimeout(function(){o(a,e)},s.delay)}else{ce(a,"htmx:trigger");o(a,e)}}};if(e.listenerInfos==null){e.listenerInfos=[]}e.listenerInfos.push({trigger:s.trigger,listener:i,on:n});n.addEventListener(s.trigger,i)})}var vt=false;var dt=null;function gt(){if(!dt){dt=function(){vt=true};window.addEventListener("scroll",dt);setInterval(function(){if(vt){vt=false;oe(re().querySelectorAll("[hx-trigger='revealed'],[data-hx-trigger='revealed']"),function(e){pt(e)})}},200)}}function pt(t){if(!o(t,"data-hx-revealed")&&X(t)){t.setAttribute("data-hx-revealed","true");var e=ae(t);if(e.initHash){ce(t,"revealed")}else{t.addEventListener("htmx:afterProcessNode",function(e){ce(t,"revealed")},{once:true})}}}function mt(e,t,r){var n=D(r);for(var i=0;i=0){var t=wt(n);setTimeout(function(){xt(s,r,n+1)},t)}};t.onopen=function(e){n=0};ae(s).webSocket=t;t.addEventListener("message",function(e){if(yt(s)){return}var t=e.data;R(s,function(e){t=e.transformResponse(t,null,s)});var r=T(s);var n=l(t);var i=M(n.children);for(var a=0;a0){ce(u,"htmx:validation:halted",i);return}t.send(JSON.stringify(l));if(ut(e,u)){e.preventDefault()}})}else{fe(u,"htmx:noWebSocketSourceError")}}function wt(e){var t=Q.config.wsReconnectDelay;if(typeof t==="function"){return t(e)}if(t==="full-jitter"){var r=Math.min(e,6);var n=1e3*Math.pow(2,r);return n*Math.random()}b('htmx.config.wsReconnectDelay must either be a function or the string "full-jitter"')}function St(e,t,r){var n=D(r);for(var i=0;i0){setTimeout(i,n)}else{i()}}function Ht(t,i,e){var a=false;oe(w,function(r){if(o(t,"hx-"+r)){var n=te(t,"hx-"+r);a=true;i.path=n;i.verb=r;e.forEach(function(e){Lt(t,e,i,function(e,t){if(v(e,Q.config.disableSelector)){m(e);return}he(r,n,e,t)})})}});return a}function Lt(n,e,t,r){if(e.sseEvent){Rt(n,r,e.sseEvent)}else if(e.trigger==="revealed"){gt();ht(n,r,t,e);pt(n)}else if(e.trigger==="intersect"){var i={};if(e.root){i.root=ue(n,e.root)}if(e.threshold){i.threshold=parseFloat(e.threshold)}var a=new IntersectionObserver(function(e){for(var t=0;t0){t.polling=true;ot(n,r,e)}else{ht(n,r,t,e)}}function At(e){if(!e.htmxExecuted&&Q.config.allowScriptTags&&(e.type==="text/javascript"||e.type==="module"||e.type==="")){var t=re().createElement("script");oe(e.attributes,function(e){t.setAttribute(e.name,e.value)});t.textContent=e.textContent;t.async=false;if(Q.config.inlineScriptNonce){t.nonce=Q.config.inlineScriptNonce}var r=e.parentElement;try{r.insertBefore(t,e)}catch(e){b(e)}finally{if(e.parentElement){e.parentElement.removeChild(e)}}}}function Nt(e){if(h(e,"script")){At(e)}oe(f(e,"script"),function(e){At(e)})}function It(e){var t=e.attributes;if(!t){return false}for(var r=0;r0){var o=n.shift();var s=o.match(/^\s*([a-zA-Z:\-\.]+:)(.*)/);if(a===0&&s){o.split(":");i=s[1].slice(0,-1);r[i]=s[2]}else{r[i]+=o}a+=Bt(o)}for(var l in r){Ft(e,l,r[l])}}}function jt(e){Ae(e);for(var t=0;tQ.config.historyCacheSize){i.shift()}while(i.length>0){try{localStorage.setItem("htmx-history-cache",JSON.stringify(i));break}catch(e){fe(re().body,"htmx:historyCacheError",{cause:e,cache:i});i.shift()}}}function Yt(e){if(!U()){return null}e=B(e);var t=E(localStorage.getItem("htmx-history-cache"))||[];for(var r=0;r=200&&this.status<400){ce(re().body,"htmx:historyCacheMissLoad",o);var e=l(this.response);e=e.querySelector("[hx-history-elt],[data-hx-history-elt]")||e;var t=Zt();var r=T(t);var n=Ve(this.response);if(n){var i=C("title");if(i){i.innerHTML=n}else{window.document.title=n}}Ue(t,e,r);nr(r.tasks);Jt=a;ce(re().body,"htmx:historyRestore",{path:a,cacheMiss:true,serverResponse:this.response})}else{fe(re().body,"htmx:historyCacheMissLoadError",o)}};e.send()}function ar(e){er();e=e||location.pathname+location.search;var t=Yt(e);if(t){var r=l(t.content);var n=Zt();var i=T(n);Ue(n,r,i);nr(i.tasks);document.title=t.title;setTimeout(function(){window.scrollTo(0,t.scroll)},0);Jt=e;ce(re().body,"htmx:historyRestore",{path:e,item:t})}else{if(Q.config.refreshOnHistoryMiss){window.location.reload(true)}else{ir(e)}}}function or(e){var t=me(e,"hx-indicator");if(t==null){t=[e]}oe(t,function(e){var t=ae(e);t.requestCount=(t.requestCount||0)+1;e.classList["add"].call(e.classList,Q.config.requestClass)});return t}function sr(e){var t=me(e,"hx-disabled-elt");if(t==null){t=[]}oe(t,function(e){var t=ae(e);t.requestCount=(t.requestCount||0)+1;e.setAttribute("disabled","")});return t}function lr(e,t){oe(e,function(e){var t=ae(e);t.requestCount=(t.requestCount||0)-1;if(t.requestCount===0){e.classList["remove"].call(e.classList,Q.config.requestClass)}});oe(t,function(e){var t=ae(e);t.requestCount=(t.requestCount||0)-1;if(t.requestCount===0){e.removeAttribute("disabled")}})}function ur(e,t){for(var r=0;r=0}function wr(e,t){var r=t?t:ne(e,"hx-swap");var n={swapStyle:ae(e).boosted?"innerHTML":Q.config.defaultSwapStyle,swapDelay:Q.config.defaultSwapDelay,settleDelay:Q.config.defaultSettleDelay};if(Q.config.scrollIntoViewOnBoost&&ae(e).boosted&&!br(e)){n["show"]="top"}if(r){var i=D(r);if(i.length>0){for(var a=0;a0?l.join(":"):null;n["scroll"]=u;n["scrollTarget"]=f}else if(o.indexOf("show:")===0){var c=o.substr(5);var l=c.split(":");var h=l.pop();var f=l.length>0?l.join(":"):null;n["show"]=h;n["showTarget"]=f}else if(o.indexOf("focus-scroll:")===0){var v=o.substr("focus-scroll:".length);n["focusScroll"]=v=="true"}else if(a==0){n["swapStyle"]=o}else{b("Unknown modifier in hx-swap: "+o)}}}}return n}function Sr(e){return ne(e,"hx-encoding")==="multipart/form-data"||h(e,"form")&&ee(e,"enctype")==="multipart/form-data"}function Er(t,r,n){var i=null;R(r,function(e){if(i==null){i=e.encodeParameters(t,n,r)}});if(i!=null){return i}else{if(Sr(r)){return mr(n)}else{return pr(n)}}}function T(e){return{tasks:[],elts:[e]}}function Cr(e,t){var r=e[0];var n=e[e.length-1];if(t.scroll){var i=null;if(t.scrollTarget){i=ue(r,t.scrollTarget)}if(t.scroll==="top"&&(r||i)){i=i||r;i.scrollTop=0}if(t.scroll==="bottom"&&(n||i)){i=i||n;i.scrollTop=i.scrollHeight}}if(t.show){var i=null;if(t.showTarget){var a=t.showTarget;if(t.showTarget==="window"){a="body"}i=ue(r,a)}if(t.show==="top"&&(r||i)){i=i||r;i.scrollIntoView({block:"start",behavior:Q.config.scrollBehavior})}if(t.show==="bottom"&&(n||i)){i=i||n;i.scrollIntoView({block:"end",behavior:Q.config.scrollBehavior})}}}function Rr(e,t,r,n){if(n==null){n={}}if(e==null){return n}var i=te(e,t);if(i){var a=i.trim();var o=r;if(a==="unset"){return null}if(a.indexOf("javascript:")===0){a=a.substr(11);o=true}else if(a.indexOf("js:")===0){a=a.substr(3);o=true}if(a.indexOf("{")!==0){a="{"+a+"}"}var s;if(o){s=Tr(e,function(){return Function("return ("+a+")")()},{})}else{s=E(a)}for(var l in s){if(s.hasOwnProperty(l)){if(n[l]==null){n[l]=s[l]}}}}return Rr(u(e),t,r,n)}function Tr(e,t,r){if(Q.config.allowEval){return t()}else{fe(e,"htmx:evalDisallowedError");return r}}function Or(e,t){return Rr(e,"hx-vars",true,t)}function qr(e,t){return Rr(e,"hx-vals",false,t)}function Hr(e){return le(Or(e),qr(e))}function Lr(t,r,n){if(n!==null){try{t.setRequestHeader(r,n)}catch(e){t.setRequestHeader(r,encodeURIComponent(n));t.setRequestHeader(r+"-URI-AutoEncoded","true")}}}function Ar(t){if(t.responseURL&&typeof URL!=="undefined"){try{var e=new URL(t.responseURL);return e.pathname+e.search}catch(e){fe(re().body,"htmx:badResponseUrl",{url:t.responseURL})}}}function O(e,t){return t.test(e.getAllResponseHeaders())}function Nr(e,t,r){e=e.toLowerCase();if(r){if(r instanceof Element||I(r,"String")){return he(e,t,null,null,{targetOverride:p(r),returnPromise:true})}else{return he(e,t,p(r.source),r.event,{handler:r.handler,headers:r.headers,values:r.values,targetOverride:p(r.target),swapOverride:r.swap,select:r.select,returnPromise:true})}}else{return he(e,t,null,null,{returnPromise:true})}}function Ir(e){var t=[];while(e){t.push(e);e=e.parentElement}return t}function kr(e,t,r){var n;var i;if(typeof URL==="function"){i=new URL(t,document.location.href);var a=document.location.origin;n=a===i.origin}else{i=t;n=g(t,document.location.origin)}if(Q.config.selfRequestsOnly){if(!n){return false}}return ce(e,"htmx:validateUrl",le({url:i,sameHost:n},r))}function he(t,r,n,i,a,e){var o=null;var s=null;a=a!=null?a:{};if(a.returnPromise&&typeof Promise!=="undefined"){var l=new Promise(function(e,t){o=e;s=t})}if(n==null){n=re().body}var M=a.handler||Mr;var X=a.select||null;if(!se(n)){ie(o);return l}var u=a.targetOverride||ye(n);if(u==null||u==pe){fe(n,"htmx:targetError",{target:te(n,"hx-target")});ie(s);return l}var f=ae(n);var c=f.lastButtonClicked;if(c){var h=ee(c,"formaction");if(h!=null){r=h}var v=ee(c,"formmethod");if(v!=null){if(v.toLowerCase()!=="dialog"){t=v}}}var d=ne(n,"hx-confirm");if(e===undefined){var D=function(e){return he(t,r,n,i,a,!!e)};var U={target:u,elt:n,path:r,verb:t,triggeringEvent:i,etc:a,issueRequest:D,question:d};if(ce(n,"htmx:confirm",U)===false){ie(o);return l}}var g=n;var p=ne(n,"hx-sync");var m=null;var x=false;if(p){var B=p.split(":");var F=B[0].trim();if(F==="this"){g=xe(n,"hx-sync")}else{g=ue(n,F)}p=(B[1]||"drop").trim();f=ae(g);if(p==="drop"&&f.xhr&&f.abortable!==true){ie(o);return l}else if(p==="abort"){if(f.xhr){ie(o);return l}else{x=true}}else if(p==="replace"){ce(g,"htmx:abort")}else if(p.indexOf("queue")===0){var V=p.split(" ");m=(V[1]||"last").trim()}}if(f.xhr){if(f.abortable){ce(g,"htmx:abort")}else{if(m==null){if(i){var y=ae(i);if(y&&y.triggerSpec&&y.triggerSpec.queue){m=y.triggerSpec.queue}}if(m==null){m="last"}}if(f.queuedRequests==null){f.queuedRequests=[]}if(m==="first"&&f.queuedRequests.length===0){f.queuedRequests.push(function(){he(t,r,n,i,a)})}else if(m==="all"){f.queuedRequests.push(function(){he(t,r,n,i,a)})}else if(m==="last"){f.queuedRequests=[];f.queuedRequests.push(function(){he(t,r,n,i,a)})}ie(o);return l}}var b=new XMLHttpRequest;f.xhr=b;f.abortable=x;var w=function(){f.xhr=null;f.abortable=false;if(f.queuedRequests!=null&&f.queuedRequests.length>0){var e=f.queuedRequests.shift();e()}};var j=ne(n,"hx-prompt");if(j){var S=prompt(j);if(S===null||!ce(n,"htmx:prompt",{prompt:S,target:u})){ie(o);w();return l}}if(d&&!e){if(!confirm(d)){ie(o);w();return l}}var E=xr(n,u,S);if(t!=="get"&&!Sr(n)){E["Content-Type"]="application/x-www-form-urlencoded"}if(a.headers){E=le(E,a.headers)}var _=dr(n,t);var C=_.errors;var R=_.values;if(a.values){R=le(R,a.values)}var z=Hr(n);var $=le(R,z);var T=yr($,n);if(Q.config.getCacheBusterParam&&t==="get"){T["org.htmx.cache-buster"]=ee(u,"id")||"true"}if(r==null||r===""){r=re().location.href}var O=Rr(n,"hx-request");var W=ae(n).boosted;var q=Q.config.methodsThatUseUrlParams.indexOf(t)>=0;var H={boosted:W,useUrlParams:q,parameters:T,unfilteredParameters:$,headers:E,target:u,verb:t,errors:C,withCredentials:a.credentials||O.credentials||Q.config.withCredentials,timeout:a.timeout||O.timeout||Q.config.timeout,path:r,triggeringEvent:i};if(!ce(n,"htmx:configRequest",H)){ie(o);w();return l}r=H.path;t=H.verb;E=H.headers;T=H.parameters;C=H.errors;q=H.useUrlParams;if(C&&C.length>0){ce(n,"htmx:validation:halted",H);ie(o);w();return l}var G=r.split("#");var J=G[0];var L=G[1];var A=r;if(q){A=J;var Z=Object.keys(T).length!==0;if(Z){if(A.indexOf("?")<0){A+="?"}else{A+="&"}A+=pr(T);if(L){A+="#"+L}}}if(!kr(n,A,H)){fe(n,"htmx:invalidPath",H);ie(s);return l}b.open(t.toUpperCase(),A,true);b.overrideMimeType("text/html");b.withCredentials=H.withCredentials;b.timeout=H.timeout;if(O.noHeaders){}else{for(var N in E){if(E.hasOwnProperty(N)){var K=E[N];Lr(b,N,K)}}}var I={xhr:b,target:u,requestConfig:H,etc:a,boosted:W,select:X,pathInfo:{requestPath:r,finalRequestPath:A,anchor:L}};b.onload=function(){try{var e=Ir(n);I.pathInfo.responsePath=Ar(b);M(n,I);lr(k,P);ce(n,"htmx:afterRequest",I);ce(n,"htmx:afterOnLoad",I);if(!se(n)){var t=null;while(e.length>0&&t==null){var r=e.shift();if(se(r)){t=r}}if(t){ce(t,"htmx:afterRequest",I);ce(t,"htmx:afterOnLoad",I)}}ie(o);w()}catch(e){fe(n,"htmx:onLoadError",le({error:e},I));throw e}};b.onerror=function(){lr(k,P);fe(n,"htmx:afterRequest",I);fe(n,"htmx:sendError",I);ie(s);w()};b.onabort=function(){lr(k,P);fe(n,"htmx:afterRequest",I);fe(n,"htmx:sendAbort",I);ie(s);w()};b.ontimeout=function(){lr(k,P);fe(n,"htmx:afterRequest",I);fe(n,"htmx:timeout",I);ie(s);w()};if(!ce(n,"htmx:beforeRequest",I)){ie(o);w();return l}var k=or(n);var P=sr(n);oe(["loadstart","loadend","progress","abort"],function(t){oe([b,b.upload],function(e){e.addEventListener(t,function(e){ce(n,"htmx:xhr:"+t,{lengthComputable:e.lengthComputable,loaded:e.loaded,total:e.total})})})});ce(n,"htmx:beforeSend",I);var Y=q?null:Er(b,n,T);b.send(Y);return l}function Pr(e,t){var r=t.xhr;var n=null;var i=null;if(O(r,/HX-Push:/i)){n=r.getResponseHeader("HX-Push");i="push"}else if(O(r,/HX-Push-Url:/i)){n=r.getResponseHeader("HX-Push-Url");i="push"}else if(O(r,/HX-Replace-Url:/i)){n=r.getResponseHeader("HX-Replace-Url");i="replace"}if(n){if(n==="false"){return{}}else{return{type:i,path:n}}}var a=t.pathInfo.finalRequestPath;var o=t.pathInfo.responsePath;var s=ne(e,"hx-push-url");var l=ne(e,"hx-replace-url");var u=ae(e).boosted;var f=null;var c=null;if(s){f="push";c=s}else if(l){f="replace";c=l}else if(u){f="push";c=o||a}if(c){if(c==="false"){return{}}if(c==="true"){c=o||a}if(t.pathInfo.anchor&&c.indexOf("#")===-1){c=c+"#"+t.pathInfo.anchor}return{type:f,path:c}}else{return{}}}function Mr(l,u){var f=u.xhr;var c=u.target;var e=u.etc;var t=u.requestConfig;var h=u.select;if(!ce(l,"htmx:beforeOnLoad",u))return;if(O(f,/HX-Trigger:/i)){_e(f,"HX-Trigger",l)}if(O(f,/HX-Location:/i)){er();var r=f.getResponseHeader("HX-Location");var v;if(r.indexOf("{")===0){v=E(r);r=v["path"];delete v["path"]}Nr("GET",r,v).then(function(){tr(r)});return}var n=O(f,/HX-Refresh:/i)&&"true"===f.getResponseHeader("HX-Refresh");if(O(f,/HX-Redirect:/i)){location.href=f.getResponseHeader("HX-Redirect");n&&location.reload();return}if(n){location.reload();return}if(O(f,/HX-Retarget:/i)){if(f.getResponseHeader("HX-Retarget")==="this"){u.target=l}else{u.target=ue(l,f.getResponseHeader("HX-Retarget"))}}var d=Pr(l,u);var i=f.status>=200&&f.status<400&&f.status!==204;var g=f.response;var a=f.status>=400;var p=Q.config.ignoreTitle;var o=le({shouldSwap:i,serverResponse:g,isError:a,ignoreTitle:p},u);if(!ce(c,"htmx:beforeSwap",o))return;c=o.target;g=o.serverResponse;a=o.isError;p=o.ignoreTitle;u.target=c;u.failed=a;u.successful=!a;if(o.shouldSwap){if(f.status===286){at(l)}R(l,function(e){g=e.transformResponse(g,f,l)});if(d.type){er()}var s=e.swapOverride;if(O(f,/HX-Reswap:/i)){s=f.getResponseHeader("HX-Reswap")}var v=wr(l,s);if(v.hasOwnProperty("ignoreTitle")){p=v.ignoreTitle}c.classList.add(Q.config.swappingClass);var m=null;var x=null;var y=function(){try{var e=document.activeElement;var t={};try{t={elt:e,start:e?e.selectionStart:null,end:e?e.selectionEnd:null}}catch(e){}var r;if(h){r=h}if(O(f,/HX-Reselect:/i)){r=f.getResponseHeader("HX-Reselect")}if(d.type){ce(re().body,"htmx:beforeHistoryUpdate",le({history:d},u));if(d.type==="push"){tr(d.path);ce(re().body,"htmx:pushedIntoHistory",{path:d.path})}else{rr(d.path);ce(re().body,"htmx:replacedInHistory",{path:d.path})}}var n=T(c);je(v.swapStyle,c,l,g,n,r);if(t.elt&&!se(t.elt)&&ee(t.elt,"id")){var i=document.getElementById(ee(t.elt,"id"));var a={preventScroll:v.focusScroll!==undefined?!v.focusScroll:!Q.config.defaultFocusScroll};if(i){if(t.start&&i.setSelectionRange){try{i.setSelectionRange(t.start,t.end)}catch(e){}}i.focus(a)}}c.classList.remove(Q.config.swappingClass);oe(n.elts,function(e){if(e.classList){e.classList.add(Q.config.settlingClass)}ce(e,"htmx:afterSwap",u)});if(O(f,/HX-Trigger-After-Swap:/i)){var o=l;if(!se(l)){o=re().body}_e(f,"HX-Trigger-After-Swap",o)}var s=function(){oe(n.tasks,function(e){e.call()});oe(n.elts,function(e){if(e.classList){e.classList.remove(Q.config.settlingClass)}ce(e,"htmx:afterSettle",u)});if(u.pathInfo.anchor){var e=re().getElementById(u.pathInfo.anchor);if(e){e.scrollIntoView({block:"start",behavior:"auto"})}}if(n.title&&!p){var t=C("title");if(t){t.innerHTML=n.title}else{window.document.title=n.title}}Cr(n.elts,v);if(O(f,/HX-Trigger-After-Settle:/i)){var r=l;if(!se(l)){r=re().body}_e(f,"HX-Trigger-After-Settle",r)}ie(m)};if(v.settleDelay>0){setTimeout(s,v.settleDelay)}else{s()}}catch(e){fe(l,"htmx:swapError",u);ie(x);throw e}};var b=Q.config.globalViewTransitions;if(v.hasOwnProperty("transition")){b=v.transition}if(b&&ce(l,"htmx:beforeTransition",u)&&typeof Promise!=="undefined"&&document.startViewTransition){var w=new Promise(function(e,t){m=e;x=t});var S=y;y=function(){document.startViewTransition(function(){S();return w})}}if(v.swapDelay>0){setTimeout(y,v.swapDelay)}else{y()}}if(a){fe(l,"htmx:responseError",le({error:"Response Status Error Code "+f.status+" from "+u.pathInfo.requestPath},u))}}var Xr={};function Dr(){return{init:function(e){return null},onEvent:function(e,t){return true},transformResponse:function(e,t,r){return e},isInlineSwap:function(e){return false},handleSwap:function(e,t,r,n){return false},encodeParameters:function(e,t,r){return null}}}function Ur(e,t){if(t.init){t.init(r)}Xr[e]=le(Dr(),t)}function Br(e){delete Xr[e]}function Fr(e,r,n){if(e==undefined){return r}if(r==undefined){r=[]}if(n==undefined){n=[]}var t=te(e,"hx-ext");if(t){oe(t.split(","),function(e){e=e.replace(/ /g,"");if(e.slice(0,7)=="ignore:"){n.push(e.slice(7));return}if(n.indexOf(e)<0){var t=Xr[e];if(t&&r.indexOf(t)<0){r.push(t)}}})}return Fr(u(e),r,n)}var Vr=false;re().addEventListener("DOMContentLoaded",function(){Vr=true});function jr(e){if(Vr||re().readyState==="complete"){e()}else{re().addEventListener("DOMContentLoaded",e)}}function _r(){if(Q.config.includeIndicatorStyles!==false){re().head.insertAdjacentHTML("beforeend","")}}function zr(){var e=re().querySelector('meta[name="htmx-config"]');if(e){return E(e.content)}else{return null}}function $r(){var e=zr();if(e){Q.config=le(Q.config,e)}}jr(function(){$r();_r();var e=re().body;zt(e);var t=re().querySelectorAll("[hx-trigger='restored'],[data-hx-trigger='restored']");e.addEventListener("htmx:abort",function(e){var t=e.target;var r=ae(t);if(r&&r.xhr){r.xhr.abort()}});const r=window.onpopstate?window.onpopstate.bind(window):null;window.onpopstate=function(e){if(e.state&&e.state.htmx){ar();oe(t,function(e){ce(e,"htmx:restored",{document:re(),triggerEvent:ce})})}else{if(r){r(e)}}};setTimeout(function(){ce(e,"htmx:load",{});e=null},0)});return Q}()}); \ No newline at end of file diff --git a/goodprice/web/static/navigation.js b/goodprice/web/static/navigation.js new file mode 100644 index 0000000..109aec8 --- /dev/null +++ b/goodprice/web/static/navigation.js @@ -0,0 +1,81 @@ +(function () { + 'use strict'; + if (location.pathname !== '/listings') return; + function currentContext() { + var raw = new URLSearchParams(location.search), clean = new URLSearchParams(); + var task = raw.get('task_id'); + if (task && /^[0-9]+$/.test(task) && Number(task) > 0 && Number(task) <= 2147483647) clean.set('task_id', String(Number(task))); + var show = raw.get('show'), sort = raw.get('sort'); + clean.set('show', ['active','gone','blocked','all','unseen'].includes(show) ? show : 'active'); + clean.set('sort', ['satisfaction','price_asc','price_desc','newest'].includes(sort) ? sort : 'satisfaction'); + var state = raw.get('price_state'); + if (['exact','pending'].includes(state)) clean.set('price_state', state); + var loaded = raw.get('loaded'); + if (loaded && /^[0-9]+$/.test(loaded) && Number(loaded) > 100) clean.set('loaded', String(Math.min(1000, Number(loaded)))); + return '/listings?' + clean.toString(); + } + var key = function () { return 'second-eye:list:' + currentContext(); }; + var lastInteraction = 0; + function save(anchor) { + try { + sessionStorage.setItem(key(), JSON.stringify({y: window.scrollY, anchor: anchor || null})); + } catch (_) {} + } + function restore() { + try { + var state = JSON.parse(sessionStorage.getItem(key()) || 'null'); + if (!state) return; + requestAnimationFrame(function () { requestAnimationFrame(function () { + var anchor = state.anchor && document.getElementById('listing-' + state.anchor); + if (anchor && Math.abs(anchor.offsetTop - state.y) < 500) anchor.scrollIntoView(); + else window.scrollTo(0, state.y || 0); + }); }); + } catch (_) {} + } + function updateLinks() { + var context = currentContext(); + document.querySelectorAll('#listings-grid a[href^="/listings/"], #listings-grid form[action^="/listings/"]').forEach(function(el) { + var attr = el.tagName === 'FORM' ? 'action' : 'href'; + var url = new URL(el.getAttribute(attr), location.origin); + url.searchParams.set('return_to', context); + el.setAttribute(attr, url.pathname + url.search); + }); + var poller = document.getElementById('list-poller'); + if (poller) { + var pollURL = new URL(context, location.origin); + pollURL.searchParams.set('partial', '1'); + poller.setAttribute('hx-get', pollURL.pathname + pollURL.search); + if (window.htmx) htmx.process(poller); + } + var refresh = document.getElementById('new-list-results'); + if (refresh) refresh.href = context; + } + document.addEventListener('click', function(event) { + lastInteraction = Date.now(); + var card = event.target.closest('[data-listing-id]'); + save(card && card.dataset.listingId); + }); + document.addEventListener('input', function() { lastInteraction = Date.now(); }); + window.addEventListener('pagehide', function() { save(); }); + window.addEventListener('pageshow', restore); + document.addEventListener('htmx:beforeSwap', function(event) { + if (!event.detail.target || event.detail.target.id !== 'listings-grid') return; + var poller = event.detail.requestConfig && event.detail.requestConfig.elt; + if (poller && poller.id === 'list-poller' && ( + document.querySelector('.row-check:checked') || window.scrollY > 100 || + Date.now() - lastInteraction < 15000 || document.activeElement.closest('form'))) { + event.detail.shouldSwap = false; + document.getElementById('new-list-results').hidden = false; + } + }); + document.addEventListener('htmx:afterSettle', function() { + var seen = new Set(); + document.querySelectorAll('[data-listing-id]').forEach(function(card) { + if (seen.has(card.dataset.listingId)) card.remove(); + else seen.add(card.dataset.listingId); + }); + updateLinks(); + save(); + }); + updateLinks(); +})(); diff --git a/goodprice/web/static/tailwind.LICENSE b/goodprice/web/static/tailwind.LICENSE new file mode 100644 index 0000000..d6a8229 --- /dev/null +++ b/goodprice/web/static/tailwind.LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) Tailwind Labs, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/goodprice/web/static/tailwind.input.css b/goodprice/web/static/tailwind.input.css new file mode 100644 index 0000000..b5c61c9 --- /dev/null +++ b/goodprice/web/static/tailwind.input.css @@ -0,0 +1,3 @@ +@tailwind base; +@tailwind components; +@tailwind utilities; diff --git a/goodprice/web/templates/base.html b/goodprice/web/templates/base.html index cfc91da..afc57f3 100644 --- a/goodprice/web/templates/base.html +++ b/goodprice/web/templates/base.html @@ -4,8 +4,9 @@ {% block title %}闲鱼盯价助手{% endblock %} - - + + +