[ { "path": ".dockerignore", "content": "__pycache__\n*.pyc\n.env\n.venv\nvenv\n.idea\n.claude\n.serena\n.pytest_cache\nlogs/\njsonl/\nweb-ui/node_modules\nweb-ui/dist\ndist/\n.git\n.DS_Store\ntask_images/\nimages/\narchive/\ntests/\ndata/\nprice_history/\n*.md\n!README.md\n" }, { "path": ".github/workflows/claude.yml", "content": "name: Claude Code\n\non:\n issue_comment:\n types: [created]\n pull_request_review_comment:\n types: [created]\n issues:\n types: [opened, assigned]\n pull_request_review:\n types: [submitted]\n\njobs:\n claude:\n if: |\n (github.event_name == 'issue_comment' && contains(github.event.comment.body, '@claude')) ||\n (github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '@claude')) ||\n (github.event_name == 'pull_request_review' && contains(github.event.review.body, '@claude')) ||\n (github.event_name == 'issues' && (contains(github.event.issue.body, '@claude') || contains(github.event.issue.title, '@claude')))\n runs-on: ubuntu-latest\n permissions:\n contents: read\n pull-requests: read\n issues: read\n id-token: write\n steps:\n - name: Checkout repository\n uses: actions/checkout@v4\n with:\n fetch-depth: 1\n\n - name: Prepare Environment\n run: |\n curl -fsSL https://bun.sh/install | bash\n mkdir -p $HOME/.claude-code-router\n cat << 'EOF' > $HOME/.claude-code-router/config.json\n {\n \"log\": true,\n \"NON_INTERACTIVE_MODE\": true,\n \"OPENAI_API_KEY\": \"${{ secrets.OPENAI_API_KEY }}\",\n \"OPENAI_BASE_URL\": \"https://api-inference.modelscope.cn/v1\",\n \"OPENAI_MODEL\": \"MiniMax/MiniMax-M2.5\"\n }\n EOF\n shell: bash\n\n - name: Start Claude Code Router\n run: |\n nohup ~/.bun/bin/bunx @musistudio/claude-code-router@1.0.8 start &\n shell: bash\n\n - name: Run Claude Code\n id: claude\n uses: anthropics/claude-code-action@beta\n env:\n ANTHROPIC_BASE_URL: http://localhost:3456\n with:\n anthropic_api_key: \"Any-string-is-ok\"\n" }, { "path": ".github/workflows/docker-image.yml", "content": "name: Docker Image CI on merge to master\n\non:\n workflow_dispatch:\n pull_request:\n types: [closed]\n branches: [\"master\"]\n\nenv:\n IMAGE_NAME: ai-goofish\n BASE_IMAGE_NAME: ai-goofish-base\n\njobs:\n build-base:\n if: |\n github.event_name == 'workflow_dispatch' ||\n (github.event_name == 'pull_request' && github.event.pull_request.merged == true)\n runs-on: ubuntu-latest\n permissions:\n contents: read\n packages: write\n outputs:\n base_image: ${{ steps.prepare.outputs.base_image }}\n\n steps:\n - name: Checkout code\n uses: actions/checkout@v4\n with:\n fetch-depth: 0\n\n - name: Set up QEMU\n uses: docker/setup-qemu-action@v3\n\n - name: Set up Docker Buildx\n uses: docker/setup-buildx-action@v3\n\n - name: Prepare image tags\n id: prepare\n env:\n REPO_OWNER: ${{ github.repository_owner }}\n BASE_IMAGE_NAME: ${{ env.BASE_IMAGE_NAME }}\n run: |\n set -euo pipefail\n\n owner_lower=$(echo \"$REPO_OWNER\" | tr '[:upper:]' '[:lower:]')\n base_image=\"ghcr.io/${owner_lower}/${BASE_IMAGE_NAME}:latest\"\n\n base_tags=(\n \"${base_image}\"\n \"ghcr.io/${owner_lower}/${BASE_IMAGE_NAME}:${GITHUB_SHA}\"\n )\n\n {\n echo \"base_image=${base_image}\"\n echo 'base_tags<> \"$GITHUB_OUTPUT\"\n\n - name: Log in to GitHub Container Registry\n uses: docker/login-action@v3\n with:\n registry: ghcr.io\n username: ${{ github.actor }}\n password: ${{ secrets.GITHUB_TOKEN }}\n\n - name: Build and push base Docker image\n uses: docker/build-push-action@v5\n with:\n context: .\n file: ./Dockerfile.base\n platforms: linux/amd64,linux/arm64\n pull: true\n push: true\n tags: ${{ steps.prepare.outputs.base_tags }}\n cache-from: type=gha,scope=ai-goofish-base-docker\n cache-to: type=gha,scope=ai-goofish-base-docker,mode=max\n\n build-and-push:\n needs: build-base\n if: ${{ needs.build-base.result == 'success' }}\n runs-on: ubuntu-latest\n permissions:\n contents: read\n packages: write\n\n steps:\n - name: Checkout code\n uses: actions/checkout@v4\n with:\n fetch-depth: 0\n\n - name: Set up QEMU\n uses: docker/setup-qemu-action@v3\n\n - name: Set up Docker Buildx\n uses: docker/setup-buildx-action@v3\n\n - name: Prepare release tags\n id: prepare\n env:\n REPO_OWNER: ${{ github.repository_owner }}\n IMAGE_NAME: ${{ env.IMAGE_NAME }}\n run: |\n set -euo pipefail\n\n owner_lower=$(echo \"$REPO_OWNER\" | tr '[:upper:]' '[:lower:]')\n\n app_tags=(\n \"ghcr.io/${owner_lower}/${IMAGE_NAME}:latest\"\n \"ghcr.io/${owner_lower}/${IMAGE_NAME}:${GITHUB_SHA}\"\n )\n\n {\n echo 'app_tags<> \"$GITHUB_OUTPUT\"\n\n - name: Log in to GitHub Container Registry\n uses: docker/login-action@v3\n with:\n registry: ghcr.io\n username: ${{ github.actor }}\n password: ${{ secrets.GITHUB_TOKEN }}\n\n - name: Build and push multi-arch Docker images\n uses: docker/build-push-action@v5\n with:\n context: .\n file: ./Dockerfile.release\n platforms: linux/amd64,linux/arm64\n pull: true\n push: true\n build-args: |\n BASE_IMAGE=${{ needs.build-base.outputs.base_image }}\n tags: ${{ steps.prepare.outputs.app_tags }}\n cache-from: type=gha,scope=ai-goofish-release-docker\n cache-to: type=gha,scope=ai-goofish-release-docker,mode=max\n" }, { "path": ".gitignore", "content": ".idea\n*.iml\nxianyu_state.json\n.env\n.aider*\nimages/\nlogs/\njsonl/\n__pycache__/\nsrc/__pycache__/\ndist/\nstate/\nconfig.json\nprompts/\n.serena/\nprice_history/\ndata/\n" }, { "path": "AGENTS.md", "content": "# Repository Guidelines\n\n## 项目结构与模块组织\n- 后端位于 `src/`，入口 `src/app.py`，API 路由在 `src/api/routes/`，服务层在 `src/services/`，领域模型在 `src/domain/`，基础设施在 `src/infrastructure/`。\n- 前端在 `web-ui/`（Vue 3 + Vite），视图放于 `web-ui/src/views/`，组件在 `web-ui/src/components/`，构建产物会复制到根目录 `dist/`。\n- 测试位于 `tests/`，命名遵循 `test_*.py` 或 `tests/*/test_*.py`。\n- 运行数据与资源：`prompts/`、`jsonl/`、`logs/`、`images/`、`static/`、`state/`，配置文件 `config.json` 与 `.env` 位于仓库根目录。\n\n## 构建、测试与本地开发\n- 后端开发：`python -m src.app` 或 `uvicorn src.app:app --host 0.0.0.0 --port 8000 --reload`。\n- 爬虫任务：`python spider_v2.py --task-name \"MacBook Air M1\" --debug-limit 3`（可用 `--config` 指定自定义配置）。\n- 前端开发：`cd web-ui && npm install && npm run dev`；构建：`cd web-ui && npm run build`（产物复制到根目录 `dist/`）。\n- 一键本地启动：`bash start.sh`（自动安装依赖、前端构建并启动后端）。\n- Docker：`docker compose up --build -d`，查看日志 `docker compose logs -f app`，停止 `docker compose down`。\n\n## 编码风格与命名约定\n- 保持分层：API → services → domain → infrastructure，避免跨层耦合，模块保持精简。\n- Python 测试函数命名为 `test_*`，文件与路径遵循上述测试目录规范。\n- 使用描述性、任务导向的命名（如爬虫任务名、配置键），与业务含义对应。\n\n## 架构与运行时\n- 后端使用 FastAPI 提供 API 与静态资源，爬虫与 AI 推理在独立任务进程中协作，前后端通过 HTTP/Web UI 交互。\n- 任务运行会在 `jsonl/` 写入结果、在 `logs/` 留存运行日志、在 `images/` 下载图片，前端监控页面依赖这些数据。\n- 默认监听 8000 端口，前端构建后静态文件可由后端或 Docker 镜像直接提供。\n\n## 测试指南\n- 测试框架：`pytest`（默认同步测试，无需 `pytest-asyncio`）。\n- 运行全部测试：`pytest`；覆盖率：`pytest --cov=src` 或 `coverage run -m pytest`；定向测试：`pytest tests/test_utils.py::test_safe_get`。\n- 优先覆盖核心服务、爬虫管道的异常分支与重试逻辑，避免回归。\n- PR 前请运行相关测试，新增逻辑补充针对性用例。\n\n## 提交与 PR 规范\n- Commit 采用类 Conventional Commits：`feat(...)`、`fix(...)`、`refactor(...)`、`chore(...)`、`docs(...)` 等。\n- PR 需说明变更范围与影响模块；UI 变更在 `web-ui/` 提供截图；关联相关 Issue；提及配置或迁移步骤。\n\n## 安全与配置提示\n- 复制 `.env.example` 为 `.env`，设置必填项 `OPENAI_API_KEY`、`OPENAI_BASE_URL`、`OPENAI_MODEL_NAME` 等。\n- 不要提交真实凭据或 cookies（如 `state.json`）；Playwright 需本地浏览器，Docker 镜像已预装 Chromium。\n- Web 认证默认 `admin/admin123`，生产环境务必修改，推荐启用 HTTPS 并限制访问来源。\n" }, { "path": "CLAUDE.md", "content": "# CLAUDE.md\n\nThis file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.\n\n## 项目概述\n\n基于 Playwright + AI 的闲鱼智能监控机器人。FastAPI 后端 + Vue 3 前端，支持多任务并发监控、多模态 AI 商品分析、多渠道通知推送。\n\n## 核心架构\n\n```\nAPI层 (src/api/routes/)\n ↓\n服务层 (src/services/)\n ↓\n领域层 (src/domain/)\n ↓\n基础设施层 (src/infrastructure/)\n```\n\n关键入口：\n- `src/app.py` - FastAPI 应用主入口\n- `spider_v2.py` - 爬虫 CLI 入口\n- `src/scraper.py` - Playwright 爬虫核心逻辑\n\n服务层：\n- `TaskService` - 任务 CRUD\n- `ProcessService` - 爬虫子进程管理\n- `SchedulerService` - APScheduler 定时调度\n- `AIAnalysisService` - 多模态 AI 分析\n- `NotificationService` - 多渠道通知（ntfy/Bark/企业微信/Telegram/Webhook）\n\n前端 (`web-ui/`)：Vue 3 + Vite + shadcn-vue + Tailwind CSS\n\n## 开发命令\n\n```bash\n# 后端开发\npython -m src.app\n# 或\nuvicorn src.app:app --host 0.0.0.0 --port 8000 --reload\n\n# 前端开发\ncd web-ui && npm install && npm run dev\n\n# 前端构建\ncd web-ui && npm run build\n\n# 一键本地启动（构建前端 + 启动后端）\nbash start.sh\n\n# Docker 部署\ndocker compose up --build -d\n```\n\n## 爬虫命令\n\n```bash\npython spider_v2.py # 运行所有启用任务\npython spider_v2.py --task-name \"MacBook\" # 运行指定任务\npython spider_v2.py --debug-limit 3 # 调试模式，限制商品数\npython spider_v2.py --config custom.json # 自定义配置文件\n```\n\n## 测试\n\n```bash\npytest # 运行所有测试\npytest --cov=src # 覆盖率报告\npytest tests/unit/test_utils.py # 运行单个测试文件\npytest tests/unit/test_utils.py::test_safe_get # 运行单个测试函数\n```\n\n测试规范：文件 `tests/**/test_*.py`，函数 `test_*`\n\n## 配置\n\n环境变量 (`.env`)：\n- AI 模型：`OPENAI_API_KEY`, `OPENAI_BASE_URL`, `OPENAI_MODEL_NAME`\n- 通知：`NTFY_TOPIC_URL`, `BARK_URL`, `WX_BOT_URL`, `TELEGRAM_BOT_TOKEN`, `TELEGRAM_CHAT_ID`\n- 爬虫：`RUN_HEADLESS`, `LOGIN_IS_EDGE`\n- Web 认证：`WEB_USERNAME`, `WEB_PASSWORD`\n- 端口：`SERVER_PORT`\n\n任务配置 (`config.json`)：定义监控任务（关键词、价格范围、cron 表达式、AI prompt 文件等）\n\n## 数据流\n\n1. Web UI / config.json 创建任务\n2. SchedulerService 按 cron 触发或手动启动\n3. ProcessService 启动 spider_v2.py 子进程\n4. scraper.py 使用 Playwright 抓取商品\n5. AIAnalysisService 调用多模态模型分析\n6. NotificationService 推送符合条件的商品\n7. 结果存储：`jsonl/`（数据）、`images/`（图片）、`logs/`（日志）\n\n## 注意事项\n\n- AI 模型必须支持图片上传（多模态）\n- Docker 部署需通过 Web UI 手动更新登录状态（`state.json`）\n- 遇到滑动验证码时设置 `RUN_HEADLESS=false` 手动处理\n- 生产环境务必修改默认 Web 认证密码\n" }, { "path": "DISCLAIMER.md", "content": "# 免责声明 / Disclaimer\n\n## 中文版本\n\n本项目是一个开源软件，仅供学习和研究目的使用。使用者在使用本软件时，必须遵守所在国家/地区的所有相关法律法规。\n\n项目作者及贡献者明确声明：\n\n1. 本项目仅用于技术学习和研究目的，不得用于任何违法或不道德的活动。\n2. 使用者对本软件的使用行为承担全部责任，包括但不限于任何修改、分发或商业应用。\n3. 项目作者及贡献者不对因使用本软件而导致的任何直接、间接、附带或特殊的损害或损失承担责任，即使已被告知可能发生此类损害。\n4. 如果您的使用行为违反了所在司法管辖区的法律，请立即停止使用并删除本软件。\n5. 本项目按\"现状\"提供，不提供任何形式的担保，包括但不限于适销性、特定用途适用性和非侵权性担保。\n\n本项目采用 MIT 许可证发布。根据该许可证，您可以自由使用、复制、修改、分发本软件，但必须保留原始版权声明和本免责声明。\n\n项目作者保留随时更改本免责声明的权利，恕不另行通知。使用本软件即表示您同意受本免责声明条款的约束。\n\n## English Version\n\nThis project is open source software provided for learning and research purposes only. Users must comply with all relevant laws and regulations in their jurisdiction when using this software.\n\nThe project owner and contributors explicitly state:\n\n1. This project is for technical learning and research purposes only and must not be used for any illegal or unethical activities.\n2. Users assume full responsibility for their use of the software, including but not limited to any modifications, distributions, or commercial applications.\n3. The project owner and contributors are not liable for any direct, indirect, incidental, or special damages or losses resulting from the use of this software, even if advised of the possibility of such damages.\n4. If your use violates the laws of your jurisdiction, please stop using and delete this software immediately.\n5. This project is provided \"as is\" without warranty of any kind, either express or implied, including but not limited to the warranties of merchantability, fitness for a particular purpose, or non-infringement.\n\nThis project is released under the MIT License. Under this license, you are free to use, copy, modify, and distribute this software, but you must retain the original copyright notice and this disclaimer.\n\nThe project owner reserves the right to change this disclaimer at any time without notice. Your use of the software indicates your acceptance of the terms of this disclaimer." }, { "path": "Dockerfile", "content": "# Stage 1: Build the Vue application\nFROM node:22-alpine AS frontend-builder\nWORKDIR /web-ui\nCOPY web-ui/package*.json ./\nRUN npm ci\nCOPY web-ui/ .\nRUN npm run build\n\n# Stage 2: Build the python environment with dependencies\nFROM python:3.11-slim-bookworm AS builder\n\n# 设置环境变量以防止交互式提示\nENV DEBIAN_FRONTEND=noninteractive \\\n VIRTUAL_ENV=/opt/venv \\\n PATH=\"/opt/venv/bin:$PATH\"\n\n# 创建虚拟环境并安装 Python 运行时依赖\nRUN python3 -m venv $VIRTUAL_ENV\nCOPY requirements-runtime.txt .\nRUN pip install --no-cache-dir -i https://pypi.tuna.tsinghua.edu.cn/simple -r requirements-runtime.txt\n\n# Stage 3: Create the final, lean image\nFROM python:3.11-slim-bookworm\n\nWORKDIR /app\nENV DEBIAN_FRONTEND=noninteractive \\\n VIRTUAL_ENV=/opt/venv \\\n PATH=\"/opt/venv/bin:$PATH\" \\\n PYTHONUNBUFFERED=1 \\\n RUNNING_IN_DOCKER=true \\\n PLAYWRIGHT_BROWSERS_PATH=/ms-playwright \\\n TZ=Asia/Shanghai\n\nCOPY --from=builder ${VIRTUAL_ENV} ${VIRTUAL_ENV}\n\nRUN apt-get update \\\n && apt-get install -y --no-install-recommends \\\n tzdata \\\n tini \\\n libzbar0 \\\n && playwright install --with-deps --no-shell chromium \\\n && apt-get clean \\\n && rm -rf /var/lib/apt/lists/*\n\nCOPY --from=frontend-builder /dist /app/dist\n\nCOPY src /app/src\nCOPY spider_v2.py /app/spider_v2.py\nCOPY prompts /app/prompts\nCOPY static /app/static\nCOPY config.json.example /app/config.json.example\n\nRUN mkdir -p /app/data /app/state /app/logs /app/images /app/jsonl /app/price_history\n\nEXPOSE 8000\n\nUSER root\n\nENTRYPOINT [\"tini\", \"--\"]\n\nCMD [\"python\", \"-m\", \"src.app\"]\n" }, { "path": "Dockerfile.base", "content": "# syntax=docker/dockerfile:1.7\n\nFROM python:3.11-slim-bookworm\n\nWORKDIR /app\n\nENV DEBIAN_FRONTEND=noninteractive \\\n VIRTUAL_ENV=/opt/venv \\\n PATH=\"/opt/venv/bin:$PATH\" \\\n PYTHONUNBUFFERED=1 \\\n RUNNING_IN_DOCKER=true \\\n PLAYWRIGHT_BROWSERS_PATH=/ms-playwright \\\n TZ=Asia/Shanghai\n\nCOPY requirements-runtime.txt /tmp/requirements-runtime.txt\n\nRUN python3 -m venv \"$VIRTUAL_ENV\"\n\nRUN --mount=type=cache,target=/root/.cache/pip \\\n pip install -i https://pypi.tuna.tsinghua.edu.cn/simple -r /tmp/requirements-runtime.txt\n\nRUN apt-get update \\\n && apt-get install -y --no-install-recommends \\\n tzdata \\\n tini \\\n libzbar0 \\\n && playwright install --with-deps --no-shell chromium \\\n && apt-get clean \\\n && rm -rf /var/lib/apt/lists/* /tmp/requirements-runtime.txt\n\nRUN mkdir -p /app/data /app/state /app/logs /app/images /app/jsonl /app/price_history\n\nENTRYPOINT [\"tini\", \"--\"]\n" }, { "path": "Dockerfile.release", "content": "# syntax=docker/dockerfile:1.7\n\nARG BASE_IMAGE=ghcr.io/usagi-org/ai-goofish-base:latest\n\nFROM node:22-alpine AS frontend-builder\n\nWORKDIR /web-ui\n\nCOPY web-ui/package*.json ./\n\nRUN --mount=type=cache,target=/root/.npm npm ci\n\nCOPY web-ui/ .\n\nRUN npm run build\n\nFROM ${BASE_IMAGE}\n\nWORKDIR /app\n\nCOPY --from=frontend-builder /dist /app/dist\n\nCOPY src /app/src\nCOPY spider_v2.py /app/spider_v2.py\nCOPY prompts /app/prompts\nCOPY static /app/static\nCOPY config.json.example /app/config.json.example\n\nEXPOSE 8000\n\nCMD [\"python\", \"-m\", \"src.app\"]\n" }, { "path": "LICENSE", "content": "MIT License\n\nCopyright (c) 2025 dingyufei615\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE." }, { "path": "README.md", "content": "# 闲鱼智能监控机器人\n\n[English README](README_EN.md)\n\n基于 Playwright 和 AI 的闲鱼多任务实时监控工具，提供完整的 Web 管理界面。\n\n## 核心特性\n\n- **Web 可视化管理**: 任务管理、账号管理、AI 标准编辑、运行日志、结果浏览\n- **AI 驱动**: 自然语言创建任务，多模态模型深度分析商品\n- **多任务并发**: 独立配置关键词、价格、筛选条件和 AI Prompt\n- **高级筛选**: 包邮、新发布时间范围、省/市/区三级区域筛选\n- **即时通知**: 支持 ntfy.sh、企业微信、Bark、Telegram、Webhook\n- **定时调度**: Cron 表达式配置周期性任务\n- **账号与代理轮换**: 多账号管理、任务绑定账号、代理池轮换与失败重试\n- **Docker 部署**: 一键容器化部署\n\n## 截图\n\n![监控概览](static/img.png)\n![任务管理](static/img_1.png)\n![结果查看](static/img_2.png)\n![通知推送](static/img_3.png)\n\n## 🐳 Docker 部署（推荐）\n\n```bash\ngit clone https://github.com/Usagi-org/ai-goofish-monitor && cd ai-goofish-monitor\ncp .env.example .env\nvim .env # 填写相关配置项\ndocker compose up -d\ndocker compose logs -f app\ndocker compose down\n```\n\n如果镜像无法访问或下载速度慢，可尝试使用加速：\n```bash\n\ndocker pull ghcr.nju.edu.cn/usagi-org/ai-goofish:latest\ndocker tag ghcr.nju.edu.cn/usagi-org/ai-goofish:latest ghcr.io/usagi-org/ai-goofish:latest\ndocker compose up -d\n\n```\n\n- 默认 Web UI 地址：`http://127.0.0.1:8000`\n- Docker 镜像已内置 Chromium，无需宿主机额外安装浏览器。\n- 官方镜像地址：`ghcr.io/usagi-org/ai-goofish:latest`\n- 更新镜像：`docker compose pull && docker compose up -d`\n- 如果你修改了 `.env` 中的 `SERVER_PORT`，请同步更新 `docker-compose.yaml` 里的端口映射。\n- `docker-compose.yaml` 默认会把 SQLite 主库挂载到 `./data:/app/data`，数据库文件默认为 `data/app.sqlite3`\n- 目前默认持久化这些目录：\n - `data/` SQLite 主存储（任务、结果、价格历史）\n - `state/` 登录状态 cookie 文件\n - `prompts/` 任务提示词\n - `logs/` 运行日志\n - `images/` 商品图片与任务临时图片目录\n - `config.json`、`jsonl/`、`price_history/` 首次升级到 SQLite 时用于兼容导入的旧数据源\n\n### 数据存储与迁移\n\n- 当前在线主存储为 SQLite，默认路径 `data/app.sqlite3`\n- 可通过环境变量 `APP_DATABASE_FILE` 自定义数据库路径；Docker 默认设置为 `/app/data/app.sqlite3`\n- 应用启动时会自动建库建表，并尝试从旧的 `config.json`、`jsonl/`、`price_history/` 导入一次历史数据\n- `state/`、`prompts/`、`logs/`、`images/` 仍然是文件系统目录，不在 SQLite 中\n- 商品图片会临时落到 `images/task_images_/`，任务结束后默认会清理\n- 首次升级完成并确认 `data/app.sqlite3` 中数据正确后，可视部署方式决定是否继续保留旧的 `config.json`、`jsonl/`、`price_history/` 挂载\n\n### 最少配置\n\n| 变量 | 说明 | 必填 |\n|------|------|------|\n| `OPENAI_API_KEY` | AI 模型 API Key | 是 |\n| `OPENAI_BASE_URL` | OpenAI 兼容接口地址 | 是 |\n| `OPENAI_MODEL_NAME` | 支持图片输入的模型名称 | 是 |\n| `WEB_USERNAME` / `WEB_PASSWORD` | Web UI 登录账号密码，默认 `admin/admin123` | 否 |\n\n其余配置见下方“配置说明”。\n\n\n### 第一次使用\n\n1. 打开默认 Web UI `http://127.0.0.1:8000` 并登录。\n2. 进入“闲鱼账号管理”，使用 [Chrome 扩展](https://chromewebstore.google.com/detail/xianyu-login-state-extrac/eidlpfjiodpigmfcahkmlenhppfklcoa) 导出并粘贴闲鱼登录态 JSON。\n3. 登录态文件会保存到 `state/` 目录，例如 `state/acc_1.json`。\n4. 回到“任务管理”，创建任务并绑定账号后即可运行。\n\n### 创建第一个任务\n\n- `AI判断`：填写“详细需求”，提交后会弹出独立进度弹窗，后台异步生成分析标准。\n- `关键词判断`：填写关键词规则，任务会直接创建，不经过 AI 生成流程。\n- `区域筛选`：已改为省 / 市 / 区三级选择器，数据基于闲鱼页面抓取快照内置。\n\n\n\n## 用户使用说明\n\n
\n点击展开 Web UI 功能说明\n\n### 任务管理\n\n- 支持 AI 创建、关键词规则、价格范围、新发布范围、区域筛选、账号绑定、定时规则。\n- AI 任务创建是后台 job 流程，提交后会打开单独的进度弹窗。\n- 区域筛选会显著缩小结果集，默认留空。\n\n### 账号管理\n\n- 支持导入、更新、删除闲鱼账号登录态。\n- 每个任务可指定账号，也可不绑定并交给系统自动选择。\n\n### 结果查看与运行日志\n\n- 结果页和导出功能现在从 SQLite 查询，不再直接扫描 `jsonl` 文件。\n- 日志页按任务展示运行过程，便于排查登录态失效、风控和 AI 调用问题。\n\n### 系统设置\n\n- 可查看系统状态、编辑 Prompt、调整代理与轮换相关配置。\n\n
\n\n\n\n## 开发者开发\n\n### 环境要求\n\n- Python 3.10+\n- Node.js + npm（本地验证 `Node v20.18.3` 可完成前端构建）\n- Playwright CLI 与 Chromium，首次运行前建议执行 `python3 -m pip install playwright && python3 -m playwright install chromium`\n- Chrome / Edge 浏览器（Linux 环境也可使用 Chromium；`start.sh` 会先检查浏览器是否存在）\n\n```bash\ngit clone https://github.com/Usagi-org/ai-goofish-monitor\ncd ai-goofish-monitor\ncp .env.example .env\n```\n\n### 一键启动\n\n```bash\nchmod +x start.sh\n./start.sh\n```\n\n`start.sh` 会先检查 Playwright CLI 和浏览器前置条件；在前置条件满足后自动安装项目依赖、构建前端、复制构建产物并启动后端。\n\n### 手动启动\n\n```bash\n# 后端\npython -m src.app\n# 或\nuvicorn src.app:app --host 0.0.0.0 --port 8000 --reload\n\n# 前端\ncd web-ui\nnpm install\nnpm run dev\n```\n\n- FastAPI 启动时会自动初始化 SQLite，并在首次启动时尝试导入旧的 `config.json/jsonl/price_history`\n- `spider_v2.py` 默认从 SQLite 读取任务；只有显式传入 `--config ` 时才会走 JSON 配置兼容模式\n- 默认数据库路径为 `data/app.sqlite3`\n- Vite 开发服务器会将 `/api`、`/auth`、`/ws` 代理到 `http://127.0.0.1:8000`。\n- `npm run build` 先生成 `web-ui/dist/`，`start.sh` 再复制到仓库根目录 `dist/`。\n- FastAPI 负责提供根目录 `dist/index.html` 和 `dist/assets/`。\n- `./start.sh` 默认输出访问地址 `http://localhost:8000` 和 API 文档 `http://localhost:8000/docs`。\n\n### 测试与校验\n\n```bash\nPYTEST_DISABLE_PLUGIN_AUTOLOAD=1 pytest\ncd web-ui && npm run build\n```\n\n### 任务创建 API\n\n
\n点击展开 API 行为说明\n\n- `POST /api/tasks/generate`\n - `decision_mode=ai`：返回 `202` 和 `job`，需要继续轮询进度。\n - `decision_mode=keyword`：直接返回已创建任务。\n- `GET /api/tasks/generate-jobs/{job_id}`：查询 AI 任务生成进度。\n- `POST /auth/status`：校验 Web UI 登录凭据。\n\n
\n\n## 配置说明\n\n
\n点击展开常用配置项\n\n### AI 与运行时\n\n- `OPENAI_API_KEY` / `OPENAI_BASE_URL` / `OPENAI_MODEL_NAME`：AI 模型接入必填项。\n- `PROXY_URL`：为 AI 请求单独指定 HTTP/SOCKS5 代理。\n- `RUN_HEADLESS`：是否以无头模式运行爬虫；Docker 中应保持 `true`。\n- `SERVER_PORT`：后端监听端口，默认 `8000`。\n- `LOGIN_IS_EDGE`：本地环境可切换为 Edge 内核；Docker 镜像未内置 Edge，容器内会固定使用 Chromium。\n- `PCURL_TO_MOBILE`：是否将 PC 商品链接转换为移动端链接。\n\n### 通知\n\n- `NTFY_TOPIC_URL`\n- `GOTIFY_URL` / `GOTIFY_TOKEN`\n- `BARK_URL`\n- `WX_BOT_URL`\n- `TELEGRAM_BOT_TOKEN` / `TELEGRAM_CHAT_ID` / `TELEGRAM_API_BASE_URL`\n- `WEBHOOK_*`\n\n### 代理轮换与失败保护\n\n- `PROXY_ROTATION_ENABLED`\n- `PROXY_ROTATION_MODE`\n- `PROXY_POOL`\n- `PROXY_ROTATION_RETRY_LIMIT`\n- `PROXY_BLACKLIST_TTL`\n- `TASK_FAILURE_THRESHOLD`\n- `TASK_FAILURE_PAUSE_SECONDS`\n- `TASK_FAILURE_GUARD_PATH`\n\n完整示例见 `.env.example`。\n\n
\n\n## Web 界面认证\n\n
\n点击展开认证说明\n\n- Web UI 当前使用登录页收集账号密码，并通过 `POST /auth/status` 校验。\n- 登录成功后，前端会在浏览器本地保存登录状态，用于路由守卫和 WebSocket 初始化。\n- 默认账号密码为 `admin/admin123`，生产环境请务必修改。\n\n
\n\n## 🚀 工作流程\n\n下图描述了单个监控任务从启动到完成的核心处理逻辑。主服务运行于 `src.app`，按用户操作或定时调度启动一个或多个任务进程。\n\n```mermaid\ngraph TD\n A[启动监控任务] --> B[选择账号/代理配置];\n B --> C[任务: 搜索商品];\n C --> D{发现新商品?};\n D -- 是 --> E[抓取商品详情 & 卖家信息];\n E --> F[下载商品图片];\n F --> G[调用AI进行分析];\n G --> H{AI是否推荐?};\n H -- 是 --> I[发送通知];\n H -- 否 --> J[保存记录到 SQLite];\n I --> J;\n D -- 否 --> K[翻页/等待];\n K --> C;\n J --> C;\n C --> L{触发风控/异常?};\n L -- 是 --> M[账号/代理轮换并重试];\n M --> C;\n```\n\n## 常见问题\n\n
\n点击展开常见问题\n\n### AI 任务创建为什么不是立即完成？\n\nAI 模式会先生成分析标准，再创建任务。现在该流程已改为后台 job，提交后会显示独立进度弹窗，避免表单长时间卡住。\n\n### 区域筛选为什么默认建议留空？\n\n区域筛选会显著减少搜索结果，适合明确只看某个区域的场景。若你先验证整体市场，建议先不填。\n\n### 本地页面打开后提示前端构建产物不存在？\n\n说明根目录 `dist/` 缺失。可直接执行 `./start.sh`，或先在 `web-ui/` 里执行 `npm run build`，再确认构建产物已复制到仓库根目录。\n\n### `./start.sh` 为什么提示缺少 Playwright 或浏览器？\n\n这是脚本的前置检查。请先安装 Playwright CLI 与 Chromium，并确保系统中可用 Chrome / Edge（Linux 环境也可用 Chromium），然后重新执行 `./start.sh`。\n\n
\n\n\n\n## 致谢\n\n
\n点击展开致谢内容\n\n本项目在开发过程中参考了以下优秀项目，特此感谢：\n\n- [superboyyy/xianyu_spider](https://github.com/superboyyy/xianyu_spider)\n\n以及感谢LinuxDo相关人员的脚本贡献\n\n- [@jooooody](https://linux.do/u/jooooody/summary)\n\n以及感谢 [LinuxDo](https://linux.do/) 社区。\n\n以及感谢 ClaudeCode/Gemini/Codex 等模型工具，解放双手 体验Vibe Coding的快乐。\n\n
\n\n\n## 注意事项\n\n
\n点击展开注意事项详情\n\n- 请遵守闲鱼的用户协议和robots.txt规则，不要进行过于频繁的请求，以免对服务器造成负担或导致账号被限制。\n- 本项目仅供学习和技术研究使用，请勿用于非法用途。\n- 本项目采用 [MIT 许可证](LICENSE) 发布，按\"现状\"提供，不提供任何形式的担保。\n- 项目作者及贡献者不对因使用本软件而导致的任何直接、间接、附带或特殊的损害或损失承担责任。\n- 如需了解更多详细信息，请查看 [免责声明](DISCLAIMER.md) 文件。\n\n
\n\n## Star History\n\n[![Star History Chart](https://api.star-history.com/svg?repos=Usagi-org/ai-goofish-monitor&type=Date)](https://www.star-history.com/#Usagi-org/ai-goofish-monitor&Date)\n\n![Alt](https://repobeats.axiom.co/api/embed/b40d8a112271b4bddabadd8fe2635be3c1aa28a3.svg \"Repobeats analytics image\")\n" }, { "path": "README_EN.md", "content": "# Xianyu Intelligent Monitor Bot\n\n[中文说明](README.md)\n\nA Playwright and AI-powered multi-task real-time monitoring tool for Xianyu (闲鱼), featuring a complete web management interface.\n\n## Core Features\n\n- **Web Visual Management**: Task management, account management, AI criteria editing, run logs, results browsing\n- **AI-Driven**: Natural language task creation, multimodal model for in-depth product analysis\n- **Multi-Task Concurrency**: Independent configuration for keywords, prices, filters, and AI prompts\n- **SQLite as Primary Storage**: Tasks, results, and price history are persisted in one embedded database instead of repeatedly scanning `jsonl`\n- **Advanced Filtering**: Free shipping, new listing time range, province/city/district filtering\n- **Instant Notifications**: Supports ntfy.sh, WeChat Work (企业微信), Bark, Telegram, Webhook\n- **Scheduled Tasks**: Cron expression configuration for periodic tasks\n- **Account & Proxy Rotation**: Multi-account management, task-account binding, proxy pool rotation with failure retry\n- **Docker Deployment**: One-click containerized deployment\n\n## Screenshots\n\n![Monitoring Overview](static/img.png)\n![Task Management](static/img_1.png)\n![Result Viewer](static/img_2.png)\n![Notification Settings](static/img_3.png)\n\n## Quick Start\n\n### Requirements\n\n- Python 3.10+\n- Node.js + npm (`Node v20.18.3` has been verified to complete the frontend build)\n- Playwright CLI and Chromium. Before the first local run, install them with `python3 -m pip install playwright && python3 -m playwright install chromium`\n- Chrome or Edge on desktop systems. On Linux, Chromium also works. `start.sh` checks this prerequisite before continuing\n\n```bash\ngit clone https://github.com/Usagi-org/ai-goofish-monitor\ncd ai-goofish-monitor\ncp .env.example .env\n```\n\n### Minimum Configuration\n\n| Variable | Description | Required |\n|----------|-------------|----------|\n| `OPENAI_API_KEY` | AI model API key | Yes |\n| `OPENAI_BASE_URL` | OpenAI-compatible API base URL | Yes |\n| `OPENAI_MODEL_NAME` | Model name with image input support | Yes |\n| `WEB_USERNAME` / `WEB_PASSWORD` | Web UI login credentials, default `admin/admin123` | No |\n\nSee \"Configuration\" below for the rest.\n\n### Start Locally\n\n```bash\nchmod +x start.sh\n./start.sh\n```\n\n`start.sh` first validates the Playwright CLI and browser prerequisites. Once they are available, it installs project dependencies, builds the frontend, copies the artifacts, and starts the backend.\n\n### First-Time Setup\n\n1. Open the default Web UI at `http://127.0.0.1:8000` and sign in.\n2. Go to \"Xianyu Account Management\" and use the [Chrome Extension](https://chromewebstore.google.com/detail/xianyu-login-state-extrac/eidlpfjiodpigmfcahkmlenhppfklcoa) to export and paste the Xianyu login-state JSON.\n3. Login-state files are stored in `state/`, for example `state/acc_1.json`.\n4. Go back to \"Task Management\", create a task, bind an account if needed, and run it.\n\n### Create Your First Task\n\n- `AI mode`: fill in the requirement description. Submission opens a separate progress dialog while the criteria are generated asynchronously.\n- `Keyword mode`: provide keyword rules and the task is created immediately.\n- `Region filter`: now uses a province / city / district selector backed by an embedded Xianyu page snapshot instead of manual text input.\n\n## 🐳 Docker Deployment (Recommended)\n\n```bash\ngit clone https://github.com/Usagi-org/ai-goofish-monitor && cd ai-goofish-monitor\ncp .env.example .env\nvim .env # fill in the required values\ndocker compose up -d\ndocker compose logs -f app\ndocker compose down\n```\n\n- Default Web UI: `http://127.0.0.1:8000`\n- The published Docker image already includes Chromium, so no extra browser install is required on the host.\n- Update image: `docker compose pull && docker compose up -d`\n- If you change `SERVER_PORT` in `.env`, update the `ports` mapping in `docker-compose.yaml` as well.\n- `docker-compose.yaml` now mounts the primary SQLite database directory as `./data:/app/data`, with the default database file at `data/app.sqlite3`\n- These paths are persisted by default:\n - `data/` for the SQLite primary store (tasks, results, price history)\n - `state/` for login-state cookie files\n - `prompts/` for task prompt files\n - `logs/` for runtime logs\n - `images/` for downloaded product images and per-task temporary image folders\n - `config.json`, `jsonl/`, and `price_history/` as legacy sources for the first SQLite migration\n\n### Storage and Migration\n\n- SQLite is now the online primary storage, with the default path `data/app.sqlite3`\n- You can override the database path with `APP_DATABASE_FILE`; Docker sets it to `/app/data/app.sqlite3`\n- On startup, the app initializes the schema and tries to import existing data once from legacy `config.json`, `jsonl/`, and `price_history/`\n- `state/`, `prompts/`, `logs/`, and `images/` remain filesystem-based and are not stored in SQLite\n- Product images are temporarily downloaded to `images/task_images_/` and are normally cleaned up when the task finishes\n- After the first upgrade and after verifying the database contents in `data/app.sqlite3`, you can decide whether to keep the legacy `config.json`, `jsonl/`, and `price_history/` mounts\n\n## User Guide\n\n
\nClick to expand Web UI usage notes\n\n### Task Management\n\n- Supports AI creation, keyword rules, price range, new listing filters, region filters, account binding, and cron scheduling.\n- AI task creation runs as a background job and shows a dedicated progress dialog after submission.\n- Region filtering can greatly reduce results, so leaving it empty is the safer default.\n\n### Account Management\n\n- Import, update, and delete Xianyu login states.\n- Each task can bind a specific account or leave account selection to the system.\n\n### Results and Logs\n\n- The results page and export endpoints now query SQLite instead of directly scanning `jsonl` files.\n- The logs page is the first place to inspect login-state expiry, anti-bot issues, or AI call failures.\n\n### System Settings\n\n- View system status, edit prompts, and adjust proxy / rotation-related settings.\n\n
\n\n## Developer Guide\n\n### Local Development\n\n```bash\n# backend\npython -m src.app\n# or\nuvicorn src.app:app --host 0.0.0.0 --port 8000 --reload\n\n# frontend\ncd web-ui\nnpm install\nnpm run dev\n```\n\n- FastAPI initializes SQLite on startup and performs the one-time legacy import from `config.json/jsonl/price_history` when needed\n- `spider_v2.py` now loads tasks from SQLite by default; JSON config is only used when `--config ` is passed explicitly\n- The default local database path is `data/app.sqlite3`\n- The Vite dev server proxies `/api`, `/auth`, and `/ws` to `http://127.0.0.1:8000`.\n- `npm run build` writes `web-ui/dist/`, and `start.sh` copies it to the repository root `dist/`.\n- FastAPI serves `dist/index.html` and `dist/assets/` from the repository root.\n- `./start.sh` prints the default app URL `http://localhost:8000` and API docs URL `http://localhost:8000/docs`.\n\n### Validation\n\n```bash\nPYTEST_DISABLE_PLUGIN_AUTOLOAD=1 pytest\ncd web-ui && npm run build\n```\n\n### Task Creation API\n\n
\nClick to expand API behavior\n\n- `POST /api/tasks/generate`\n - `decision_mode=ai`: returns `202` with a `job`; the client should poll for progress.\n - `decision_mode=keyword`: returns the created task directly.\n- `GET /api/tasks/generate-jobs/{job_id}`: fetch AI task-generation progress.\n- `POST /auth/status`: validate Web UI credentials.\n\n
\n\n## Configuration\n\n
\nClick to expand common configuration items\n\n### AI and Runtime\n\n- `OPENAI_API_KEY` / `OPENAI_BASE_URL` / `OPENAI_MODEL_NAME`: required AI model settings.\n- `PROXY_URL`: dedicated HTTP/SOCKS5 proxy for AI requests.\n- `RUN_HEADLESS`: whether the scraper runs headless; keep it `true` in Docker.\n- `SERVER_PORT`: backend port, default `8000`.\n- `LOGIN_IS_EDGE`: use Edge instead of Chrome locally; Docker images do not bundle Edge and always run with Chromium.\n- `PCURL_TO_MOBILE`: convert desktop item URLs to mobile URLs.\n\n### Notifications\n\n- `NTFY_TOPIC_URL`\n- `GOTIFY_URL` / `GOTIFY_TOKEN`\n- `BARK_URL`\n- `WX_BOT_URL`\n- `TELEGRAM_BOT_TOKEN` / `TELEGRAM_CHAT_ID` / `TELEGRAM_API_BASE_URL`\n- `WEBHOOK_*`\n\n### Proxy Rotation and Failure Guard\n\n- `PROXY_ROTATION_ENABLED`\n- `PROXY_ROTATION_MODE`\n- `PROXY_POOL`\n- `PROXY_ROTATION_RETRY_LIMIT`\n- `PROXY_BLACKLIST_TTL`\n- `TASK_FAILURE_THRESHOLD`\n- `TASK_FAILURE_PAUSE_SECONDS`\n- `TASK_FAILURE_GUARD_PATH`\n\nSee `.env.example` for the full list.\n\n
\n\n## Web Authentication\n\n
\nClick to expand authentication notes\n\n- The Web UI uses a login page and validates credentials through `POST /auth/status`.\n- After login, the frontend stores local auth state for route guards and WebSocket startup.\n- The default credentials are `admin/admin123`; change them in production.\n\n
\n\n## 🚀 Workflow\n\nThe diagram below shows the core processing flow of a monitoring task. The main service runs in `src.app` and launches one or more task processes based on user actions or schedule triggers.\n\n```mermaid\ngraph TD\n A[Start Monitoring Task] --> B[Select Account/Proxy Configuration];\n B --> C[Task: Search Products];\n C --> D{Found New Products?};\n D -- Yes --> E[Scrape Product Details & Seller Info];\n E --> F[Download Product Images];\n F --> G[Call AI for Analysis];\n G --> H{AI Recommended?};\n H -- Yes --> I[Send Notification];\n H -- No --> J[Save Record to SQLite];\n I --> J;\n D -- No --> K[Next Page/Wait];\n K --> C;\n J --> C;\n C --> L{Risk Control/Exception?};\n L -- Yes --> M[Account/Proxy Rotation and Retry];\n M --> C;\n```\n\n## FAQ\n\n
\nClick to expand FAQ\n\n### Why does AI task creation take time?\n\nIn AI mode, the system generates analysis criteria before the task itself is created. This now runs as a background job with a separate progress dialog instead of blocking the task form.\n\n### Why is the region filter optional by default?\n\nRegion filtering can sharply reduce result volume. Leave it empty if you want a broader market scan first.\n\n### Why does the app say the frontend build artifacts are missing?\n\nIt means the repository root `dist/` directory is missing. Run `./start.sh`, or build the frontend in `web-ui/` and make sure the artifacts are copied to the root `dist/`.\n\n### Why does `./start.sh` complain about missing Playwright or a browser?\n\nThe script performs a prerequisite check before installing project dependencies. Install the Playwright CLI and Chromium first, then make sure Chrome, Edge, or Chromium is available on the system and rerun `./start.sh`.\n\n
\n\n## Acknowledgments\n\n
\nClick to expand acknowledgments\n\nThis project referenced the following excellent projects during development. Special thanks to:\n\n- [superboyyy/xianyu_spider](https://github.com/superboyyy/xianyu_spider)\n\nAlso thanks to LinuxDo contributors for script contributions:\n\n- [@jooooody](https://linux.do/u/jooooody/summary)\n\nAnd thanks to the [LinuxDo](https://linux.do/) community.\n\nAlso thanks to ClaudeCode/Gemini/Codex and other model tools for freeing our hands and experiencing the joy of Vibe Coding.\n\n
\n\n\n## Notices\n\n
\nClick to expand notice details\n\n- Please comply with Xianyu's user agreement and robots.txt rules. Do not make frequent requests to avoid burdening the server or having your account restricted.\n- This project is for learning and technical research purposes only. Do not use it for illegal purposes.\n- This project is released under the [MIT License](LICENSE), provided \"as is\", without any form of warranty.\n- The project authors and contributors are not responsible for any direct, indirect, incidental, or special damages or losses caused by the use of this software.\n- For more details, please refer to the [Disclaimer](DISCLAIMER.md) file.\n\n
\n\n## Star History\n\n[![Star History Chart](https://api.star-history.com/svg?repos=Usagi-org/ai-goofish-monitor&type=Date)](https://www.star-history.com/#Usagi-org/ai-goofish-monitor&Date)\n" }, { "path": "chrome-extension/README.md", "content": "# Xianyu Login State Extractor Chrome Extension\n\nThis Chrome extension helps extract complete login state information from Xianyu (Goofish) for use with the monitoring robot. It also records browser environment hints and request headers to better mimic a real session.\n\n## Installation\n\n1. Open Chrome and navigate to `chrome://extensions`\n2. Enable \"Developer mode\" in the top right corner\n3. Click \"Load unpacked\" and select the `chrome-extension` directory\n4. The extension icon should now appear in your toolbar\n\n## Usage\n\n1. Navigate to [https://www.goofish.com](https://www.goofish.com)\n2. Log in to your account\n3. Click the extension icon in the toolbar\n4. Click \"Extract Login State\" (collects cookies + environment + headers，自动过滤无用/超大字段)\n5. The complete JSON will be displayed - click \"Copy to Clipboard\"\n6. Save the JSON文本到 `xianyu_state.json`（或自定义文件名）即可\n\n## Features\n\n- Extracts all cookies including HttpOnly cookies\n- Captures browser environment (UA, locale, timezone, screen size, device memory, hardware concurrency)\n- Captures observed request headers for the current tab\n- Captures localStorage/sessionStorage snapshot for the current domain（会自动丢弃超大或无用字段）\n- Outputs a single JSON payload, ready for the monitoring robot\n- Copy to clipboard with real-time status feedback\n\n## How It Works\n\nThe extension uses the `chrome.cookies` API to access all cookies for the `.goofish.com` domain, including those with the HttpOnly flag set. This bypasses the normal JavaScript security restrictions that prevent access to these cookies.\n" }, { "path": "chrome-extension/background.js", "content": "// Service worker for capturing browser environment, headers, storage, and cookies\n\nconst GOOFISH_HOST_PATTERN = \"*://*.goofish.com/*\";\nconst MAX_STORAGE_ENTRY_LENGTH = 4096; // bytes limit to drop oversized values\n\nfunction mapSameSiteValue(chromeSameSite) {\n if (chromeSameSite === undefined || chromeSameSite === null) return \"Lax\";\n const sameSiteMap = {\n no_restriction: \"None\",\n lax: \"Lax\",\n strict: \"Strict\",\n unspecified: \"Lax\",\n };\n return sameSiteMap[chromeSameSite] || \"Lax\";\n}\n\nfunction headersArrayToObject(headers = []) {\n const result = {};\n headers.forEach((item) => {\n if (item && item.name) {\n result[item.name] = item.value || \"\";\n }\n });\n return result;\n}\n\nasync function getActiveGoofishTab() {\n const tabs = await chrome.tabs.query({ active: true, currentWindow: true });\n const tab = tabs?.[0];\n if (!tab || !tab.id || !tab.url) {\n throw new Error(\"未找到活动标签页\");\n }\n if (!tab.url.includes(\"goofish.com\")) {\n throw new Error(\"请先打开 goofish.com 页面\");\n }\n return tab;\n}\n\nasync function capturePageData(tabId) {\n const [result] = await chrome.scripting.executeScript({\n target: { tabId },\n func: () => {\n const safeEntries = (storage) => {\n try {\n const obj = {};\n for (let i = 0; i < storage.length; i += 1) {\n const key = storage.key(i);\n if (key !== null) {\n obj[key] = storage.getItem(key);\n }\n }\n return obj;\n } catch (e) {\n return {};\n }\n };\n\n const intl = (() => {\n try {\n return Intl.DateTimeFormat().resolvedOptions();\n } catch (e) {\n return {};\n }\n })();\n\n const uaData = (() => {\n try {\n return navigator.userAgentData ? navigator.userAgentData.toJSON() : null;\n } catch (e) {\n return null;\n }\n })();\n\n return {\n page: {\n pageUrl: location.href,\n referrer: document.referrer || null,\n visibilityState: document.visibilityState,\n },\n env: {\n navigator: {\n userAgent: navigator.userAgent,\n platform: navigator.platform,\n vendor: navigator.vendor,\n language: navigator.language,\n languages: navigator.languages,\n hardwareConcurrency: navigator.hardwareConcurrency,\n deviceMemory: navigator.deviceMemory,\n webdriver: navigator.webdriver,\n doNotTrack: navigator.doNotTrack,\n maxTouchPoints: navigator.maxTouchPoints,\n userAgentData: uaData,\n },\n screen: {\n width: screen.width,\n height: screen.height,\n availWidth: screen.availWidth,\n availHeight: screen.availHeight,\n colorDepth: screen.colorDepth,\n pixelDepth: screen.pixelDepth,\n devicePixelRatio: window.devicePixelRatio,\n },\n intl,\n },\n storage: {\n local: safeEntries(localStorage),\n session: safeEntries(sessionStorage),\n },\n };\n },\n });\n\n if (!result || !result.result) {\n throw new Error(\"无法获取页面信息\");\n }\n\n return result.result;\n}\n\nfunction filterEnvData(env = {}) {\n const nav = env.navigator || {};\n const screen = env.screen || {};\n const intl = env.intl || {};\n\n return {\n navigator: {\n userAgent: nav.userAgent,\n platform: nav.platform,\n language: nav.language,\n languages: nav.languages,\n hardwareConcurrency: nav.hardwareConcurrency,\n deviceMemory: nav.deviceMemory,\n maxTouchPoints: nav.maxTouchPoints,\n webdriver: nav.webdriver,\n doNotTrack: nav.doNotTrack,\n userAgentData: nav.userAgentData || null,\n },\n screen: {\n width: screen.width,\n height: screen.height,\n devicePixelRatio: screen.devicePixelRatio,\n colorDepth: screen.colorDepth,\n },\n intl: {\n timeZone: intl.timeZone,\n locale: intl.locale,\n },\n };\n}\n\nfunction pruneStorageEntries(entries = {}) {\n const data = {};\n const dropped = [];\n Object.entries(entries).forEach(([key, value]) => {\n const str = value == null ? \"\" : String(value);\n if (str.length <= MAX_STORAGE_ENTRY_LENGTH) {\n data[key] = value;\n } else {\n dropped.push(key);\n }\n });\n return { data, dropped };\n}\n\nfunction filterHeaders(rawHeaders = {}) {\n const allowList = [\n \"user-agent\",\n \"accept\",\n \"accept-language\",\n \"accept-encoding\",\n \"referer\",\n \"sec-ch-ua\",\n \"sec-ch-ua-mobile\",\n \"sec-ch-ua-platform\",\n \"sec-fetch-site\",\n \"sec-fetch-mode\",\n \"sec-fetch-dest\",\n \"sec-fetch-user\",\n \"origin\",\n \"cache-control\",\n \"pragma\",\n \"upgrade-insecure-requests\",\n \"content-type\",\n ];\n const normalized = {};\n Object.entries(rawHeaders).forEach(([k, v]) => {\n const lower = k.toLowerCase();\n if (allowList.includes(lower)) {\n normalized[k] = v;\n }\n });\n return normalized;\n}\n\nasync function captureHeaders(tabId) {\n return new Promise((resolve) => {\n let resolved = false;\n\n const cleanup = (headers) => {\n if (resolved) return;\n resolved = true;\n chrome.webRequest.onBeforeSendHeaders.removeListener(listener);\n clearTimeout(timer);\n resolve(headersArrayToObject(headers || []));\n };\n\n const listener = (details) => {\n if (details.tabId !== tabId) return;\n cleanup(details.requestHeaders || []);\n };\n\n const extraInfo = [\"requestHeaders\"];\n // extraHeaders 提供更完整的 header 视图，在新版本 Chrome 需要显式声明\n extraInfo.push(\"extraHeaders\");\n\n chrome.webRequest.onBeforeSendHeaders.addListener(\n listener,\n { urls: [GOOFISH_HOST_PATTERN], tabId },\n extraInfo,\n );\n\n const timer = setTimeout(() => cleanup(null), 2000);\n\n // 触发一次轻量请求以获取真实请求头\n chrome.scripting\n .executeScript({\n target: { tabId },\n func: () => {\n try {\n fetch(`${window.location.origin}/__codex_probe?ts=${Date.now()}`, {\n credentials: \"include\",\n cache: \"no-store\",\n redirect: \"follow\",\n }).catch(() => {});\n } catch (e) {\n /* ignore */\n }\n },\n })\n .catch(() => {\n // 如果注入失败，继续等待可能已有的请求\n });\n });\n}\n\nasync function captureCookies(url) {\n const cookies = await chrome.cookies.getAll({ url });\n return cookies.map((cookie) => ({\n name: cookie.name,\n value: cookie.value,\n domain: cookie.domain,\n path: cookie.path,\n expires: cookie.expirationDate,\n httpOnly: cookie.httpOnly,\n secure: cookie.secure,\n sameSite: mapSameSiteValue(cookie.sameSite),\n }));\n}\n\nasync function buildSnapshot() {\n const tab = await getActiveGoofishTab();\n const pageData = await capturePageData(tab.id);\n const headers = await captureHeaders(tab.id);\n const cookies = await captureCookies(new URL(tab.url).origin);\n\n const filteredEnv = filterEnvData(pageData.env);\n const localPruned = pruneStorageEntries(pageData.storage.local);\n const sessionPruned = pruneStorageEntries(pageData.storage.session);\n const filteredStorage = {\n local: localPruned.data,\n session: sessionPruned.data,\n };\n\n return {\n capturedAt: new Date().toISOString(),\n pageUrl: tab.url,\n page: pageData.page,\n env: filteredEnv,\n storage: filteredStorage,\n meta: {\n droppedStorageKeys: {\n local: localPruned.dropped,\n session: sessionPruned.dropped,\n },\n },\n headers: filterHeaders(headers),\n cookies,\n };\n}\n\nchrome.runtime.onMessage.addListener((message, sender, sendResponse) => {\n if (!message || message.type !== \"captureSnapshot\") {\n return false;\n }\n\n buildSnapshot()\n .then((data) => sendResponse({ ok: true, data }))\n .catch((error) => sendResponse({ ok: false, error: error.message }));\n\n return true;\n});\n" }, { "path": "chrome-extension/manifest.json", "content": "{\n \"manifest_version\": 3,\n \"name\": \"Xianyu Login State Extractor\",\n \"version\": \"1.1\",\n \"description\": \"Extract login state and browser environment for Xianyu monitoring robot\",\n \"permissions\": [\n \"activeTab\",\n \"cookies\",\n \"scripting\",\n \"storage\",\n \"tabs\",\n \"webRequest\"\n ],\n \"host_permissions\": [\n \"*://*.goofish.com/*\"\n ],\n \"background\": {\n \"service_worker\": \"background.js\"\n },\n \"action\": {\n \"default_popup\": \"popup.html\",\n \"default_title\": \"Extract Xianyu Login State\"\n }\n}\n" }, { "path": "chrome-extension/popup.html", "content": "\n\n\n\n \n\n\n

Xianyu Login State Extractor

\n \n
\n \n \n\n \n\n\n" }, { "path": "chrome-extension/popup.js", "content": "// Popup script for the Chrome extension\ndocument.addEventListener('DOMContentLoaded', function() {\n const extractBtn = document.getElementById('extractBtn');\n const copyBtn = document.getElementById('copyBtn');\n const stateOutput = document.getElementById('stateOutput');\n const statusDiv = document.getElementById('status');\n\n let latestSnapshot = null;\n\n function setLoading(isLoading) {\n extractBtn.disabled = isLoading;\n extractBtn.textContent = isLoading ? '采集中，请稍候...' : '1.点击获取环境+登录状态';\n }\n\n function updateStatus(message, isSuccess = false) {\n statusDiv.textContent = message;\n statusDiv.className = 'status ' + (isSuccess ? 'success' : 'error');\n setTimeout(() => {\n statusDiv.textContent = '';\n statusDiv.className = 'status';\n }, 4000);\n }\n\n function renderSnapshot(snapshot) {\n latestSnapshot = snapshot;\n stateOutput.value = JSON.stringify(snapshot, null, 2);\n }\n\n async function captureSnapshot() {\n setLoading(true);\n updateStatus('正在采集浏览器环境与登录状态...');\n stateOutput.value = '';\n\n chrome.runtime.sendMessage({ type: 'captureSnapshot' }, (response) => {\n setLoading(false);\n\n if (chrome.runtime.lastError) {\n updateStatus('通信失败: ' + chrome.runtime.lastError.message);\n return;\n }\n if (!response || !response.ok) {\n updateStatus('采集失败: ' + (response?.error || '未知错误'));\n return;\n }\n\n renderSnapshot(response.data);\n updateStatus('采集完成，已生成JSON', true);\n });\n }\n\n function copySnapshot() {\n if (!stateOutput.value) {\n updateStatus('没有可复制的数据');\n return;\n }\n navigator.clipboard.writeText(stateOutput.value)\n .then(() => updateStatus('已复制到剪贴板', true))\n .catch(err => updateStatus('复制失败: ' + err));\n }\n\n extractBtn.addEventListener('click', captureSnapshot);\n copyBtn.addEventListener('click', copySnapshot);\n});\n" }, { "path": "config.json.example", "content": "[\n {\n \"task_name\": \"苹果watch S10\",\n \"enabled\": true,\n \"keyword\": \"苹果watch S10\",\n \"description\": \"九成新，充电线包装盒齐全，无明显磕碰，卖家信用优秀\",\n \"max_pages\": 10,\n \"personal_only\": true,\n \"min_price\": \"8000\",\n \"max_price\": \"2000\",\n \"cron\": null,\n \"ai_prompt_base_file\": \"prompts/base_prompt.txt\",\n \"ai_prompt_criteria_file\": \"prompts/苹果watch_s10_criteria.txt\",\n \"account_state_file\": \"state/acc1.json\",\n \"free_shipping\": true,\n \"new_publish_option\": \"14天内\",\n \"region\": \"江苏/南京/全南京\",\n \"is_running\": false\n }\n]\n" }, { "path": "desktop_launcher.py", "content": "\"\"\"\n桌面启动入口\n使用 PyInstaller 打包后作为单一可执行文件的入口，自动启动 FastAPI 服务并打开浏览器。\n\"\"\"\nimport os\nimport sys\nimport time\nimport webbrowser\nfrom pathlib import Path\n\nimport uvicorn\n\n# PyInstaller 运行时资源目录：_MEIPASS；未打包时则为当前文件所在目录\nBASE_DIR = Path(getattr(sys, \"_MEIPASS\", Path(__file__).resolve().parent))\n\n\ndef _prepare_environment() -> None:\n \"\"\"确保工作目录和模块路径正确\"\"\"\n os.chdir(BASE_DIR)\n if str(BASE_DIR) not in sys.path:\n sys.path.insert(0, str(BASE_DIR))\n\n\ndef run_app() -> None:\n \"\"\"启动 FastAPI 应用并自动打开浏览器\"\"\"\n _prepare_environment()\n\n from src.app import app\n from src.infrastructure.config.settings import settings\n\n # 先尝试打开浏览器，稍等服务起来\n url = f\"http://127.0.0.1:{settings.server_port}\"\n webbrowser.open(url)\n time.sleep(0.5)\n\n uvicorn.run(\n app,\n host=\"127.0.0.1\",\n port=settings.server_port,\n log_level=\"info\",\n reload=False,\n )\n\n\nif __name__ == \"__main__\":\n run_app()\n" }, { "path": "docker-compose.dev.yaml", "content": "services:\n app:\n build: .\n container_name: ai-goofish-monitor-app\n init: true\n ports:\n - \"8000:8000\"\n env_file:\n - .env\n volumes:\n - .:/app\n - /app/dist\n restart: unless-stopped\n" }, { "path": "docker-compose.yaml", "content": "services:\n app:\n image: ${APP_IMAGE:-ghcr.io/usagi-org/ai-goofish:latest}\n container_name: ai-goofish-monitor-app\n pull_policy: always\n init: true\n ports:\n - \"8000:8000\"\n env_file:\n - .env\n environment:\n APP_DATABASE_FILE: /app/data/app.sqlite3\n volumes:\n - ./.env:/app/.env\n - ./data:/app/data\n - ./state:/app/state\n - ./config.json:/app/config.json\n - ./prompts:/app/prompts\n - ./jsonl:/app/jsonl\n - ./logs:/app/logs\n - ./images:/app/images\n - ./price_history:/app/price_history\n restart: unless-stopped\n" }, { "path": "pyproject.toml", "content": "[tool.pytest.ini_options]\naddopts = \"-v --tb=short\"\ntestpaths = [\"tests\"]\npython_files = [\"test_*.py\"]\npython_classes = [\"Test*\"]\npython_functions = [\"test_*\"]\nmarkers = [\n \"live: real traffic smoke tests that require real credentials and external services\",\n \"live_slow: slower optional live smoke tests such as AI task generation\",\n]\n\n[tool.coverage.run]\nsource = [\"src\"]\n\n[tool.coverage.report]\nexclude_lines = [\n \"pragma: no cover\",\n \"def __repr__\",\n \"raise AssertionError\",\n \"raise NotImplementedError\",\n \"if __name__ == .__main__.:\",\n]\n" }, { "path": "requirements-runtime.txt", "content": "python-dotenv\nplaywright\nrequests\nopenai\nfastapi\nuvicorn[standard]\npydantic-settings\njinja2\naiofiles\npython-socks\napscheduler\nhttpx[socks]\nPillow\npyzbar\nqrcode\n" }, { "path": "requirements.txt", "content": "python-dotenv\nplaywright\nrequests\nopenai\nfastapi\nuvicorn[standard]\npydantic-settings\njinja2\naiofiles\npython-socks\napscheduler\nhttpx[socks]\nPillow\npyzbar\nqrcode\npytest\npytest-asyncio\ncoverage\n" }, { "path": "run_live_smoke.sh", "content": "#!/bin/bash\n\nset -euo pipefail\n\nRED='\\033[0;31m'\nGREEN='\\033[0;32m'\nYELLOW='\\033[1;33m'\nNC='\\033[0m'\n\nSCRIPT_DIR=\"$(cd \"$(dirname \"${BASH_SOURCE[0]}\")\" && pwd)\"\ncd \"$SCRIPT_DIR\"\n\nPYTHON_CMD=\"${PYTHON_CMD:-python3}\"\nMARK_EXPRESSION=\"\"\nDRY_RUN=false\nWITH_GENERATION=true\nPYTEST_ARGS=()\nTASK_CREATE_TEST=\"tests/integration/test_api_tasks.py::test_create_list_update_delete_task\"\nTEST_TARGETS=(\n \"$TASK_CREATE_TEST\"\n \"tests/live\"\n)\n\nusage() {\n cat <<'EOF'\n用法:\n ./run_live_smoke.sh [选项] [-- pytest额外参数]\n\n选项:\n --keyword <关键词> 覆盖 LIVE_TEST_KEYWORD\n --account-file <路径> 覆盖 LIVE_TEST_ACCOUNT_STATE_FILE\n --task-name <名称> 覆盖 LIVE_TEST_TASK_NAME\n --timeout <秒> 覆盖 LIVE_TIMEOUT_SECONDS\n --min-items <数量> 覆盖 LIVE_EXPECT_MIN_ITEMS\n --debug-limit <数量> 覆盖 LIVE_TEST_DEBUG_LIMIT（默认 1，仅分析前 N 个新商品）\n --with-generation 显式开启 live_slow（默认已开启）\n --without-generation 关闭 live_slow，只执行主 smoke\n --dry-run 只打印配置和将执行的命令，不真正运行\n --help 显示帮助\n\n示例:\n ./run_live_smoke.sh\n ./run_live_smoke.sh --keyword \"MacBook Air M1\" --min-items 2\n ./run_live_smoke.sh --without-generation\n ./run_live_smoke.sh -- -k live_real_traffic\n\n说明:\n 0. 默认先执行任务创建 CRUD 集成测试，再执行 tests/live 真实流量 smoke\n 1. 脚本会自动设置 RUN_LIVE_TESTS=1\n 2. 若未设置 LIVE_TEST_ACCOUNT_STATE_FILE，会自动尝试使用 state/ 下第一个 *.json\n 3. 默认使用 PYTEST_DISABLE_PLUGIN_AUTOLOAD=1，避免本机第三方 pytest 插件干扰\n 4. 默认设置 LIVE_TEST_DEBUG_LIMIT=1，使冒烟测试只抓取并分析 1 个新商品\nEOF\n}\n\nrequire_value() {\n local option=\"$1\"\n local value=\"${2:-}\"\n if [[ -z \"$value\" ]]; then\n echo -e \"${RED}错误:${NC} ${option} 需要一个值\"\n exit 1\n fi\n}\n\nresolve_default_account_file() {\n local first_match=\"\"\n while IFS= read -r file; do\n first_match=\"$file\"\n break\n done < <(find \"$SCRIPT_DIR/state\" -maxdepth 1 -type f -name '*.json' | sort)\n printf '%s' \"$first_match\"\n}\n\nwhile [[ $# -gt 0 ]]; do\n case \"$1\" in\n --keyword)\n require_value \"$1\" \"${2:-}\"\n export LIVE_TEST_KEYWORD=\"$2\"\n shift 2\n ;;\n --account-file)\n require_value \"$1\" \"${2:-}\"\n export LIVE_TEST_ACCOUNT_STATE_FILE=\"$2\"\n shift 2\n ;;\n --task-name)\n require_value \"$1\" \"${2:-}\"\n export LIVE_TEST_TASK_NAME=\"$2\"\n shift 2\n ;;\n --timeout)\n require_value \"$1\" \"${2:-}\"\n export LIVE_TIMEOUT_SECONDS=\"$2\"\n shift 2\n ;;\n --min-items)\n require_value \"$1\" \"${2:-}\"\n export LIVE_EXPECT_MIN_ITEMS=\"$2\"\n shift 2\n ;;\n --debug-limit)\n require_value \"$1\" \"${2:-}\"\n export LIVE_TEST_DEBUG_LIMIT=\"$2\"\n shift 2\n ;;\n --with-generation)\n WITH_GENERATION=true\n shift\n ;;\n --without-generation)\n WITH_GENERATION=false\n shift\n ;;\n --dry-run)\n DRY_RUN=true\n shift\n ;;\n --help|-h)\n usage\n exit 0\n ;;\n --)\n shift\n PYTEST_ARGS+=(\"$@\")\n break\n ;;\n *)\n PYTEST_ARGS+=(\"$1\")\n shift\n ;;\n esac\ndone\n\nif ! command -v \"$PYTHON_CMD\" >/dev/null 2>&1; then\n echo -e \"${RED}错误:${NC} 未找到 Python 命令: $PYTHON_CMD\"\n exit 1\nfi\n\nif ! \"$PYTHON_CMD\" -m pytest --version >/dev/null 2>&1; then\n echo -e \"${RED}错误:${NC} 当前 Python 环境缺少 pytest\"\n exit 1\nfi\n\nif ! \"$PYTHON_CMD\" -m playwright --version >/dev/null 2>&1; then\n echo -e \"${RED}错误:${NC} 当前 Python 环境缺少 Playwright，请先安装浏览器依赖\"\n exit 1\nfi\n\nexport RUN_LIVE_TESTS=1\nexport PYTEST_DISABLE_PLUGIN_AUTOLOAD=\"${PYTEST_DISABLE_PLUGIN_AUTOLOAD:-1}\"\nexport LIVE_TEST_KEYWORD=\"${LIVE_TEST_KEYWORD:-MacBook Pro M2}\"\nexport LIVE_TEST_TASK_NAME=\"${LIVE_TEST_TASK_NAME:-Live Smoke Task}\"\nexport LIVE_EXPECT_MIN_ITEMS=\"${LIVE_EXPECT_MIN_ITEMS:-1}\"\nexport LIVE_TEST_DEBUG_LIMIT=\"${LIVE_TEST_DEBUG_LIMIT:-1}\"\nexport LIVE_TIMEOUT_SECONDS=\"${LIVE_TIMEOUT_SECONDS:-180}\"\n\nif [[ -z \"${LIVE_TEST_ACCOUNT_STATE_FILE:-}\" ]]; then\n DEFAULT_ACCOUNT_FILE=\"$(resolve_default_account_file)\"\n if [[ -n \"$DEFAULT_ACCOUNT_FILE\" ]]; then\n export LIVE_TEST_ACCOUNT_STATE_FILE=\"$DEFAULT_ACCOUNT_FILE\"\n fi\nfi\n\nif [[ -z \"${LIVE_TEST_ACCOUNT_STATE_FILE:-}\" ]]; then\n echo -e \"${RED}错误:${NC} 未找到 live 登录态文件。请使用 --account-file 指定，或在 state/ 下放置 *.json\"\n exit 1\nfi\n\nif [[ ! -f \"${LIVE_TEST_ACCOUNT_STATE_FILE}\" ]]; then\n echo -e \"${RED}错误:${NC} 登录态文件不存在: ${LIVE_TEST_ACCOUNT_STATE_FILE}\"\n exit 1\nfi\n\nif [[ \"$WITH_GENERATION\" == \"true\" ]]; then\n export LIVE_ENABLE_TASK_GENERATION=1\n MARK_EXPRESSION=\"\"\nelse\n export LIVE_ENABLE_TASK_GENERATION=0\n MARK_EXPRESSION=\"not live_slow\"\nfi\n\necho -e \"${GREEN}========================================${NC}\"\necho -e \"${GREEN}闲鱼真实流量 Live Smoke 一键测试${NC}\"\necho -e \"${GREEN}========================================${NC}\"\necho -e \"${YELLOW}Python:${NC} $PYTHON_CMD\"\necho -e \"${YELLOW}关键词:${NC} ${LIVE_TEST_KEYWORD}\"\necho -e \"${YELLOW}任务名:${NC} ${LIVE_TEST_TASK_NAME}\"\necho -e \"${YELLOW}登录态:${NC} ${LIVE_TEST_ACCOUNT_STATE_FILE}\"\necho -e \"${YELLOW}最少结果数:${NC} ${LIVE_EXPECT_MIN_ITEMS}\"\necho -e \"${YELLOW}抓取/分析商品上限:${NC} ${LIVE_TEST_DEBUG_LIMIT}\"\necho -e \"${YELLOW}超时(秒):${NC} ${LIVE_TIMEOUT_SECONDS}\"\necho -e \"${YELLOW}任务生成慢用例:${NC} ${LIVE_ENABLE_TASK_GENERATION}\"\necho -e \"${YELLOW}任务创建前置用例:${NC} ${TASK_CREATE_TEST}\"\nif [[ -n \"$MARK_EXPRESSION\" ]]; then\n echo -e \"${YELLOW}Pytest Marker:${NC} ${MARK_EXPRESSION}\"\nelse\n echo -e \"${YELLOW}Pytest Marker:${NC} \"\nfi\necho -e \"${YELLOW}禁用插件自动加载:${NC} ${PYTEST_DISABLE_PLUGIN_AUTOLOAD}\"\n\nCMD=(\n \"$PYTHON_CMD\" -m pytest\n \"${TEST_TARGETS[@]}\"\n -v\n)\n\nif [[ -n \"$MARK_EXPRESSION\" ]]; then\n CMD+=(-m \"$MARK_EXPRESSION\")\nfi\n\nif [[ ${#PYTEST_ARGS[@]} -gt 0 ]]; then\n CMD+=(\"${PYTEST_ARGS[@]}\")\nfi\n\necho -e \"${YELLOW}执行命令:${NC} ${CMD[*]}\"\n\nif [[ \"$DRY_RUN\" == \"true\" ]]; then\n echo -e \"${GREEN}Dry run 完成，未实际执行测试。${NC}\"\n exit 0\nfi\n\n\"${CMD[@]}\"\n" }, { "path": "spider_v2.py", "content": "import asyncio\nimport sys\nimport os\nimport argparse\nimport json\nimport signal\nimport contextlib\nimport re\n\nfrom src.config import STATE_FILE\nfrom src.infrastructure.persistence.sqlite_task_repository import SqliteTaskRepository\nfrom src.scraper import scrape_xianyu\n\n\nasync def main():\n parser = argparse.ArgumentParser(\n description=\"闲鱼商品监控脚本，支持多任务配置和实时AI分析。\",\n epilog=\"\"\"\n使用示例:\n # 运行 config.json 中定义的所有任务\n python spider_v2.py\n\n # 只运行名为 \"Sony A7M4\" 的任务 (通常由调度器调用)\n python spider_v2.py --task-name \"Sony A7M4\"\n\n # 调试模式: 运行所有任务，但每个任务只处理前3个新发现的商品\n python spider_v2.py --debug-limit 3\n\"\"\",\n formatter_class=argparse.RawDescriptionHelpFormatter\n )\n parser.add_argument(\"--debug-limit\", type=int, default=0, help=\"调试模式：每个任务仅处理前 N 个新商品（0 表示无限制）\")\n parser.add_argument(\"--config\", type=str, help=\"指定任务配置文件路径（传入时优先读取 JSON）\")\n parser.add_argument(\"--task-name\", type=str, help=\"只运行指定名称的单个任务 (用于定时任务调度)\")\n args = parser.parse_args()\n\n if args.config:\n if not os.path.exists(args.config):\n sys.exit(f\"错误: 配置文件 '{args.config}' 不存在。\")\n try:\n with open(args.config, 'r', encoding='utf-8') as f:\n tasks_config = json.load(f)\n except (json.JSONDecodeError, IOError) as e:\n sys.exit(f\"错误: 读取或解析配置文件 '{args.config}' 失败: {e}\")\n else:\n repository = SqliteTaskRepository()\n tasks = await repository.find_all()\n tasks_config = [task.dict() for task in tasks]\n\n def normalize_keywords(value):\n if value is None:\n return []\n if isinstance(value, str):\n raw_values = re.split(r\"[\\n,]+\", value)\n elif isinstance(value, (list, tuple, set)):\n raw_values = list(value)\n else:\n raw_values = [value]\n\n normalized = []\n seen = set()\n for item in raw_values:\n text = str(item).strip()\n if not text:\n continue\n key = text.lower()\n if key in seen:\n continue\n seen.add(key)\n normalized.append(text)\n return normalized\n\n def flatten_legacy_groups(groups):\n merged = []\n for group in groups or []:\n if isinstance(group, dict):\n merged.extend(normalize_keywords(group.get(\"include_keywords\")))\n return normalize_keywords(merged)\n\n def has_bound_account(tasks: list) -> bool:\n for task in tasks:\n account = task.get(\"account_state_file\")\n if isinstance(account, str) and account.strip():\n return True\n return False\n\n def has_any_state_file() -> bool:\n state_dir = os.getenv(\"ACCOUNT_STATE_DIR\", \"state\").strip().strip('\"').strip(\"'\")\n if os.path.isdir(state_dir):\n for name in os.listdir(state_dir):\n if name.endswith(\".json\"):\n return True\n return False\n\n if not os.path.exists(STATE_FILE) and not has_bound_account(tasks_config) and not has_any_state_file():\n sys.exit(\n f\"错误: 未找到登录状态文件。请在 state/ 中添加账号或配置 account_state_file。\"\n )\n\n # 读取所有prompt文件内容（关键词模式不需要加载prompt）\n for task in tasks_config:\n decision_mode = str(task.get(\"decision_mode\", \"ai\")).strip().lower()\n if decision_mode not in {\"ai\", \"keyword\"}:\n decision_mode = \"ai\"\n task[\"decision_mode\"] = decision_mode\n keyword_rules = task.get(\"keyword_rules\")\n if keyword_rules is None and task.get(\"keyword_rule_groups\") is not None:\n task[\"keyword_rules\"] = flatten_legacy_groups(task.get(\"keyword_rule_groups\") or [])\n else:\n task[\"keyword_rules\"] = normalize_keywords(keyword_rules)\n\n if decision_mode == \"keyword\":\n task[\"ai_prompt_text\"] = \"\"\n continue\n\n if task.get(\"enabled\", False) and task.get(\"ai_prompt_base_file\") and task.get(\"ai_prompt_criteria_file\"):\n try:\n with open(task[\"ai_prompt_base_file\"], 'r', encoding='utf-8') as f_base:\n base_prompt = f_base.read()\n with open(task[\"ai_prompt_criteria_file\"], 'r', encoding='utf-8') as f_criteria:\n criteria_text = f_criteria.read()\n \n # 动态组合成最终的Prompt\n task['ai_prompt_text'] = base_prompt.replace(\"{{CRITERIA_SECTION}}\", criteria_text)\n \n # 验证生成的prompt是否有效\n if len(task['ai_prompt_text']) < 100:\n print(f\"警告: 任务 '{task['task_name']}' 生成的prompt过短 ({len(task['ai_prompt_text'])} 字符)，可能存在问题。\")\n elif \"{{CRITERIA_SECTION}}\" in task['ai_prompt_text']:\n print(f\"警告: 任务 '{task['task_name']}' 的prompt中仍包含占位符，替换可能失败。\")\n else:\n print(f\"✅ 任务 '{task['task_name']}' 的prompt生成成功，长度: {len(task['ai_prompt_text'])} 字符\")\n\n except FileNotFoundError as e:\n print(f\"警告: 任务 '{task['task_name']}' 的prompt文件缺失: {e}，该任务的AI分析将被跳过。\")\n task['ai_prompt_text'] = \"\"\n except Exception as e:\n print(f\"错误: 任务 '{task['task_name']}' 处理prompt文件时发生异常: {e}，该任务的AI分析将被跳过。\")\n task['ai_prompt_text'] = \"\"\n elif task.get(\"enabled\", False) and task.get(\"ai_prompt_file\"):\n try:\n with open(task[\"ai_prompt_file\"], 'r', encoding='utf-8') as f:\n task['ai_prompt_text'] = f.read()\n print(f\"✅ 任务 '{task['task_name']}' 的prompt文件读取成功，长度: {len(task['ai_prompt_text'])} 字符\")\n except FileNotFoundError:\n print(f\"警告: 任务 '{task['task_name']}' 的prompt文件 '{task['ai_prompt_file']}' 未找到，该任务的AI分析将被跳过。\")\n task['ai_prompt_text'] = \"\"\n except Exception as e:\n print(f\"错误: 任务 '{task['task_name']}' 读取prompt文件时发生异常: {e}，该任务的AI分析将被跳过。\")\n task['ai_prompt_text'] = \"\"\n\n print(\"\\n--- 开始执行监控任务 ---\")\n if args.debug_limit > 0:\n print(f\"** 调试模式已激活，每个任务最多处理 {args.debug_limit} 个新商品 **\")\n \n if args.task_name:\n print(f\"** 定时任务模式：只执行任务 '{args.task_name}' **\")\n\n print(\"--------------------\")\n\n active_task_configs = []\n if args.task_name:\n # 如果指定了任务名称，只查找该任务\n task_found = next((task for task in tasks_config if task.get('task_name') == args.task_name), None)\n if task_found:\n if task_found.get(\"enabled\", False):\n active_task_configs.append(task_found)\n else:\n print(f\"任务 '{args.task_name}' 已被禁用，跳过执行。\")\n else:\n print(f\"错误：在配置文件中未找到名为 '{args.task_name}' 的任务。\")\n return\n else:\n # 否则，按原计划加载所有启用的任务\n active_task_configs = [task for task in tasks_config if task.get(\"enabled\", False)]\n\n if not active_task_configs:\n print(\"没有需要执行的任务，程序退出。\")\n return\n\n # 为每个启用的任务创建一个异步执行协程\n stop_event = asyncio.Event()\n loop = asyncio.get_running_loop()\n for sig in (signal.SIGTERM, signal.SIGINT):\n try:\n loop.add_signal_handler(sig, stop_event.set)\n except NotImplementedError:\n pass\n\n tasks = []\n for task_conf in active_task_configs:\n print(f\"-> 任务 '{task_conf['task_name']}' 已加入执行队列。\")\n tasks.append(asyncio.create_task(scrape_xianyu(task_config=task_conf, debug_limit=args.debug_limit)))\n\n async def _shutdown_watcher():\n await stop_event.wait()\n print(\"\\n收到终止信号，正在优雅退出，取消所有爬虫任务...\")\n for t in tasks:\n if not t.done():\n t.cancel()\n\n shutdown_task = asyncio.create_task(_shutdown_watcher())\n\n try:\n # 并发执行所有任务\n results = await asyncio.gather(*tasks, return_exceptions=True)\n finally:\n shutdown_task.cancel()\n with contextlib.suppress(asyncio.CancelledError):\n await shutdown_task\n\n print(\"\\n--- 所有任务执行完毕 ---\")\n for i, result in enumerate(results):\n task_name = active_task_configs[i]['task_name']\n if isinstance(result, Exception):\n print(f\"任务 '{task_name}' 因异常而终止: {result}\")\n else:\n print(f\"任务 '{task_name}' 正常结束，本次运行共处理了 {result} 个新商品。\")\n\nif __name__ == \"__main__\":\n asyncio.run(main())\n" }, { "path": "src/__init__.py", "content": "# This file makes src a Python package\n" }, { "path": "src/ai_handler.py", "content": "import asyncio\nimport base64\nimport json\nimport os\nimport re\nimport sys\nimport shutil\nimport traceback\nfrom datetime import datetime, timedelta\nfrom urllib.parse import urlencode, urlparse, urlunparse, parse_qsl\n\nimport requests\n\n# 设置标准输出编码为UTF-8，解决Windows控制台编码问题\nif sys.platform.startswith('win'):\n import codecs\n sys.stdout = codecs.getwriter('utf-8')(sys.stdout.detach())\n sys.stderr = codecs.getwriter('utf-8')(sys.stderr.detach())\n\nfrom src.config import (\n AI_DEBUG_MODE,\n IMAGE_DOWNLOAD_HEADERS,\n IMAGE_SAVE_DIR,\n TASK_IMAGE_DIR_PREFIX,\n MODEL_NAME,\n ENABLE_RESPONSE_FORMAT,\n client,\n)\nfrom src.ai_message_builder import (\n build_analysis_text_prompt,\n build_user_message_content,\n)\nfrom src.services.ai_response_parser import (\n EmptyAIResponseError,\n extract_ai_response_content,\n parse_ai_response_json,\n)\nfrom src.services.ai_request_compat import (\n CHAT_COMPLETIONS_API_MODE,\n RESPONSES_API_MODE,\n build_ai_request_params,\n create_ai_response_async,\n is_chat_completions_api_unsupported_error,\n is_json_output_unsupported_error,\n is_responses_api_unsupported_error,\n is_temperature_unsupported_error,\n remove_temperature_param,\n)\nfrom src.services.notification_service import build_notification_service\nfrom src.utils import convert_goofish_link, retry_on_failure\n\n\ndef _positive_int(value, default: int) -> int:\n try:\n return max(1, int(value))\n except (TypeError, ValueError):\n return default\n\n\nDEFAULT_IMAGE_DOWNLOAD_CONCURRENCY = max(\n 1,\n _positive_int(os.getenv(\"IMAGE_DOWNLOAD_CONCURRENCY\", \"3\"), 3),\n)\n\n\ndef safe_print(text):\n \"\"\"安全的打印函数，处理编码错误\"\"\"\n try:\n print(text)\n except UnicodeEncodeError:\n # 如果遇到编码错误，尝试用ASCII编码并忽略无法编码的字符\n try:\n print(text.encode('ascii', errors='ignore').decode('ascii'))\n except:\n # 如果还是失败，打印一个简化的消息\n print(\"[输出包含无法显示的字符]\")\n\n\ndef _build_debug_request_summary(api_mode: str, request_params: dict) -> dict:\n summary = {\n \"api_mode\": api_mode,\n \"model\": request_params.get(\"model\"),\n }\n if \"temperature\" in request_params:\n summary[\"temperature\"] = request_params[\"temperature\"]\n if \"max_output_tokens\" in request_params:\n summary[\"max_output_tokens\"] = request_params[\"max_output_tokens\"]\n if \"max_tokens\" in request_params:\n summary[\"max_tokens\"] = request_params[\"max_tokens\"]\n if \"text\" in request_params:\n summary[\"text\"] = request_params[\"text\"]\n if \"response_format\" in request_params:\n summary[\"response_format\"] = request_params[\"response_format\"]\n if \"input\" in request_params:\n summary[\"input_content_types\"] = [\n [item.get(\"type\") for item in message.get(\"content\", [])]\n for message in request_params[\"input\"]\n ]\n if \"messages\" in request_params:\n summary[\"message_content_types\"] = [\n _extract_message_content_types(message)\n for message in request_params[\"messages\"]\n ]\n return summary\n\n\ndef _extract_message_content_types(message: dict) -> list[str]:\n content = message.get(\"content\")\n if isinstance(content, str):\n return [\"text\"]\n if not isinstance(content, list):\n return [type(content).__name__]\n return [str(item.get(\"type\")) for item in content if isinstance(item, dict)]\n\n\n@retry_on_failure(retries=2, delay=3)\nasync def _download_single_image(url, save_path):\n \"\"\"一个带重试的内部函数，用于异步下载单个图片。\"\"\"\n loop = asyncio.get_running_loop()\n # 使用 run_in_executor 运行同步的 requests 代码，避免阻塞事件循环\n response = await loop.run_in_executor(\n None,\n lambda: requests.get(url, headers=IMAGE_DOWNLOAD_HEADERS, timeout=20, stream=True)\n )\n response.raise_for_status()\n with open(save_path, 'wb') as f:\n for chunk in response.iter_content(chunk_size=8192):\n f.write(chunk)\n return save_path\n\n\ndef _build_image_save_path(\n product_id: str,\n index: int,\n url: str,\n task_image_dir: str,\n) -> str:\n clean_url = url.split('.heic')[0] if '.heic' in url else url\n file_name_base = os.path.basename(clean_url).split('?')[0]\n file_name = f\"product_{product_id}_{index}_{file_name_base}\"\n file_name = re.sub(r'[\\\\/*?:\"<>|]', \"\", file_name)\n if not os.path.splitext(file_name)[1]:\n file_name += \".jpg\"\n return os.path.join(task_image_dir, file_name)\n\n\nasync def download_all_images(product_id, image_urls, task_name=\"default\", concurrency=None):\n \"\"\"异步下载一个商品的所有图片。如果图片已存在则跳过。支持任务隔离。\"\"\"\n if not image_urls:\n return []\n\n # 为每个任务创建独立的图片目录\n task_image_dir = os.path.join(IMAGE_SAVE_DIR, f\"{TASK_IMAGE_DIR_PREFIX}{task_name}\")\n os.makedirs(task_image_dir, exist_ok=True)\n\n urls = [url.strip() for url in image_urls if url.strip().startswith('http')]\n if not urls:\n return []\n\n max_concurrency = _positive_int(concurrency, DEFAULT_IMAGE_DOWNLOAD_CONCURRENCY)\n semaphore = asyncio.Semaphore(max_concurrency)\n total_images = len(urls)\n\n async def _download_one(index: int, url: str):\n save_path = _build_image_save_path(product_id, index, url, task_image_dir)\n if os.path.exists(save_path):\n safe_print(\n f\" [图片] 图片 {index}/{total_images} 已存在，跳过下载: {os.path.basename(save_path)}\"\n )\n return save_path\n async with semaphore:\n safe_print(f\" [图片] 正在下载图片 {index}/{total_images}: {url}\")\n if await _download_single_image(url, save_path):\n safe_print(\n f\" [图片] 图片 {index}/{total_images} 已成功下载到: {os.path.basename(save_path)}\"\n )\n return save_path\n return None\n\n tasks = [\n asyncio.create_task(_download_one(index, url))\n for index, url in enumerate(urls, start=1)\n ]\n results = await asyncio.gather(*tasks, return_exceptions=True)\n\n saved_paths = []\n for url, result in zip(urls, results):\n try:\n if isinstance(result, Exception):\n raise result\n if result:\n saved_paths.append(result)\n except Exception as e:\n safe_print(f\" [图片] 处理图片 {url} 时发生错误，已跳过此图: {e}\")\n\n return saved_paths\n\n\ndef cleanup_task_images(task_name):\n \"\"\"清理指定任务的图片目录\"\"\"\n task_image_dir = os.path.join(IMAGE_SAVE_DIR, f\"{TASK_IMAGE_DIR_PREFIX}{task_name}\")\n if os.path.exists(task_image_dir):\n try:\n shutil.rmtree(task_image_dir)\n safe_print(f\" [清理] 已删除任务 '{task_name}' 的临时图片目录: {task_image_dir}\")\n except Exception as e:\n safe_print(f\" [清理] 删除任务 '{task_name}' 的临时图片目录时出错: {e}\")\n else:\n safe_print(f\" [清理] 任务 '{task_name}' 的临时图片目录不存在: {task_image_dir}\")\n\n\ndef cleanup_ai_logs(logs_dir: str, keep_days: int = 1) -> None:\n try:\n cutoff = datetime.now() - timedelta(days=keep_days)\n for filename in os.listdir(logs_dir):\n if not filename.endswith(\".log\"):\n continue\n try:\n timestamp = datetime.strptime(filename[:15], \"%Y%m%d_%H%M%S\")\n except ValueError:\n continue\n if timestamp < cutoff:\n os.remove(os.path.join(logs_dir, filename))\n except Exception as e:\n safe_print(f\" [日志] 清理AI日志时出错: {e}\")\n\n\ndef encode_image_to_base64(image_path):\n \"\"\"将本地图片文件编码为 Base64 字符串。\"\"\"\n if not image_path or not os.path.exists(image_path):\n return None\n try:\n with open(image_path, \"rb\") as image_file:\n return base64.b64encode(image_file.read()).decode('utf-8')\n except Exception as e:\n safe_print(f\"编码图片时出错: {e}\")\n return None\n\n\ndef validate_ai_response_format(parsed_response):\n \"\"\"验证AI响应的格式是否符合预期结构\"\"\"\n required_fields = [\n \"prompt_version\",\n \"is_recommended\",\n \"reason\",\n \"risk_tags\",\n \"criteria_analysis\"\n ]\n\n # 检查顶层字段\n for field in required_fields:\n if field not in parsed_response:\n safe_print(f\" [AI分析] 警告：响应缺少必需字段 '{field}'\")\n return False\n\n # 检查criteria_analysis是否为字典且不为空\n criteria_analysis = parsed_response.get(\"criteria_analysis\", {})\n if not isinstance(criteria_analysis, dict) or not criteria_analysis:\n safe_print(\" [AI分析] 警告：criteria_analysis必须是非空字典\")\n return False\n\n # 检查seller_type字段（所有商品都需要）\n if \"seller_type\" not in criteria_analysis:\n safe_print(\" [AI分析] 警告：criteria_analysis缺少必需字段 'seller_type'\")\n return False\n\n # 检查数据类型\n if not isinstance(parsed_response.get(\"is_recommended\"), bool):\n safe_print(\" [AI分析] 警告：is_recommended字段不是布尔类型\")\n return False\n\n if not isinstance(parsed_response.get(\"risk_tags\"), list):\n safe_print(\" [AI分析] 警告：risk_tags字段不是列表类型\")\n return False\n\n return True\n\n\n@retry_on_failure(retries=3, delay=5)\nasync def send_ntfy_notification(product_data, reason):\n \"\"\"兼容旧调用名，内部统一走 NotificationService。\"\"\"\n service = build_notification_service()\n if not service.clients:\n safe_print(\n \"警告：未在 .env 文件中配置任何通知服务，跳过通知。\"\n )\n return {}\n\n results = await service.send_notification(product_data, reason)\n for channel, result in results.items():\n if result[\"success\"]:\n safe_print(f\" -> {channel} 通知发送成功。\")\n continue\n safe_print(f\" -> {channel} 通知发送失败: {result['message']}\")\n return results\n\n\nasync def get_ai_analysis(product_data, image_paths=None, prompt_text=\"\"):\n \"\"\"将完整的商品JSON数据和所有图片发送给 AI 进行分析（异步）。\"\"\"\n if not client:\n safe_print(\" [AI分析] 错误：AI客户端未初始化，跳过分析。\")\n return None\n\n item_info = product_data.get('商品信息', {})\n product_id = item_info.get('商品ID', 'N/A')\n\n safe_print(f\"\\n [AI分析] 开始分析商品 #{product_id} (含 {len(image_paths or [])} 张图片)...\")\n safe_print(f\" [AI分析] 标题: {item_info.get('商品标题', '无')}\")\n\n if not prompt_text:\n safe_print(\" [AI分析] 错误：未提供AI分析所需的prompt文本。\")\n return None\n\n product_details_json = json.dumps(product_data, ensure_ascii=False, indent=2)\n system_prompt = prompt_text\n\n if AI_DEBUG_MODE:\n safe_print(\"\\n--- [AI DEBUG] ---\")\n safe_print(\"--- PRODUCT DATA (JSON) ---\")\n safe_print(product_details_json)\n safe_print(\"--- PROMPT TEXT (完整内容) ---\")\n safe_print(prompt_text)\n safe_print(\"-------------------\\n\")\n\n image_data_urls = []\n if image_paths:\n for path in image_paths:\n base64_image = encode_image_to_base64(path)\n if base64_image:\n image_data_urls.append(f\"data:image/jpeg;base64,{base64_image}\")\n\n combined_text_prompt = build_analysis_text_prompt(\n product_details_json,\n system_prompt,\n include_images=bool(image_data_urls),\n )\n user_content = build_user_message_content(combined_text_prompt, image_data_urls)\n messages = [{\"role\": \"user\", \"content\": user_content}]\n\n # 保存最终传输内容到日志文件\n try:\n # 创建logs文件夹\n logs_dir = os.path.join(\"logs\", \"ai\")\n os.makedirs(logs_dir, exist_ok=True)\n cleanup_ai_logs(logs_dir, keep_days=1)\n\n # 生成日志文件名（当前时间）\n current_time = datetime.now().strftime(\"%Y%m%d_%H%M%S\")\n log_filename = f\"{current_time}.log\"\n log_filepath = os.path.join(logs_dir, log_filename)\n\n task_name = product_data.get(\"任务名称\") or product_data.get(\"任务名\") or \"unknown\"\n log_payload = {\n \"timestamp\": current_time,\n \"task_name\": task_name,\n \"product_id\": product_id,\n \"title\": item_info.get(\"商品标题\", \"无\"),\n \"image_count\": len(image_data_urls),\n }\n log_content = json.dumps(log_payload, ensure_ascii=False)\n\n # 写入日志文件\n with open(log_filepath, 'w', encoding='utf-8') as f:\n f.write(log_content)\n\n safe_print(f\" [日志] AI分析请求已保存到: {log_filepath}\")\n\n except Exception as e:\n safe_print(f\" [日志] 保存AI分析日志时出错: {e}\")\n\n # 增强的AI调用，包含更严格的结构化输出控制和重试机制\n max_retries = 4\n api_mode = CHAT_COMPLETIONS_API_MODE\n use_response_format = ENABLE_RESPONSE_FORMAT\n use_temperature = True\n for attempt in range(max_retries):\n try:\n # 根据重试次数调整参数\n current_temperature = 0.1 if attempt == 0 else 0.05 # 重试时使用更低的温度\n\n from src.config import get_ai_request_params\n\n request_params = build_ai_request_params(\n api_mode,\n model=MODEL_NAME,\n messages=messages,\n temperature=current_temperature,\n max_output_tokens=4000,\n enable_json_output=use_response_format,\n )\n if not use_temperature:\n request_params = remove_temperature_param(request_params)\n\n request_params = get_ai_request_params(**request_params)\n\n if AI_DEBUG_MODE:\n safe_print(f\"\\n--- [AI DEBUG] 第{attempt + 1}次尝试 REQUEST ---\")\n safe_print(\n json.dumps(\n _build_debug_request_summary(api_mode, request_params),\n ensure_ascii=False,\n indent=2,\n )\n )\n safe_print(\"-----------------------------------\\n\")\n\n response = await create_ai_response_async(\n client,\n api_mode,\n request_params,\n )\n ai_response_content = extract_ai_response_content(response)\n\n if AI_DEBUG_MODE:\n safe_print(f\"\\n--- [AI DEBUG] 第{attempt + 1}次尝试 ---\")\n safe_print(\"--- RAW AI RESPONSE ---\")\n safe_print(ai_response_content)\n safe_print(\"---------------------\\n\")\n\n try:\n parsed_response = parse_ai_response_json(ai_response_content)\n\n # 验证响应格式\n if validate_ai_response_format(parsed_response):\n safe_print(f\" [AI分析] 第{attempt + 1}次尝试成功，响应格式验证通过\")\n return parsed_response\n safe_print(f\" [AI分析] 第{attempt + 1}次尝试格式验证失败\")\n if attempt < max_retries - 1:\n safe_print(f\" [AI分析] 准备第{attempt + 2}次重试...\")\n continue\n raise ValueError(\"AI响应格式缺少必需字段或字段类型不正确。\")\n except json.JSONDecodeError as e:\n safe_print(f\" [AI分析] 第{attempt + 1}次尝试JSON解析失败: {e}\")\n if attempt < max_retries - 1:\n safe_print(f\" [AI分析] 准备第{attempt + 2}次重试...\")\n continue\n raise e\n except EmptyAIResponseError as e:\n safe_print(f\" [AI分析] 第{attempt + 1}次尝试返回空响应: {e}\")\n if attempt < max_retries - 1:\n safe_print(f\" [AI分析] 准备第{attempt + 2}次重试...\")\n continue\n raise e\n\n except Exception as e:\n if (\n api_mode == CHAT_COMPLETIONS_API_MODE\n and is_chat_completions_api_unsupported_error(e)\n ):\n api_mode = RESPONSES_API_MODE\n safe_print(\n \" [AI分析] 当前服务未实现 Chat Completions API，后续重试将自动回退到 Responses API。\"\n )\n elif api_mode == RESPONSES_API_MODE and is_responses_api_unsupported_error(e):\n api_mode = CHAT_COMPLETIONS_API_MODE\n safe_print(\n \" [AI分析] 当前服务未实现 Responses API，后续重试将自动回退到 Chat Completions API。\"\n )\n if use_response_format and is_json_output_unsupported_error(e):\n use_response_format = False\n safe_print(\n \" [AI分析] 当前模型不支持结构化 JSON 输出，后续重试将自动禁用该参数。\"\n )\n if use_temperature and is_temperature_unsupported_error(e):\n use_temperature = False\n safe_print(\n \" [AI分析] 当前模型不支持 temperature 参数，后续重试将自动禁用该参数。\"\n )\n if AI_DEBUG_MODE:\n safe_print(f\"\\n--- [AI DEBUG] 第{attempt + 1}次尝试 EXCEPTION ---\")\n safe_print(repr(e))\n safe_print(traceback.format_exc())\n safe_print(\"-------------------------------------\\n\")\n safe_print(f\" [AI分析] 第{attempt + 1}次尝试AI调用失败: {e}\")\n if attempt < max_retries - 1:\n safe_print(f\" [AI分析] 准备第{attempt + 2}次重试...\")\n continue\n else:\n raise e\n" }, { "path": "src/ai_message_builder.py", "content": "\"\"\"\nAI 请求消息构造辅助函数\n\"\"\"\nfrom typing import Dict, List, Union\n\n\nTEXT_ONLY_ANALYSIS_NOTE = (\n \"补充说明：本次未提供商品图片，请仅根据商品文字字段和卖家信息判断，不要推断图片内容。\"\n)\n\n\ndef build_analysis_text_prompt(\n product_json: str,\n prompt_text: str,\n *,\n include_images: bool,\n) -> str:\n note = \"\" if include_images else f\"\\n{TEXT_ONLY_ANALYSIS_NOTE}\\n\"\n value_note = (\n \"\\n如果商品 JSON 中包含“价格参考”或 price_insight，请结合价格位置、历史走势、\"\n \"配置、成色、附件、卖家信息综合判断性价比。\"\n \"你可以额外输出可选字段 value_score(0-100) 和 value_summary，\"\n \"但必须保留原有 is_recommended/reason 等字段。\\n\"\n )\n return f\"\"\"请基于你的专业知识和我的要求，分析以下完整的商品JSON数据：\n\n```json\n{product_json}\n```\n\n {prompt_text}\n {value_note}\n {note}\"\"\"\n\n\ndef build_user_message_content(\n text_prompt: str,\n image_data_urls: List[str],\n) -> Union[str, List[Dict[str, object]]]:\n if not image_data_urls:\n return text_prompt\n\n user_content: List[Dict[str, object]] = [\n {\"type\": \"image_url\", \"image_url\": {\"url\": url}}\n for url in image_data_urls\n ]\n user_content.append({\"type\": \"text\", \"text\": text_prompt})\n return user_content\n" }, { "path": "src/api/__init__.py", "content": "" }, { "path": "src/api/dependencies.py", "content": "\"\"\"\nFastAPI 依赖注入\n提供服务实例的创建和管理\n\"\"\"\nfrom fastapi import Depends\nfrom src.services.task_service import TaskService\nfrom src.services.notification_service import NotificationService, build_notification_service\nfrom src.services.ai_service import AIAnalysisService\nfrom src.services.process_service import ProcessService\nfrom src.services.scheduler_service import SchedulerService\nfrom src.services.task_generation_service import TaskGenerationService\nfrom src.infrastructure.persistence.sqlite_task_repository import SqliteTaskRepository\nfrom src.infrastructure.external.ai_client import AIClient\n\n\n# 全局 ProcessService 实例（将在 app.py 中设置）\n_process_service_instance = None\n_scheduler_service_instance = None\n_task_generation_service_instance = None\n\n\ndef set_process_service(service: ProcessService):\n \"\"\"设置全局 ProcessService 实例\"\"\"\n global _process_service_instance\n _process_service_instance = service\n\n\ndef set_scheduler_service(service: SchedulerService):\n \"\"\"设置全局 SchedulerService 实例\"\"\"\n global _scheduler_service_instance\n _scheduler_service_instance = service\n\n\ndef set_task_generation_service(service: TaskGenerationService):\n \"\"\"设置全局 TaskGenerationService 实例\"\"\"\n global _task_generation_service_instance\n _task_generation_service_instance = service\n\n\n# 服务依赖注入\ndef get_task_service() -> TaskService:\n \"\"\"获取任务管理服务实例\"\"\"\n repository = SqliteTaskRepository()\n return TaskService(repository)\n\n\ndef get_notification_service() -> NotificationService:\n \"\"\"获取通知服务实例\"\"\"\n return build_notification_service()\n\n\ndef get_ai_service() -> AIAnalysisService:\n \"\"\"获取AI分析服务实例\"\"\"\n ai_client = AIClient()\n return AIAnalysisService(ai_client)\n\n\ndef get_process_service() -> ProcessService:\n \"\"\"获取进程管理服务实例\"\"\"\n if _process_service_instance is None:\n raise RuntimeError(\"ProcessService 未初始化\")\n return _process_service_instance\n\n\ndef get_scheduler_service() -> SchedulerService:\n \"\"\"获取调度服务实例\"\"\"\n if _scheduler_service_instance is None:\n raise RuntimeError(\"SchedulerService 未初始化\")\n return _scheduler_service_instance\n\n\ndef get_task_generation_service() -> TaskGenerationService:\n \"\"\"获取任务生成作业服务实例\"\"\"\n if _task_generation_service_instance is None:\n raise RuntimeError(\"TaskGenerationService 未初始化\")\n return _task_generation_service_instance\n" }, { "path": "src/api/routes/__init__.py", "content": "" }, { "path": "src/api/routes/accounts.py", "content": "\"\"\"\n闲鱼账号管理路由\n\"\"\"\nimport json\nimport os\nimport re\nimport aiofiles\nfrom fastapi import APIRouter, HTTPException\nfrom pydantic import BaseModel\nfrom typing import List\nfrom src.infrastructure.config.env_manager import env_manager\n\n\nrouter = APIRouter(prefix=\"/api/accounts\", tags=[\"accounts\"])\n\nACCOUNT_NAME_RE = re.compile(r\"^[a-zA-Z0-9_-]{1,50}$\")\n\n\nclass AccountCreate(BaseModel):\n name: str\n content: str\n\n\nclass AccountUpdate(BaseModel):\n content: str\n\n\ndef _strip_quotes(value: str) -> str:\n if not value:\n return value\n if value.startswith((\"\\\"\", \"'\")) and value.endswith((\"\\\"\", \"'\")):\n return value[1:-1]\n return value\n\n\ndef _state_dir() -> str:\n raw = env_manager.get_value(\"ACCOUNT_STATE_DIR\", \"state\") or \"state\"\n return _strip_quotes(raw.strip())\n\n\ndef _ensure_state_dir(path: str) -> None:\n os.makedirs(path, exist_ok=True)\n\n\ndef _validate_name(name: str) -> str:\n trimmed = name.strip()\n if not trimmed or not ACCOUNT_NAME_RE.match(trimmed):\n raise HTTPException(status_code=400, detail=\"账号名称只能包含字母、数字、下划线或短横线。\")\n return trimmed\n\n\ndef _account_path(name: str) -> str:\n filename = f\"{name}.json\"\n return os.path.join(_state_dir(), filename)\n\n\ndef _validate_json(content: str) -> None:\n try:\n json.loads(content)\n except json.JSONDecodeError:\n raise HTTPException(status_code=400, detail=\"提供的内容不是有效的JSON格式。\")\n\n\n@router.get(\"\", response_model=List[dict])\nasync def list_accounts():\n state_dir = _state_dir()\n if not os.path.isdir(state_dir):\n return []\n files = [f for f in os.listdir(state_dir) if f.endswith(\".json\")]\n accounts = []\n for filename in sorted(files):\n name = filename[:-5]\n accounts.append({\n \"name\": name,\n \"path\": os.path.join(state_dir, filename),\n })\n return accounts\n\n\n@router.get(\"/{name}\", response_model=dict)\nasync def get_account(name: str):\n account_name = _validate_name(name)\n path = _account_path(account_name)\n if not os.path.exists(path):\n raise HTTPException(status_code=404, detail=\"账号不存在\")\n async with aiofiles.open(path, \"r\", encoding=\"utf-8\") as f:\n content = await f.read()\n return {\"name\": account_name, \"path\": path, \"content\": content}\n\n\n@router.post(\"\", response_model=dict)\nasync def create_account(data: AccountCreate):\n account_name = _validate_name(data.name)\n _validate_json(data.content)\n state_dir = _state_dir()\n _ensure_state_dir(state_dir)\n path = _account_path(account_name)\n if os.path.exists(path):\n raise HTTPException(status_code=409, detail=\"账号已存在\")\n async with aiofiles.open(path, \"w\", encoding=\"utf-8\") as f:\n await f.write(data.content)\n return {\"message\": \"账号已添加\", \"name\": account_name, \"path\": path}\n\n\n@router.put(\"/{name}\", response_model=dict)\nasync def update_account(name: str, data: AccountUpdate):\n account_name = _validate_name(name)\n _validate_json(data.content)\n state_dir = _state_dir()\n _ensure_state_dir(state_dir)\n path = _account_path(account_name)\n if not os.path.exists(path):\n raise HTTPException(status_code=404, detail=\"账号不存在\")\n async with aiofiles.open(path, \"w\", encoding=\"utf-8\") as f:\n await f.write(data.content)\n return {\"message\": \"账号已更新\", \"name\": account_name, \"path\": path}\n\n\n@router.delete(\"/{name}\", response_model=dict)\nasync def delete_account(name: str):\n account_name = _validate_name(name)\n path = _account_path(account_name)\n if not os.path.exists(path):\n raise HTTPException(status_code=404, detail=\"账号不存在\")\n os.remove(path)\n return {\"message\": \"账号已删除\"}\n" }, { "path": "src/api/routes/dashboard.py", "content": "\"\"\"\nDashboard 概览路由\n\"\"\"\nfrom fastapi import APIRouter, Depends, HTTPException\n\nfrom src.api.dependencies import get_task_service\nfrom src.services.dashboard_service import build_dashboard_snapshot\nfrom src.services.task_service import TaskService\n\n\nrouter = APIRouter(prefix=\"/api/dashboard\", tags=[\"dashboard\"])\n\n\n@router.get(\"/summary\")\nasync def get_dashboard_summary(\n task_service: TaskService = Depends(get_task_service),\n):\n try:\n tasks = await task_service.get_all_tasks()\n return await build_dashboard_snapshot(tasks)\n except Exception as exc:\n raise HTTPException(status_code=500, detail=f\"加载 dashboard 数据失败: {exc}\")\n" }, { "path": "src/api/routes/login_state.py", "content": "\"\"\"\n登录状态管理路由\n\"\"\"\nimport os\nimport json\nimport aiofiles\nfrom fastapi import APIRouter, HTTPException\nfrom pydantic import BaseModel\n\n\nrouter = APIRouter(prefix=\"/api/login-state\", tags=[\"login-state\"])\n\n\nclass LoginStateUpdate(BaseModel):\n \"\"\"登录状态更新模型\"\"\"\n content: str\n\n\n@router.post(\"\", response_model=dict)\nasync def update_login_state(\n data: LoginStateUpdate,\n):\n \"\"\"接收前端发送的登录状态JSON字符串，并保存到 xianyu_state.json\"\"\"\n state_file = \"xianyu_state.json\"\n\n try:\n # 验证是否是有效的JSON\n json.loads(data.content)\n except json.JSONDecodeError:\n raise HTTPException(status_code=400, detail=\"提供的内容不是有效的JSON格式。\")\n\n try:\n async with aiofiles.open(state_file, 'w', encoding='utf-8') as f:\n await f.write(data.content)\n return {\"message\": f\"登录状态文件 '{state_file}' 已成功更新。\"}\n except Exception as e:\n raise HTTPException(status_code=500, detail=f\"写入登录状态文件时出错: {e}\")\n\n\n@router.delete(\"\", response_model=dict)\nasync def delete_login_state():\n \"\"\"删除 xianyu_state.json 文件\"\"\"\n state_file = \"xianyu_state.json\"\n\n if os.path.exists(state_file):\n try:\n os.remove(state_file)\n return {\"message\": \"登录状态文件已成功删除。\"}\n except OSError as e:\n raise HTTPException(status_code=500, detail=f\"删除登录状态文件时出错: {e}\")\n\n return {\"message\": \"登录状态文件不存在，无需删除。\"}\n" }, { "path": "src/api/routes/logs.py", "content": "\"\"\"\n日志管理路由\n\"\"\"\nimport os\nfrom typing import Optional, Tuple, List\nimport aiofiles\nfrom fastapi import APIRouter, Depends, Query\nfrom fastapi.responses import JSONResponse\nfrom src.api.dependencies import get_task_service\nfrom src.services.task_service import TaskService\nfrom src.utils import resolve_task_log_path\n\n\nrouter = APIRouter(prefix=\"/api/logs\", tags=[\"logs\"])\n\n\nasync def _read_tail_lines(\n log_file_path: str,\n offset_lines: int,\n limit_lines: int,\n chunk_size: int = 8192\n) -> Tuple[List[str], bool, int]:\n async with aiofiles.open(log_file_path, 'rb') as f:\n await f.seek(0, os.SEEK_END)\n file_size = await f.tell()\n\n if file_size == 0 or limit_lines <= 0:\n return [], False, file_size\n\n offset_lines = max(0, int(offset_lines))\n limit_lines = max(0, int(limit_lines))\n lines_needed = offset_lines + limit_lines\n\n pos = file_size\n buffer = b\"\"\n lines: List[bytes] = []\n\n while pos > 0 and len(lines) < lines_needed:\n read_size = min(chunk_size, pos)\n pos -= read_size\n await f.seek(pos)\n chunk = await f.read(read_size)\n buffer = chunk + buffer\n lines = buffer.splitlines()\n\n start = max(0, len(lines) - lines_needed)\n end = max(0, len(lines) - offset_lines)\n selected = lines[start:end] if end > start else []\n\n has_more = pos > 0 or len(lines) > lines_needed\n decoded = [line.decode('utf-8', errors='replace') for line in selected]\n return decoded, has_more, file_size\n\n\n@router.get(\"\")\nasync def get_logs(\n from_pos: int = 0,\n task_id: Optional[int] = Query(default=None, ge=0),\n task_service: TaskService = Depends(get_task_service),\n):\n \"\"\"获取日志内容（增量读取）\"\"\"\n if task_id is None:\n return JSONResponse(content={\n \"new_content\": \"请选择任务后查看日志。\",\n \"new_pos\": 0\n })\n\n task = await task_service.get_task(task_id)\n if not task:\n return JSONResponse(status_code=404, content={\n \"new_content\": \"任务不存在或已删除。\",\n \"new_pos\": 0\n })\n\n log_file_path = resolve_task_log_path(task_id, task.task_name)\n\n if not os.path.exists(log_file_path):\n return JSONResponse(content={\n \"new_content\": \"\",\n \"new_pos\": 0\n })\n\n try:\n async with aiofiles.open(log_file_path, 'rb') as f:\n await f.seek(0, os.SEEK_END)\n file_size = await f.tell()\n\n if from_pos >= file_size:\n return {\"new_content\": \"\", \"new_pos\": file_size}\n\n await f.seek(from_pos)\n new_bytes = await f.read()\n\n new_content = new_bytes.decode('utf-8', errors='replace')\n return {\"new_content\": new_content, \"new_pos\": file_size}\n\n except Exception as e:\n return JSONResponse(\n status_code=500,\n content={\"new_content\": f\"\\n读取日志文件时出错: {e}\", \"new_pos\": from_pos}\n ) \n\n\n@router.get(\"/tail\")\nasync def get_logs_tail(\n task_id: Optional[int] = Query(default=None, ge=0),\n offset_lines: int = Query(default=0, ge=0),\n limit_lines: int = Query(default=50, ge=1, le=1000),\n task_service: TaskService = Depends(get_task_service),\n):\n \"\"\"获取日志尾部内容（按行分页）\"\"\"\n if task_id is None:\n return JSONResponse(content={\n \"content\": \"\",\n \"has_more\": False,\n \"next_offset\": 0,\n \"new_pos\": 0\n })\n\n task = await task_service.get_task(task_id)\n if not task:\n return JSONResponse(status_code=404, content={\n \"content\": \"\",\n \"has_more\": False,\n \"next_offset\": 0,\n \"new_pos\": 0\n })\n\n log_file_path = resolve_task_log_path(task_id, task.task_name)\n\n if not os.path.exists(log_file_path):\n return JSONResponse(content={\n \"content\": \"\",\n \"has_more\": False,\n \"next_offset\": 0,\n \"new_pos\": 0\n })\n\n try:\n lines, has_more, file_size = await _read_tail_lines(\n log_file_path,\n offset_lines=offset_lines,\n limit_lines=limit_lines\n )\n next_offset = offset_lines + len(lines)\n return {\n \"content\": \"\\n\".join(lines),\n \"has_more\": has_more,\n \"next_offset\": next_offset,\n \"new_pos\": file_size\n }\n except Exception as e:\n return JSONResponse(\n status_code=500,\n content={\n \"content\": f\"读取日志文件时出错: {e}\",\n \"has_more\": False,\n \"next_offset\": offset_lines,\n \"new_pos\": 0\n }\n )\n\n\n@router.delete(\"\", response_model=dict)\nasync def clear_logs(\n task_id: Optional[int] = Query(default=None, ge=0),\n task_service: TaskService = Depends(get_task_service),\n):\n \"\"\"清空日志文件\"\"\"\n if task_id is None:\n return {\"message\": \"未指定任务，无法清空日志。\"}\n\n task = await task_service.get_task(task_id)\n if not task:\n return {\"message\": \"任务不存在或已删除。\"}\n\n log_file_path = resolve_task_log_path(task_id, task.task_name)\n\n if not os.path.exists(log_file_path):\n return {\"message\": \"日志文件不存在，无需清空。\"}\n\n try:\n async with aiofiles.open(log_file_path, 'w', encoding='utf-8') as f:\n await f.write(\"\")\n return {\"message\": \"日志已成功清空。\"}\n except Exception as e:\n return JSONResponse(\n status_code=500,\n content={\"message\": f\"清空日志文件时出错: {e}\"}\n )\n\n if not os.path.exists(log_file_path):\n return {\"message\": \"日志文件不存在，无需清空。\"}\n\n try:\n async with aiofiles.open(log_file_path, 'w', encoding='utf-8') as f:\n await f.write(\"\")\n return {\"message\": \"日志已成功清空。\"}\n except Exception as e:\n return JSONResponse(\n status_code=500,\n content={\"message\": f\"清空日志文件时出错: {e}\"}\n )\n" }, { "path": "src/api/routes/prompts.py", "content": "\"\"\"\nPrompt 管理路由\n\"\"\"\nimport os\nimport aiofiles\nfrom fastapi import APIRouter, HTTPException\nfrom pydantic import BaseModel\n\n\nrouter = APIRouter(prefix=\"/api/prompts\", tags=[\"prompts\"])\n\n\nclass PromptUpdate(BaseModel):\n \"\"\"Prompt 更新模型\"\"\"\n content: str\n\n\n@router.get(\"\")\nasync def list_prompts():\n \"\"\"列出所有 prompt 文件\"\"\"\n prompts_dir = \"prompts\"\n if not os.path.isdir(prompts_dir):\n return []\n return [f for f in os.listdir(prompts_dir) if f.endswith(\".txt\")]\n\n\n@router.get(\"/{filename}\")\nasync def get_prompt(filename: str):\n \"\"\"获取 prompt 文件内容\"\"\"\n if \"/\" in filename or \"..\" in filename:\n raise HTTPException(status_code=400, detail=\"无效的文件名\")\n\n filepath = os.path.join(\"prompts\", filename)\n if not os.path.exists(filepath):\n raise HTTPException(status_code=404, detail=\"Prompt 文件未找到\")\n\n async with aiofiles.open(filepath, 'r', encoding='utf-8') as f:\n content = await f.read()\n return {\"filename\": filename, \"content\": content}\n\n\n@router.put(\"/{filename}\")\nasync def update_prompt(\n filename: str,\n prompt_update: PromptUpdate,\n):\n \"\"\"更新 prompt 文件内容\"\"\"\n if \"/\" in filename or \"..\" in filename:\n raise HTTPException(status_code=400, detail=\"无效的文件名\")\n\n filepath = os.path.join(\"prompts\", filename)\n if not os.path.exists(filepath):\n raise HTTPException(status_code=404, detail=\"Prompt 文件未找到\")\n\n try:\n async with aiofiles.open(filepath, 'w', encoding='utf-8') as f:\n await f.write(prompt_update.content)\n return {\"message\": f\"Prompt 文件 '{filename}' 更新成功\"}\n except Exception as e:\n raise HTTPException(status_code=500, detail=f\"写入文件时出错: {e}\")\n" }, { "path": "src/api/routes/results.py", "content": "\"\"\"\n结果文件管理路由\n\"\"\"\nfrom fastapi import APIRouter, HTTPException, Query\nfrom fastapi.responses import Response\nfrom urllib.parse import quote\n\nfrom src.services.price_history_service import build_price_history_insights\nfrom src.services.result_export_service import build_results_csv\nfrom src.services.result_file_service import (\n enrich_records_with_price_insight,\n validate_result_filename,\n)\nfrom src.services.result_storage_service import (\n build_result_ndjson,\n delete_result_file_records,\n list_result_filenames,\n load_all_result_records,\n query_result_records,\n result_file_exists,\n)\n\n\nrouter = APIRouter(prefix=\"/api/results\", tags=[\"results\"])\n\nDEFAULT_EXPORT_FILENAME = \"export.csv\"\n\n\ndef _build_download_headers(export_name: str) -> dict[str, str]:\n ascii_name = export_name.encode(\"ascii\", \"ignore\").decode(\"ascii\")\n if ascii_name != export_name or not ascii_name:\n ascii_name = DEFAULT_EXPORT_FILENAME\n encoded_name = quote(export_name, safe=\"\")\n return {\n \"Content-Disposition\": (\n f'attachment; filename=\"{ascii_name}\"; '\n f\"filename*=UTF-8''{encoded_name}\"\n )\n }\n\n\n@router.get(\"/files\")\nasync def get_result_files():\n \"\"\"获取所有结果文件列表\"\"\"\n return {\"files\": await list_result_filenames()}\n\n\n@router.get(\"/files/{filename:path}\")\nasync def download_result_file(filename: str):\n \"\"\"下载指定的结果文件\"\"\"\n if \"..\" in filename or filename.startswith(\"/\"):\n return {\"error\": \"非法的文件路径\"}\n if not filename.endswith(\".jsonl\") or not await result_file_exists(filename):\n return {\"error\": \"文件不存在\"}\n return Response(\n content=await build_result_ndjson(filename),\n media_type=\"application/x-ndjson\",\n headers={\"Content-Disposition\": f'attachment; filename=\"{filename}\"'},\n )\n\n\n@router.delete(\"/files/{filename:path}\")\nasync def delete_result_file(filename: str):\n \"\"\"删除指定的结果文件\"\"\"\n if \"..\" in filename or filename.startswith(\"/\"):\n raise HTTPException(status_code=400, detail=\"非法的文件路径\")\n if not filename.endswith(\".jsonl\"):\n raise HTTPException(status_code=400, detail=\"只能删除 .jsonl 文件\")\n deleted_rows = await delete_result_file_records(filename)\n if deleted_rows <= 0:\n raise HTTPException(status_code=404, detail=\"文件不存在\")\n return {\"message\": f\"文件 {filename} 已成功删除\"}\n\n\n@router.get(\"/{filename}\")\nasync def get_result_file_content(\n filename: str,\n page: int = Query(1, ge=1),\n limit: int = Query(20, ge=1, le=100),\n recommended_only: bool = Query(False), # 兼容旧参数，等价于 ai_recommended_only\n ai_recommended_only: bool = Query(False),\n keyword_recommended_only: bool = Query(False),\n sort_by: str = Query(\"crawl_time\"),\n sort_order: str = Query(\"desc\"),\n):\n \"\"\"读取指定的 .jsonl 文件内容，支持分页、筛选和排序\"\"\"\n if ai_recommended_only and keyword_recommended_only:\n raise HTTPException(status_code=400, detail=\"AI推荐筛选与关键词推荐筛选不能同时开启。\")\n\n if recommended_only and not ai_recommended_only and not keyword_recommended_only:\n ai_recommended_only = True\n\n try:\n validate_result_filename(filename)\n total_items, items = await query_result_records(\n filename,\n ai_recommended_only=ai_recommended_only,\n keyword_recommended_only=keyword_recommended_only,\n sort_by=sort_by,\n sort_order=sort_order,\n page=page,\n limit=limit,\n )\n except ValueError as exc:\n raise HTTPException(status_code=400, detail=str(exc))\n except Exception as exc:\n raise HTTPException(status_code=500, detail=f\"读取结果文件时出错: {exc}\")\n if total_items <= 0 and not await result_file_exists(filename):\n raise HTTPException(status_code=404, detail=\"结果文件未找到\")\n paginated_results = enrich_records_with_price_insight(items, filename)\n\n return {\n \"total_items\": total_items,\n \"page\": page,\n \"limit\": limit,\n \"items\": paginated_results\n }\n\n\n@router.get(\"/{filename}/insights\")\nasync def get_result_file_insights(filename: str):\n try:\n validate_result_filename(filename)\n keyword = filename.replace(\"_full_data.jsonl\", \"\")\n return build_price_history_insights(keyword)\n except ValueError as exc:\n raise HTTPException(status_code=400, detail=str(exc))\n\n\n@router.get(\"/{filename}/export\")\nasync def export_result_file_content(\n filename: str,\n recommended_only: bool = Query(False),\n ai_recommended_only: bool = Query(False),\n keyword_recommended_only: bool = Query(False),\n sort_by: str = Query(\"crawl_time\"),\n sort_order: str = Query(\"desc\"),\n):\n if ai_recommended_only and keyword_recommended_only:\n raise HTTPException(status_code=400, detail=\"AI推荐筛选与关键词推荐筛选不能同时开启。\")\n if recommended_only and not ai_recommended_only and not keyword_recommended_only:\n ai_recommended_only = True\n\n try:\n validate_result_filename(filename)\n results = await load_all_result_records(\n filename,\n ai_recommended_only=ai_recommended_only,\n keyword_recommended_only=keyword_recommended_only,\n sort_by=sort_by,\n sort_order=sort_order,\n )\n csv_text = build_results_csv(\n enrich_records_with_price_insight(results, filename)\n )\n except ValueError as exc:\n raise HTTPException(status_code=400, detail=str(exc))\n except Exception as exc:\n raise HTTPException(status_code=500, detail=f\"导出结果文件时出错: {exc}\")\n if not results and not await result_file_exists(filename):\n raise HTTPException(status_code=404, detail=\"结果文件未找到\")\n\n export_name = filename.replace(\".jsonl\", \".csv\")\n headers = _build_download_headers(export_name)\n return Response(content=csv_text, media_type=\"text/csv; charset=utf-8\", headers=headers)\n" }, { "path": "src/api/routes/settings.py", "content": "\"\"\"\n设置管理路由\n\"\"\"\nimport os\nfrom typing import Optional\n\nfrom dotenv import load_dotenv\nfrom fastapi import APIRouter, Depends, HTTPException\nfrom pydantic import BaseModel, Field\n\nfrom src.api.dependencies import get_process_service\nfrom src.infrastructure.config.env_manager import env_manager\nfrom src.infrastructure.config.settings import (\n AISettings,\n reload_settings,\n scraper_settings,\n)\nfrom src.services.ai_request_compat import (\n CHAT_COMPLETIONS_API_MODE,\n RESPONSES_API_MODE,\n build_ai_request_params,\n create_ai_response_sync,\n is_chat_completions_api_unsupported_error,\n is_responses_api_unsupported_error,\n)\nfrom src.services.ai_response_parser import extract_ai_response_content\nfrom src.services.notification_config_service import (\n NotificationSettingsValidationError,\n build_configured_channels,\n build_notification_settings_response,\n build_notification_status_flags,\n load_notification_settings,\n model_dump,\n prepare_notification_settings_update,\n)\nfrom src.services.notification_service import build_notification_service\nfrom src.services.process_service import ProcessService\n\n\nrouter = APIRouter(prefix=\"/api/settings\", tags=[\"settings\"])\nAI_TEST_PROMPT = \"Reply with OK only.\"\nAI_TEST_MAX_OUTPUT_TOKENS = 32\n\n\ndef _reload_env() -> None:\n load_dotenv(dotenv_path=env_manager.env_file, override=True)\n reload_settings()\n\n\ndef _env_bool(key: str, default: bool = False) -> bool:\n value = env_manager.get_value(key)\n if value is None:\n return default\n return str(value).strip().lower() in {\"1\", \"true\", \"yes\", \"y\", \"on\"}\n\n\ndef _env_int(key: str, default: int) -> int:\n value = env_manager.get_value(key)\n if value is None:\n return default\n try:\n return int(value)\n except ValueError:\n return default\n\n\ndef _normalize_bool_value(value: bool) -> str:\n return \"true\" if value else \"false\"\n\n\nclass NotificationSettingsModel(BaseModel):\n \"\"\"通知设置模型\"\"\"\n\n NTFY_TOPIC_URL: Optional[str] = None\n GOTIFY_URL: Optional[str] = None\n GOTIFY_TOKEN: Optional[str] = None\n BARK_URL: Optional[str] = None\n WX_BOT_URL: Optional[str] = None\n TELEGRAM_BOT_TOKEN: Optional[str] = None\n TELEGRAM_CHAT_ID: Optional[str] = None\n TELEGRAM_API_BASE_URL: Optional[str] = None\n WEBHOOK_URL: Optional[str] = None\n WEBHOOK_METHOD: Optional[str] = None\n WEBHOOK_HEADERS: Optional[str] = None\n WEBHOOK_CONTENT_TYPE: Optional[str] = None\n WEBHOOK_QUERY_PARAMETERS: Optional[str] = None\n WEBHOOK_BODY: Optional[str] = None\n PCURL_TO_MOBILE: Optional[bool] = None\n\n\nclass NotificationTestRequest(BaseModel):\n \"\"\"通知测试请求\"\"\"\n\n channel: Optional[str] = None\n settings: NotificationSettingsModel = Field(default_factory=NotificationSettingsModel)\n\n\nclass AISettingsModel(BaseModel):\n \"\"\"AI设置模型\"\"\"\n\n OPENAI_API_KEY: Optional[str] = None\n OPENAI_BASE_URL: Optional[str] = None\n OPENAI_MODEL_NAME: Optional[str] = None\n SKIP_AI_ANALYSIS: Optional[bool] = None\n PROXY_URL: Optional[str] = None\n\n\nclass RotationSettingsModel(BaseModel):\n ACCOUNT_ROTATION_ENABLED: Optional[bool] = None\n ACCOUNT_ROTATION_MODE: Optional[str] = None\n ACCOUNT_ROTATION_RETRY_LIMIT: Optional[int] = None\n ACCOUNT_BLACKLIST_TTL: Optional[int] = None\n ACCOUNT_STATE_DIR: Optional[str] = None\n PROXY_ROTATION_ENABLED: Optional[bool] = None\n PROXY_ROTATION_MODE: Optional[str] = None\n PROXY_POOL: Optional[str] = None\n PROXY_ROTATION_RETRY_LIMIT: Optional[int] = None\n PROXY_BLACKLIST_TTL: Optional[int] = None\n\n\n@router.get(\"/notifications\")\nasync def get_notification_settings():\n return build_notification_settings_response(load_notification_settings())\n\n\n@router.put(\"/notifications\")\nasync def update_notification_settings(settings: NotificationSettingsModel):\n try:\n updates, deletions, merged_settings = prepare_notification_settings_update(\n model_dump(settings, exclude_unset=True),\n load_notification_settings(),\n )\n except NotificationSettingsValidationError as exc:\n raise HTTPException(status_code=422, detail=str(exc)) from exc\n\n success = env_manager.apply_changes(updates=updates, deletions=deletions)\n if not success:\n raise HTTPException(status_code=500, detail=\"更新通知设置失败\")\n\n _reload_env()\n return {\n \"message\": \"通知设置已成功更新\",\n \"configured_channels\": build_configured_channels(merged_settings),\n }\n\n\n@router.post(\"/notifications/test\")\nasync def test_notification_settings(payload: NotificationTestRequest):\n try:\n _, _, merged_settings = prepare_notification_settings_update(\n model_dump(payload.settings, exclude_unset=True),\n load_notification_settings(),\n )\n except NotificationSettingsValidationError as exc:\n raise HTTPException(status_code=422, detail=str(exc)) from exc\n\n service = build_notification_service(merged_settings)\n if not service.clients:\n raise HTTPException(status_code=422, detail=\"请至少配置一个可用的通知渠道\")\n\n results = await service.send_test_notification()\n if payload.channel:\n if payload.channel not in results:\n raise HTTPException(\n status_code=422,\n detail=f\"渠道 {payload.channel} 未配置或不受支持\",\n )\n results = {payload.channel: results[payload.channel]}\n\n return {\n \"message\": \"测试通知已执行\",\n \"results\": results,\n }\n\n\n@router.get(\"/rotation\")\nasync def get_rotation_settings():\n return {\n \"ACCOUNT_ROTATION_ENABLED\": _env_bool(\"ACCOUNT_ROTATION_ENABLED\", False),\n \"ACCOUNT_ROTATION_MODE\": env_manager.get_value(\"ACCOUNT_ROTATION_MODE\", \"per_task\"),\n \"ACCOUNT_ROTATION_RETRY_LIMIT\": _env_int(\"ACCOUNT_ROTATION_RETRY_LIMIT\", 2),\n \"ACCOUNT_BLACKLIST_TTL\": _env_int(\"ACCOUNT_BLACKLIST_TTL\", 300),\n \"ACCOUNT_STATE_DIR\": env_manager.get_value(\"ACCOUNT_STATE_DIR\", \"state\"),\n \"PROXY_ROTATION_ENABLED\": _env_bool(\"PROXY_ROTATION_ENABLED\", False),\n \"PROXY_ROTATION_MODE\": env_manager.get_value(\"PROXY_ROTATION_MODE\", \"per_task\"),\n \"PROXY_POOL\": env_manager.get_value(\"PROXY_POOL\", \"\"),\n \"PROXY_ROTATION_RETRY_LIMIT\": _env_int(\"PROXY_ROTATION_RETRY_LIMIT\", 2),\n \"PROXY_BLACKLIST_TTL\": _env_int(\"PROXY_BLACKLIST_TTL\", 300),\n }\n\n\n@router.put(\"/rotation\")\nasync def update_rotation_settings(settings: RotationSettingsModel):\n updates = {}\n payload = model_dump(settings, exclude_unset=True)\n for key, value in payload.items():\n if isinstance(value, bool):\n updates[key] = _normalize_bool_value(value)\n else:\n updates[key] = str(value)\n success = env_manager.update_values(updates)\n if not success:\n raise HTTPException(status_code=500, detail=\"更新轮换设置失败\")\n _reload_env()\n return {\"message\": \"轮换设置已成功更新\"}\n\n\n@router.get(\"/status\")\nasync def get_system_status(\n process_service: ProcessService = Depends(get_process_service),\n):\n state_file = \"xianyu_state.json\"\n login_state_exists = os.path.exists(state_file)\n env_file_exists = os.path.exists(env_manager.env_file)\n openai_api_key = env_manager.get_value(\"OPENAI_API_KEY\", \"\")\n openai_base_url = env_manager.get_value(\"OPENAI_BASE_URL\", \"\")\n openai_model_name = env_manager.get_value(\"OPENAI_MODEL_NAME\", \"\")\n ai_settings = AISettings()\n notification_settings = load_notification_settings()\n running_task_ids = [\n task_id\n for task_id, process in process_service.processes.items()\n if process and process.returncode is None\n ]\n\n return {\n \"ai_configured\": ai_settings.is_configured(),\n \"notification_configured\": notification_settings.has_any_notification_enabled(),\n \"headless_mode\": scraper_settings.run_headless,\n \"running_in_docker\": scraper_settings.running_in_docker,\n \"scraper_running\": len(running_task_ids) > 0,\n \"running_task_ids\": running_task_ids,\n \"login_state_file\": {\n \"exists\": login_state_exists,\n \"path\": state_file,\n },\n \"env_file\": {\n \"exists\": env_file_exists,\n \"openai_api_key_set\": bool(openai_api_key),\n \"openai_base_url_set\": bool(openai_base_url),\n \"openai_model_name_set\": bool(openai_model_name),\n **build_notification_status_flags(notification_settings),\n },\n \"configured_notification_channels\": build_configured_channels(notification_settings),\n }\n\n\n@router.get(\"/ai\")\nasync def get_ai_settings():\n return {\n \"OPENAI_BASE_URL\": env_manager.get_value(\"OPENAI_BASE_URL\", \"\"),\n \"OPENAI_MODEL_NAME\": env_manager.get_value(\"OPENAI_MODEL_NAME\", \"\"),\n \"SKIP_AI_ANALYSIS\": env_manager.get_value(\"SKIP_AI_ANALYSIS\", \"false\").lower() == \"true\",\n \"PROXY_URL\": env_manager.get_value(\"PROXY_URL\", \"\"),\n }\n\n\n@router.put(\"/ai\")\nasync def update_ai_settings(settings: AISettingsModel):\n updates = {}\n if settings.OPENAI_API_KEY is not None:\n updates[\"OPENAI_API_KEY\"] = settings.OPENAI_API_KEY\n if settings.OPENAI_BASE_URL is not None:\n updates[\"OPENAI_BASE_URL\"] = settings.OPENAI_BASE_URL\n if settings.OPENAI_MODEL_NAME is not None:\n updates[\"OPENAI_MODEL_NAME\"] = settings.OPENAI_MODEL_NAME\n if settings.SKIP_AI_ANALYSIS is not None:\n updates[\"SKIP_AI_ANALYSIS\"] = str(settings.SKIP_AI_ANALYSIS).lower()\n if settings.PROXY_URL is not None:\n updates[\"PROXY_URL\"] = settings.PROXY_URL\n\n success = env_manager.update_values(updates)\n if not success:\n raise HTTPException(status_code=500, detail=\"更新AI设置失败\")\n _reload_env()\n return {\"message\": \"AI设置已成功更新\"}\n\n\n@router.post(\"/ai/test\")\nasync def test_ai_settings(settings: dict):\n \"\"\"测试AI模型设置是否有效\"\"\"\n try:\n from openai import OpenAI\n import httpx\n\n stored_api_key = env_manager.get_value(\"OPENAI_API_KEY\", \"\")\n submitted_api_key = settings.get(\"OPENAI_API_KEY\", \"\")\n api_key = submitted_api_key or stored_api_key\n\n client_params = {\n \"api_key\": api_key,\n \"base_url\": settings.get(\"OPENAI_BASE_URL\", \"\"),\n \"timeout\": httpx.Timeout(30.0),\n }\n\n proxy_url = settings.get(\"PROXY_URL\", \"\")\n if proxy_url:\n client_params[\"http_client\"] = httpx.Client(proxy=proxy_url)\n\n model_name = settings.get(\"OPENAI_MODEL_NAME\", \"\")\n client = OpenAI(**client_params)\n messages = [{\"role\": \"user\", \"content\": AI_TEST_PROMPT}]\n api_mode = CHAT_COMPLETIONS_API_MODE\n\n try:\n response = create_ai_response_sync(\n client,\n api_mode,\n build_ai_request_params(\n api_mode,\n model=model_name,\n messages=messages,\n max_output_tokens=AI_TEST_MAX_OUTPUT_TOKENS,\n ),\n )\n except Exception as exc:\n if not is_chat_completions_api_unsupported_error(exc):\n raise\n api_mode = RESPONSES_API_MODE\n response = create_ai_response_sync(\n client,\n api_mode,\n build_ai_request_params(\n api_mode,\n model=model_name,\n messages=messages,\n max_output_tokens=AI_TEST_MAX_OUTPUT_TOKENS,\n ),\n )\n\n return {\n \"success\": True,\n \"message\": \"AI模型连接测试成功！\",\n \"response\": extract_ai_response_content(response),\n }\n except Exception as exc:\n return {\n \"success\": False,\n \"message\": f\"AI模型连接测试失败: {exc}\",\n }\n" }, { "path": "src/api/routes/tasks.py", "content": "\"\"\"\n任务管理路由\n\"\"\"\nfrom fastapi import APIRouter, Depends, HTTPException\nfrom fastapi.responses import JSONResponse\nfrom typing import List\nimport os\nimport aiofiles\nfrom src.api.dependencies import (\n get_process_service,\n get_scheduler_service,\n get_task_generation_service,\n get_task_service,\n)\nfrom src.services.task_service import TaskService\nfrom src.services.process_service import ProcessService\nfrom src.services.scheduler_service import SchedulerService\nfrom src.services.task_generation_service import TaskGenerationService\nfrom src.services.task_generation_runner import (\n build_task_create,\n run_ai_generation_job,\n)\nfrom src.services.task_payloads import serialize_task, serialize_tasks\nfrom src.domain.models.task import TaskCreate, TaskUpdate, TaskGenerateRequest\nfrom src.prompt_utils import generate_criteria\nfrom src.utils import resolve_task_log_path\nfrom src.services.account_strategy_service import normalize_account_strategy\nfrom src.infrastructure.persistence.storage_names import build_result_filename\nfrom src.services.price_history_service import delete_price_snapshots\nfrom src.services.result_storage_service import delete_result_file_records\nrouter = APIRouter(prefix=\"/api/tasks\", tags=[\"tasks\"])\n\nasync def _reload_scheduler_if_needed(\n task_service: TaskService,\n scheduler_service: SchedulerService,\n):\n tasks = await task_service.get_all_tasks()\n await scheduler_service.reload_jobs(tasks)\n\n\ndef _has_keyword_rules(rules) -> bool:\n return bool(rules and len(rules) > 0)\n\n\ndef _validate_final_account_strategy(existing_task, task_update: TaskUpdate) -> None:\n account_state_file = (\n task_update.account_state_file\n if task_update.account_state_file is not None\n else existing_task.account_state_file\n )\n account_strategy = normalize_account_strategy(\n task_update.account_strategy,\n account_state_file,\n )\n task_update.account_strategy = account_strategy\n if account_strategy == \"fixed\" and not account_state_file:\n raise HTTPException(status_code=400, detail=\"固定账号模式下必须选择账号。\")\n@router.get(\"\", response_model=List[dict])\nasync def get_tasks(\n service: TaskService = Depends(get_task_service),\n scheduler_service: SchedulerService = Depends(get_scheduler_service),\n):\n \"\"\"获取所有任务\"\"\"\n tasks = await service.get_all_tasks()\n return serialize_tasks(tasks, scheduler_service)\n@router.get(\"/{task_id}\", response_model=dict)\nasync def get_task(\n task_id: int,\n service: TaskService = Depends(get_task_service),\n scheduler_service: SchedulerService = Depends(get_scheduler_service),\n):\n \"\"\"获取单个任务\"\"\"\n task = await service.get_task(task_id)\n if not task:\n raise HTTPException(status_code=404, detail=\"任务未找到\")\n return serialize_task(task, scheduler_service)\n@router.post(\"/\", response_model=dict)\nasync def create_task(\n task_create: TaskCreate,\n service: TaskService = Depends(get_task_service),\n scheduler_service: SchedulerService = Depends(get_scheduler_service),\n):\n \"\"\"创建新任务\"\"\"\n task = await service.create_task(task_create)\n await _reload_scheduler_if_needed(service, scheduler_service)\n return {\"message\": \"任务创建成功\", \"task\": serialize_task(task, scheduler_service)}\n@router.post(\"/generate\", response_model=dict)\nasync def generate_task(\n req: TaskGenerateRequest,\n service: TaskService = Depends(get_task_service),\n scheduler_service: SchedulerService = Depends(get_scheduler_service),\n generation_service: TaskGenerationService = Depends(get_task_generation_service),\n):\n \"\"\"创建任务。AI模式会生成分析标准，关键词模式直接保存规则。\"\"\"\n print(f\"收到任务生成请求: {req.task_name}，模式: {req.decision_mode}\")\n\n try:\n mode = req.decision_mode or \"ai\"\n if mode == \"ai\":\n job = await generation_service.create_job(req.task_name)\n generation_service.track(\n run_ai_generation_job(\n job_id=job.job_id,\n req=req,\n task_service=service,\n scheduler_service=scheduler_service,\n generation_service=generation_service,\n )\n )\n return JSONResponse(\n status_code=202,\n content={\n \"message\": \"AI 任务生成已开始。\",\n \"job\": job.model_dump(mode=\"json\"),\n },\n )\n\n task = await service.create_task(build_task_create(req, \"\"))\n await _reload_scheduler_if_needed(service, scheduler_service)\n return {\"message\": \"任务创建成功。\", \"task\": serialize_task(task, scheduler_service)}\n\n except HTTPException:\n raise\n except Exception as e:\n error_msg = f\"AI任务生成API发生未知错误: {str(e)}\"\n print(error_msg)\n import traceback\n print(traceback.format_exc())\n raise HTTPException(status_code=500, detail=error_msg)\n@router.get(\"/generate-jobs/{job_id}\", response_model=dict)\nasync def get_task_generation_job(\n job_id: str,\n generation_service: TaskGenerationService = Depends(get_task_generation_service),\n):\n \"\"\"获取任务生成作业状态\"\"\"\n job = await generation_service.get_job(job_id)\n if not job:\n raise HTTPException(status_code=404, detail=\"任务生成作业未找到\")\n return {\"job\": job.model_dump(mode=\"json\")}\n@router.patch(\"/{task_id}\", response_model=dict)\nasync def update_task(\n task_id: int,\n task_update: TaskUpdate,\n service: TaskService = Depends(get_task_service),\n scheduler_service: SchedulerService = Depends(get_scheduler_service),\n):\n \"\"\"更新任务\"\"\"\n try:\n existing_task = await service.get_task(task_id)\n if not existing_task:\n raise HTTPException(status_code=404, detail=\"任务未找到\")\n _validate_final_account_strategy(existing_task, task_update)\n\n current_mode = getattr(existing_task, \"decision_mode\", \"ai\") or \"ai\"\n target_mode = task_update.decision_mode or current_mode\n description_changed = (\n task_update.description is not None\n and task_update.description != existing_task.description\n )\n switched_to_ai = current_mode != \"ai\" and target_mode == \"ai\"\n\n if target_mode == \"keyword\":\n final_rules = (\n task_update.keyword_rules\n if task_update.keyword_rules is not None\n else getattr(existing_task, \"keyword_rules\", [])\n )\n if not _has_keyword_rules(final_rules):\n raise HTTPException(status_code=400, detail=\"关键词模式下至少需要一个关键词。\")\n if target_mode == \"ai\" and (description_changed or switched_to_ai):\n print(f\"检测到任务 {task_id} 需要刷新 AI 标准文件，开始重新生成...\")\n try:\n description_for_ai = (\n task_update.description\n if task_update.description is not None\n else existing_task.description\n )\n if not str(description_for_ai or \"\").strip():\n raise HTTPException(status_code=400, detail=\"AI 模式下详细需求不能为空。\")\n safe_keyword = \"\".join(\n c for c in existing_task.keyword.lower().replace(' ', '_')\n if c.isalnum() or c in \"_-\"\n ).rstrip()\n output_filename = f\"prompts/{safe_keyword}_criteria.txt\"\n print(f\"目标文件路径: {output_filename}\")\n print(\"开始调用 AI 生成新的分析标准...\")\n generated_criteria = await generate_criteria(\n user_description=description_for_ai,\n reference_file_path=\"prompts/macbook_criteria.txt\"\n )\n if not generated_criteria or len(generated_criteria.strip()) == 0:\n print(\"AI 返回的内容为空\")\n raise HTTPException(status_code=500, detail=\"AI 未能生成分析标准，返回内容为空。\")\n print(f\"保存新的分析标准到: {output_filename}\")\n os.makedirs(\"prompts\", exist_ok=True)\n async with aiofiles.open(output_filename, 'w', encoding='utf-8') as f:\n await f.write(generated_criteria)\n print(f\"新的分析标准已保存\")\n task_update.ai_prompt_criteria_file = output_filename\n print(f\"已更新 ai_prompt_criteria_file 字段为: {output_filename}\")\n except HTTPException:\n raise\n except Exception as e:\n error_msg = f\"重新生成 criteria 文件时出错: {str(e)}\"\n print(error_msg)\n import traceback\n print(traceback.format_exc())\n raise HTTPException(status_code=500, detail=error_msg)\n task = await service.update_task(task_id, task_update)\n await _reload_scheduler_if_needed(service, scheduler_service)\n return {\"message\": \"任务更新成功\", \"task\": serialize_task(task, scheduler_service)}\n except ValueError as e:\n raise HTTPException(status_code=404, detail=str(e))\n@router.delete(\"/{task_id}\", response_model=dict)\nasync def delete_task(\n task_id: int,\n service: TaskService = Depends(get_task_service),\n process_service: ProcessService = Depends(get_process_service),\n scheduler_service: SchedulerService = Depends(get_scheduler_service),\n):\n \"\"\"删除任务\"\"\"\n task = await service.get_task(task_id)\n if not task:\n raise HTTPException(status_code=404, detail=\"任务未找到\")\n\n await process_service.stop_task(task_id)\n success = await service.delete_task(task_id)\n if not success:\n raise HTTPException(status_code=404, detail=\"任务未找到\")\n await _reload_scheduler_if_needed(service, scheduler_service)\n try:\n keyword = (task.keyword or \"\").strip()\n if keyword:\n remaining_tasks = await service.get_all_tasks()\n keyword_still_in_use = any(\n (remaining_task.keyword or \"\").strip() == keyword\n for remaining_task in remaining_tasks\n )\n if not keyword_still_in_use:\n await delete_result_file_records(build_result_filename(keyword))\n delete_price_snapshots(keyword)\n except Exception as e:\n print(f\"删除任务结果文件时出错: {e}\")\n\n try:\n log_file_path = resolve_task_log_path(task_id, task.task_name)\n if os.path.exists(log_file_path):\n os.remove(log_file_path)\n except Exception as e:\n print(f\"删除任务日志文件时出错: {e}\")\n return {\"message\": \"任务删除成功\"}\n@router.post(\"/start/{task_id}\", response_model=dict)\nasync def start_task(\n task_id: int,\n task_service: TaskService = Depends(get_task_service),\n process_service: ProcessService = Depends(get_process_service),\n):\n \"\"\"启动单个任务\"\"\"\n task = await task_service.get_task(task_id)\n if not task:\n raise HTTPException(status_code=404, detail=\"任务未找到\")\n if not task.enabled:\n raise HTTPException(status_code=400, detail=\"任务已被禁用，无法启动\")\n if task.is_running:\n raise HTTPException(status_code=400, detail=\"任务已在运行中\")\n success = await process_service.start_task(task_id, task.task_name)\n if not success:\n raise HTTPException(status_code=500, detail=\"启动任务失败\")\n return {\"message\": f\"任务 '{task.task_name}' 已启动\"}\n@router.post(\"/stop/{task_id}\", response_model=dict)\nasync def stop_task(\n task_id: int,\n task_service: TaskService = Depends(get_task_service),\n process_service: ProcessService = Depends(get_process_service),\n):\n \"\"\"停止单个任务\"\"\"\n task = await task_service.get_task(task_id)\n if not task:\n raise HTTPException(status_code=404, detail=\"任务未找到\")\n await process_service.stop_task(task_id)\n return {\"message\": f\"任务ID {task_id} 已发送停止信号\"}\n" }, { "path": "src/api/routes/websocket.py", "content": "\"\"\"\nWebSocket 路由\n提供实时通信功能\n\"\"\"\nfrom fastapi import APIRouter, WebSocket, WebSocketDisconnect\nfrom typing import Set\n\n\nrouter = APIRouter()\n\n# 全局 WebSocket 连接管理\nactive_connections: Set[WebSocket] = set()\n\n\n@router.websocket(\"/ws\")\nasync def websocket_endpoint(\n websocket: WebSocket,\n):\n \"\"\"WebSocket 端点\"\"\"\n # 接受连接\n await websocket.accept()\n active_connections.add(websocket)\n\n try:\n # 保持连接并接收消息\n while True:\n # 接收客户端消息（如果有的话）\n data = await websocket.receive_text()\n # 这里可以处理客户端发送的消息\n # 目前我们主要用于服务端推送，所以暂时不处理\n except WebSocketDisconnect:\n active_connections.remove(websocket)\n except Exception as e:\n print(f\"WebSocket 错误: {e}\")\n if websocket in active_connections:\n active_connections.remove(websocket)\n\n\nasync def broadcast_message(message_type: str, data: dict):\n \"\"\"向所有连接的客户端广播消息\"\"\"\n message = {\n \"type\": message_type,\n \"data\": data\n }\n\n # 移除已断开的连接\n disconnected = set()\n\n for connection in active_connections:\n try:\n await connection.send_json(message)\n except Exception:\n disconnected.add(connection)\n\n # 清理断开的连接\n for connection in disconnected:\n active_connections.discard(connection)\n" }, { "path": "src/app.py", "content": "\"\"\"\n新架构的主应用入口\n整合所有路由和服务\n\"\"\"\nfrom contextlib import asynccontextmanager\nfrom fastapi import FastAPI\nfrom fastapi.staticfiles import StaticFiles\nfrom fastapi.templating import Jinja2Templates\n\nfrom src.api.routes import (\n dashboard,\n tasks,\n logs,\n settings,\n prompts,\n results,\n login_state,\n websocket,\n accounts,\n)\nfrom src.api.dependencies import (\n set_process_service,\n set_scheduler_service,\n set_task_generation_service,\n)\nfrom src.services.task_service import TaskService\nfrom src.services.process_service import ProcessService\nfrom src.services.scheduler_service import SchedulerService\nfrom src.services.task_log_cleanup_service import cleanup_task_logs\nfrom src.services.task_generation_service import TaskGenerationService\nfrom src.infrastructure.persistence.sqlite_bootstrap import bootstrap_sqlite_storage\nfrom src.infrastructure.persistence.sqlite_task_repository import SqliteTaskRepository\nfrom src.infrastructure.config.settings import settings as app_settings\n\n\n# 全局服务实例\nprocess_service = ProcessService()\nscheduler_service = SchedulerService(process_service)\ntask_generation_service = TaskGenerationService()\n\n\nasync def _sync_task_runtime_status(task_id: int, is_running: bool) -> None:\n task_service = TaskService(SqliteTaskRepository())\n task = await task_service.get_task(task_id)\n if not task or task.is_running == is_running:\n return\n await task_service.update_task_status(task_id, is_running)\n await websocket.broadcast_message(\n \"task_status_changed\",\n {\"id\": task_id, \"is_running\": is_running},\n )\n\n\nprocess_service.set_lifecycle_hooks(\n on_started=lambda task_id: _sync_task_runtime_status(task_id, True),\n on_stopped=lambda task_id: _sync_task_runtime_status(task_id, False),\n)\n\n# 设置全局 ProcessService 实例供依赖注入使用\nset_process_service(process_service)\nset_scheduler_service(scheduler_service)\nset_task_generation_service(task_generation_service)\n\n\n@asynccontextmanager\nasync def lifespan(app: FastAPI):\n \"\"\"应用生命周期管理\"\"\"\n # 启动时\n print(\"正在启动应用...\")\n bootstrap_sqlite_storage()\n cleanup_task_logs(keep_days=app_settings.task_log_retention_days)\n\n # 重置所有任务状态为停止\n task_repo = SqliteTaskRepository()\n task_service = TaskService(task_repo)\n tasks_list = await task_service.get_all_tasks()\n\n for task in tasks_list:\n if task.is_running:\n await task_service.update_task_status(task.id, False)\n\n # 加载定时任务\n await scheduler_service.reload_jobs(tasks_list)\n scheduler_service.start()\n\n print(\"应用启动完成\")\n\n yield\n\n # 关闭时\n print(\"正在关闭应用...\")\n scheduler_service.stop()\n await process_service.stop_all()\n print(\"应用已关闭\")\n\n\n# 创建 FastAPI 应用\napp = FastAPI(\n title=\"闲鱼智能监控机器人\",\n description=\"基于AI的闲鱼商品监控系统\",\n version=\"2.0.0\",\n lifespan=lifespan\n)\n\n# 注册路由\napp.include_router(tasks.router)\napp.include_router(dashboard.router)\napp.include_router(logs.router)\napp.include_router(settings.router)\napp.include_router(prompts.router)\napp.include_router(results.router)\napp.include_router(login_state.router)\napp.include_router(websocket.router)\napp.include_router(accounts.router)\n\n# 挂载静态文件\n# 旧的静态文件目录（用于截图等）\napp.mount(\"/static\", StaticFiles(directory=\"static\"), name=\"static\")\n\n# 挂载 Vue 3 前端构建产物\n# 注意：需要在所有 API 路由之后挂载，以避免覆盖 API 路由\nimport os\nif os.path.exists(\"dist\"):\n app.mount(\"/assets\", StaticFiles(directory=\"dist/assets\"), name=\"assets\")\n\n\n# 健康检查端点\n@app.get(\"/health\")\nasync def health_check():\n \"\"\"健康检查（无需认证）\"\"\"\n return {\"status\": \"healthy\", \"message\": \"服务正常运行\"}\n\n\n# 认证状态检查端点\nfrom fastapi import Request, HTTPException\nfrom fastapi.responses import FileResponse\nfrom pydantic import BaseModel\n\nclass LoginRequest(BaseModel):\n username: str\n password: str\n\n\n@app.post(\"/auth/status\")\nasync def auth_status(payload: LoginRequest):\n \"\"\"检查认证状态\"\"\"\n if payload.username == app_settings.web_username and payload.password == app_settings.web_password:\n return {\"authenticated\": True, \"username\": payload.username}\n raise HTTPException(status_code=401, detail=\"认证失败\")\n\n\n# 主页路由 - 服务 Vue 3 SPA\nfrom fastapi.responses import JSONResponse\n\n@app.get(\"/\")\nasync def read_root(request: Request):\n \"\"\"提供 Vue 3 SPA 的主页面\"\"\"\n if os.path.exists(\"dist/index.html\"):\n return FileResponse(\"dist/index.html\")\n else:\n return JSONResponse(\n status_code=500,\n content={\"error\": \"前端构建产物不存在，请先运行 cd web-ui && npm run build\"}\n )\n\n\n# Catch-all 路由 - 处理所有前端路由（必须放在最后）\n@app.get(\"/{full_path:path}\")\nasync def serve_spa(request: Request, full_path: str):\n \"\"\"\n Catch-all 路由，将所有非 API 请求重定向到 index.html\n 这样可以支持 Vue Router 的 HTML5 History 模式\n \"\"\"\n # 如果请求的是静态资源（如 favicon.ico），返回 404\n if full_path.endswith(('.ico', '.png', '.jpg', '.jpeg', '.gif', '.svg', '.css', '.js', '.json')):\n return JSONResponse(status_code=404, content={\"error\": \"资源未找到\"})\n\n # 其他所有路径都返回 index.html，让前端路由处理\n if os.path.exists(\"dist/index.html\"):\n return FileResponse(\"dist/index.html\")\n else:\n return JSONResponse(\n status_code=500,\n content={\"error\": \"前端构建产物不存在，请先运行 cd web-ui && npm run build\"}\n )\n\n\nif __name__ == \"__main__\":\n import uvicorn\n from src.infrastructure.config.settings import settings\n\n print(f\"启动新架构应用，端口: {app_settings.server_port}\")\n uvicorn.run(app, host=\"0.0.0.0\", port=app_settings.server_port)\n" }, { "path": "src/config.py", "content": "import os\nimport sys\n\nfrom dotenv import load_dotenv\nfrom openai import AsyncOpenAI\n\n# --- AI & Notification Configuration ---\nload_dotenv()\n\n# --- File Paths & Directories ---\nSTATE_FILE = \"xianyu_state.json\"\nIMAGE_SAVE_DIR = \"images\"\nCONFIG_FILE = \"config.json\"\nos.makedirs(IMAGE_SAVE_DIR, exist_ok=True)\n\n# 任务隔离的临时图片目录前缀\nTASK_IMAGE_DIR_PREFIX = \"task_images_\"\n\n# --- API URL Patterns ---\nAPI_URL_PATTERN = \"h5api.m.goofish.com/h5/mtop.taobao.idlemtopsearch.pc.search\"\nDETAIL_API_URL_PATTERN = \"h5api.m.goofish.com/h5/mtop.taobao.idle.pc.detail\"\n\n# --- Environment Variables ---\nAPI_KEY = os.getenv(\"OPENAI_API_KEY\")\nBASE_URL = os.getenv(\"OPENAI_BASE_URL\")\nMODEL_NAME = os.getenv(\"OPENAI_MODEL_NAME\")\nPROXY_URL = os.getenv(\"PROXY_URL\")\nNTFY_TOPIC_URL = os.getenv(\"NTFY_TOPIC_URL\")\nGOTIFY_URL = os.getenv(\"GOTIFY_URL\")\nGOTIFY_TOKEN = os.getenv(\"GOTIFY_TOKEN\")\nBARK_URL = os.getenv(\"BARK_URL\")\nWX_BOT_URL = os.getenv(\"WX_BOT_URL\")\nTELEGRAM_BOT_TOKEN = os.getenv(\"TELEGRAM_BOT_TOKEN\")\nTELEGRAM_CHAT_ID = os.getenv(\"TELEGRAM_CHAT_ID\")\nWEBHOOK_URL = os.getenv(\"WEBHOOK_URL\")\nWEBHOOK_METHOD = os.getenv(\"WEBHOOK_METHOD\", \"POST\").upper()\nWEBHOOK_HEADERS = os.getenv(\"WEBHOOK_HEADERS\")\nWEBHOOK_CONTENT_TYPE = os.getenv(\"WEBHOOK_CONTENT_TYPE\", \"JSON\").upper()\nWEBHOOK_QUERY_PARAMETERS = os.getenv(\"WEBHOOK_QUERY_PARAMETERS\")\nWEBHOOK_BODY = os.getenv(\"WEBHOOK_BODY\")\nPCURL_TO_MOBILE = os.getenv(\"PCURL_TO_MOBILE\", \"false\").lower() == \"true\"\nRUN_HEADLESS = os.getenv(\"RUN_HEADLESS\", \"true\").lower() != \"false\"\nLOGIN_IS_EDGE = os.getenv(\"LOGIN_IS_EDGE\", \"false\").lower() == \"true\"\nRUNNING_IN_DOCKER = os.getenv(\"RUNNING_IN_DOCKER\", \"false\").lower() == \"true\"\nAI_DEBUG_MODE = os.getenv(\"AI_DEBUG_MODE\", \"false\").lower() == \"true\"\nSKIP_AI_ANALYSIS = os.getenv(\"SKIP_AI_ANALYSIS\", \"false\").lower() == \"true\"\nENABLE_THINKING = os.getenv(\"ENABLE_THINKING\", \"false\").lower() == \"true\"\nENABLE_RESPONSE_FORMAT = os.getenv(\"ENABLE_RESPONSE_FORMAT\", \"true\").lower() == \"true\"\n\n# --- Headers ---\nIMAGE_DOWNLOAD_HEADERS = {\n 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:139.0) Gecko/20100101 Firefox/139.0',\n 'Accept': 'image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8',\n 'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',\n 'Connection': 'keep-alive',\n 'Upgrade-Insecure-Requests': '1',\n}\n\n# --- Client Initialization ---\n# 检查配置是否齐全\nif not all([BASE_URL, MODEL_NAME]):\n print(\"警告：未在 .env 文件中完整设置 OPENAI_BASE_URL 和 OPENAI_MODEL_NAME。AI相关功能可能无法使用。\")\n client = None\nelse:\n try:\n if PROXY_URL:\n print(f\"正在为AI请求使用HTTP/S代理: {PROXY_URL}\")\n # httpx 会自动从环境变量中读取代理设置\n os.environ['HTTP_PROXY'] = PROXY_URL\n os.environ['HTTPS_PROXY'] = PROXY_URL\n\n # openai 客户端内部的 httpx 会自动从环境变量中获取代理配置\n client = AsyncOpenAI(api_key=API_KEY, base_url=BASE_URL)\n except Exception as e:\n print(f\"初始化 OpenAI 客户端时出错: {e}\")\n client = None\n\n# 检查AI客户端是否成功初始化\nif not client:\n # 在 prompt_generator.py 中，如果 client 为 None，会直接报错退出\n # 在 spider_v2.py 中，AI分析会跳过\n # 为了保持一致性，这里只打印警告，具体逻辑由调用方处理\n pass\n\n# 检查关键配置\nif not all([BASE_URL, MODEL_NAME]) and 'prompt_generator.py' in sys.argv[0]:\n sys.exit(\"错误：请确保在 .env 文件中完整设置了 OPENAI_BASE_URL 和 OPENAI_MODEL_NAME。(OPENAI_API_KEY 对于某些服务是可选的)\")\n\ndef get_ai_request_params(**kwargs):\n \"\"\"\n 构建AI请求参数，根据ENABLE_THINKING和ENABLE_RESPONSE_FORMAT环境变量决定是否添加相应参数\n \"\"\"\n if ENABLE_THINKING:\n kwargs[\"extra_body\"] = {\"enable_thinking\": False}\n \n # 如果禁用结构化输出，则移除 text.format 配置\n if not ENABLE_RESPONSE_FORMAT and \"text\" in kwargs:\n text_config = kwargs.get(\"text\")\n if isinstance(text_config, dict):\n text_config = dict(text_config)\n text_config.pop(\"format\", None)\n if text_config:\n kwargs[\"text\"] = text_config\n else:\n del kwargs[\"text\"]\n \n return kwargs\n" }, { "path": "src/core/cron_utils.py", "content": "\"\"\"\nCron 解析与校验工具。\n\"\"\"\nfrom __future__ import annotations\n\nfrom typing import Optional\n\nfrom apscheduler.triggers.cron import CronTrigger\n\nCRON_ALIASES = {\n \"@yearly\": \"0 0 1 1 *\",\n \"@annually\": \"0 0 1 1 *\",\n \"@monthly\": \"0 0 1 * *\",\n \"@weekly\": \"0 0 * * 0\",\n \"@daily\": \"0 0 * * *\",\n \"@midnight\": \"0 0 * * *\",\n \"@hourly\": \"0 * * * *\",\n}\n\nCRON_FORMAT_HINT = (\n \"Cron 表达式无效。支持 5 段（分 时 日 月 周）、\"\n \"6 段（秒 分 时 日 月 周）和常见别名（@hourly/@daily/@weekly/@monthly/@yearly）。\"\n \"示例：*/15 * * * *、0 8 * * *、0 0 8 * * *、@daily。\"\n)\n\n\ndef normalize_cron_expression(value: Optional[str]) -> Optional[str]:\n if value is None:\n return None\n\n normalized = \" \".join(str(value).strip().split())\n if not normalized:\n return None\n\n return CRON_ALIASES.get(normalized.lower(), normalized)\n\n\ndef build_cron_trigger(\n expression: str,\n *,\n timezone=None,\n) -> CronTrigger:\n normalized = normalize_cron_expression(expression)\n if normalized is None:\n raise ValueError(CRON_FORMAT_HINT)\n\n parts = normalized.split()\n try:\n if len(parts) == 5:\n return CronTrigger.from_crontab(normalized, timezone=timezone)\n\n if len(parts) == 6:\n second, minute, hour, day, month, day_of_week = parts\n return CronTrigger(\n second=second,\n minute=minute,\n hour=hour,\n day=day,\n month=month,\n day_of_week=day_of_week,\n timezone=timezone,\n )\n except ValueError as exc:\n raise ValueError(CRON_FORMAT_HINT) from exc\n\n raise ValueError(CRON_FORMAT_HINT)\n\n\ndef validate_cron_expression(value: Optional[str]) -> Optional[str]:\n normalized = normalize_cron_expression(value)\n if normalized is None:\n return None\n\n build_cron_trigger(normalized)\n return normalized\n" }, { "path": "src/domain/__init__.py", "content": "" }, { "path": "src/domain/models/__init__.py", "content": "from .task import Task, TaskCreate, TaskUpdate, TaskStatus\n\n__all__ = [\"Task\", \"TaskCreate\", \"TaskUpdate\", \"TaskStatus\"]\n" }, { "path": "src/domain/models/task.py", "content": "\"\"\"\n任务领域模型\n定义任务实体及其业务逻辑\n\"\"\"\nimport re\nfrom enum import Enum\nfrom typing import Any, List, Literal, Optional\n\nfrom pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator\n\nfrom src.core.cron_utils import validate_cron_expression\nfrom src.services.account_strategy_service import (\n clean_account_state_file,\n normalize_account_strategy,\n)\n\n\nclass TaskStatus(str, Enum):\n \"\"\"任务状态枚举\"\"\"\n\n STOPPED = \"stopped\"\n RUNNING = \"running\"\n SCHEDULED = \"scheduled\"\n\n\ndef _normalize_keyword_values(value) -> List[str]:\n if value is None:\n return []\n\n raw_values = []\n if isinstance(value, (list, tuple, set)):\n raw_values = list(value)\n elif isinstance(value, str):\n raw_values = re.split(r\"[\\n,]+\", value)\n else:\n raw_values = [value]\n\n normalized: List[str] = []\n seen = set()\n for item in raw_values:\n text = str(item).strip()\n if not text:\n continue\n dedup_key = text.lower()\n if dedup_key in seen:\n continue\n seen.add(dedup_key)\n normalized.append(text)\n return normalized\n\n\ndef _extract_keywords_from_legacy_groups(groups) -> List[str]:\n if not groups:\n return []\n\n merged: List[str] = []\n for group in groups:\n include_keywords = []\n if isinstance(group, dict):\n include_keywords = group.get(\"include_keywords\") or []\n else:\n include_keywords = getattr(group, \"include_keywords\", []) or []\n merged.extend(_normalize_keyword_values(include_keywords))\n return _normalize_keyword_values(merged)\n\n\ndef _normalize_payload_keywords(payload: Any) -> Any:\n if payload is None or not isinstance(payload, dict):\n return payload\n values = dict(payload)\n values[\"account_state_file\"] = clean_account_state_file(values.get(\"account_state_file\"))\n values[\"account_strategy\"] = normalize_account_strategy(\n values.get(\"account_strategy\"),\n values.get(\"account_state_file\"),\n )\n if \"keyword_rules\" in values:\n values[\"keyword_rules\"] = _normalize_keyword_values(values.get(\"keyword_rules\"))\n elif \"keyword_rule_groups\" in values:\n values[\"keyword_rules\"] = _extract_keywords_from_legacy_groups(\n values.get(\"keyword_rule_groups\")\n )\n return values\n\n\ndef _has_keyword_rules(keyword_rules: List[str]) -> bool:\n return bool(keyword_rules and len(keyword_rules) > 0)\n\n\ndef _normalize_optional_string(value):\n if value == \"\" or value == \"null\" or value == \"undefined\" or value is None:\n return None\n return value\n\n\ndef _validate_cron_expression(value: Optional[str]) -> Optional[str]:\n return validate_cron_expression(value)\n\n\ndef _normalize_price_value(value):\n if _normalize_optional_string(value) is None:\n return None\n if isinstance(value, (int, float)):\n return str(value)\n return value\n\n\nclass Task(BaseModel):\n \"\"\"任务实体\"\"\"\n\n model_config = ConfigDict(use_enum_values=True, extra=\"ignore\")\n\n id: Optional[int] = None\n task_name: str\n enabled: bool\n keyword: str\n description: Optional[str] = \"\"\n analyze_images: bool = True\n max_pages: int\n personal_only: bool\n min_price: Optional[str] = None\n max_price: Optional[str] = None\n cron: Optional[str] = None\n ai_prompt_base_file: str\n ai_prompt_criteria_file: str\n account_state_file: Optional[str] = None\n account_strategy: Literal[\"auto\", \"fixed\", \"rotate\"] = \"auto\"\n free_shipping: bool = True\n new_publish_option: Optional[str] = None\n region: Optional[str] = None\n decision_mode: Literal[\"ai\", \"keyword\"] = \"ai\"\n keyword_rules: List[str] = Field(default_factory=list)\n is_running: bool = False\n\n @model_validator(mode=\"before\")\n @classmethod\n def normalize_legacy_keyword_payload(cls, values):\n return _normalize_payload_keywords(values)\n\n @field_validator(\"keyword_rules\", mode=\"before\")\n @classmethod\n def normalize_keyword_rules(cls, value):\n return _normalize_keyword_values(value)\n\n def can_start(self) -> bool:\n \"\"\"检查任务是否可以启动\"\"\"\n return self.enabled and not self.is_running\n\n def can_stop(self) -> bool:\n \"\"\"检查任务是否可以停止\"\"\"\n return self.is_running\n\n def apply_update(self, update: \"TaskUpdate\") -> \"Task\":\n \"\"\"应用更新并返回新的任务实例\"\"\"\n update_data = update.model_dump(exclude_unset=True)\n return self.model_copy(update=update_data)\n\n\nclass TaskCreate(BaseModel):\n \"\"\"创建任务的DTO\"\"\"\n\n model_config = ConfigDict(extra=\"ignore\")\n\n task_name: str\n enabled: bool = True\n keyword: str\n description: Optional[str] = \"\"\n analyze_images: bool = True\n max_pages: int = 3\n personal_only: bool = True\n min_price: Optional[str] = None\n max_price: Optional[str] = None\n cron: Optional[str] = None\n ai_prompt_base_file: str = \"prompts/base_prompt.txt\"\n ai_prompt_criteria_file: str = \"\"\n account_state_file: Optional[str] = None\n account_strategy: Literal[\"auto\", \"fixed\", \"rotate\"] = \"auto\"\n free_shipping: bool = True\n new_publish_option: Optional[str] = None\n region: Optional[str] = None\n decision_mode: Literal[\"ai\", \"keyword\"] = \"ai\"\n keyword_rules: List[str] = Field(default_factory=list)\n\n @model_validator(mode=\"before\")\n @classmethod\n def normalize_legacy_keyword_payload(cls, values):\n return _normalize_payload_keywords(values)\n\n @field_validator(\"min_price\", \"max_price\", mode=\"before\")\n @classmethod\n def convert_price_to_str(cls, value):\n return _normalize_price_value(value)\n\n @field_validator(\"cron\", mode=\"before\")\n @classmethod\n def normalize_cron(cls, value):\n return _normalize_optional_string(value)\n\n @field_validator(\"account_state_file\", mode=\"before\")\n @classmethod\n def normalize_account_state_file(cls, value):\n return clean_account_state_file(value)\n\n @field_validator(\"cron\")\n @classmethod\n def validate_cron(cls, value):\n return _validate_cron_expression(value)\n\n @field_validator(\"keyword_rules\", mode=\"before\")\n @classmethod\n def normalize_keyword_rules(cls, value):\n return _normalize_keyword_values(value)\n\n @model_validator(mode=\"after\")\n def validate_decision_mode_payload(self):\n description = str(self.description or \"\").strip()\n if self.decision_mode == \"ai\" and not description:\n raise ValueError(\"AI 判断模式下，详细需求(description)不能为空。\")\n if self.decision_mode == \"keyword\" and not _has_keyword_rules(self.keyword_rules):\n raise ValueError(\"关键词判断模式下，至少需要一个关键词。\")\n if self.account_strategy == \"fixed\" and not self.account_state_file:\n raise ValueError(\"固定账号模式下必须选择账号。\")\n return self\n\n\nclass TaskUpdate(BaseModel):\n \"\"\"更新任务的DTO\"\"\"\n\n model_config = ConfigDict(extra=\"ignore\")\n\n task_name: Optional[str] = None\n enabled: Optional[bool] = None\n keyword: Optional[str] = None\n description: Optional[str] = None\n analyze_images: Optional[bool] = None\n max_pages: Optional[int] = None\n personal_only: Optional[bool] = None\n min_price: Optional[str] = None\n max_price: Optional[str] = None\n cron: Optional[str] = None\n ai_prompt_base_file: Optional[str] = None\n ai_prompt_criteria_file: Optional[str] = None\n account_state_file: Optional[str] = None\n account_strategy: Optional[Literal[\"auto\", \"fixed\", \"rotate\"]] = None\n free_shipping: Optional[bool] = None\n new_publish_option: Optional[str] = None\n region: Optional[str] = None\n decision_mode: Optional[Literal[\"ai\", \"keyword\"]] = None\n keyword_rules: Optional[List[str]] = None\n is_running: Optional[bool] = None\n\n @model_validator(mode=\"before\")\n @classmethod\n def normalize_legacy_keyword_payload(cls, values):\n return _normalize_payload_keywords(values)\n\n @field_validator(\"min_price\", \"max_price\", mode=\"before\")\n @classmethod\n def convert_price_to_str(cls, value):\n return _normalize_price_value(value)\n\n @field_validator(\"cron\", mode=\"before\")\n @classmethod\n def normalize_cron(cls, value):\n return _normalize_optional_string(value)\n\n @field_validator(\"account_state_file\", mode=\"before\")\n @classmethod\n def normalize_account_state_file(cls, value):\n return clean_account_state_file(value)\n\n @field_validator(\"cron\")\n @classmethod\n def validate_cron(cls, value):\n return _validate_cron_expression(value)\n\n @field_validator(\"keyword_rules\", mode=\"before\")\n @classmethod\n def normalize_keyword_rules(cls, value):\n return _normalize_keyword_values(value)\n\n @model_validator(mode=\"after\")\n def validate_partial_keyword_payload(self):\n if self.decision_mode == \"keyword\" and self.keyword_rules is not None:\n if not _has_keyword_rules(self.keyword_rules):\n raise ValueError(\"关键词判断模式下，至少需要一个关键词。\")\n if self.decision_mode == \"ai\" and self.description is not None:\n if not str(self.description).strip():\n raise ValueError(\"AI 判断模式下，详细需求(description)不能为空。\")\n return self\n\n\nclass TaskGenerateRequest(BaseModel):\n \"\"\"任务创建请求DTO（AI模式支持自动生成标准）\"\"\"\n\n model_config = ConfigDict(extra=\"ignore\")\n\n task_name: str\n keyword: str\n description: Optional[str] = \"\"\n analyze_images: bool = True\n personal_only: bool = True\n min_price: Optional[str] = None\n max_price: Optional[str] = None\n max_pages: int = 3\n cron: Optional[str] = None\n account_state_file: Optional[str] = None\n account_strategy: Literal[\"auto\", \"fixed\", \"rotate\"] = \"auto\"\n free_shipping: bool = True\n new_publish_option: Optional[str] = None\n region: Optional[str] = None\n decision_mode: Literal[\"ai\", \"keyword\"] = \"ai\"\n keyword_rules: List[str] = Field(default_factory=list)\n\n @model_validator(mode=\"before\")\n @classmethod\n def normalize_legacy_keyword_payload(cls, values):\n return _normalize_payload_keywords(values)\n\n @field_validator(\"min_price\", \"max_price\", mode=\"before\")\n @classmethod\n def convert_price_to_str(cls, value):\n return _normalize_price_value(value)\n\n @field_validator(\"cron\", mode=\"before\")\n @classmethod\n def empty_str_to_none(cls, value):\n return _normalize_optional_string(value)\n\n @field_validator(\"cron\")\n @classmethod\n def validate_cron(cls, value):\n return _validate_cron_expression(value)\n\n @field_validator(\"account_state_file\", mode=\"before\")\n @classmethod\n def empty_account_to_none(cls, value):\n return _normalize_optional_string(value)\n\n @field_validator(\"new_publish_option\", \"region\", mode=\"before\")\n @classmethod\n def empty_str_to_none_for_strings(cls, value):\n return _normalize_optional_string(value)\n\n @field_validator(\"keyword_rules\", mode=\"before\")\n @classmethod\n def normalize_keyword_rules(cls, value):\n return _normalize_keyword_values(value)\n\n @model_validator(mode=\"after\")\n def validate_decision_mode_payload(self):\n description = str(self.description or \"\").strip()\n if self.decision_mode == \"ai\" and not description:\n raise ValueError(\"AI 判断模式下，详细需求(description)不能为空。\")\n if self.decision_mode == \"keyword\" and not _has_keyword_rules(self.keyword_rules):\n raise ValueError(\"关键词判断模式下，至少需要一个关键词。\")\n if self.account_strategy == \"fixed\" and not self.account_state_file:\n raise ValueError(\"固定账号模式下必须选择账号。\")\n return self\n" }, { "path": "src/domain/models/task_generation.py", "content": "\"\"\"\n任务生成作业模型\n\"\"\"\nfrom typing import List, Literal, Optional\n\nfrom pydantic import BaseModel, Field\n\nfrom src.domain.models.task import Task\n\n\nTaskGenerationStatus = Literal[\"queued\", \"running\", \"completed\", \"failed\"]\nTaskGenerationStepStatus = Literal[\"pending\", \"running\", \"completed\", \"failed\"]\n\n\nclass TaskGenerationStep(BaseModel):\n \"\"\"单个任务生成步骤\"\"\"\n\n key: str\n label: str\n status: TaskGenerationStepStatus = \"pending\"\n message: str = \"\"\n\n\nclass TaskGenerationJob(BaseModel):\n \"\"\"任务生成作业\"\"\"\n\n job_id: str\n task_name: str\n status: TaskGenerationStatus = \"queued\"\n message: str = \"任务已排队，等待开始。\"\n current_step: Optional[str] = None\n steps: List[TaskGenerationStep] = Field(default_factory=list)\n task: Optional[Task] = None\n error: Optional[str] = None\n" }, { "path": "src/domain/repositories/__init__.py", "content": "" }, { "path": "src/domain/repositories/task_repository.py", "content": "\"\"\"\n任务仓储层\n负责任务数据的持久化操作\n\"\"\"\nfrom typing import List, Optional\nfrom abc import ABC, abstractmethod\nimport json\nimport aiofiles\nfrom src.domain.models.task import Task\n\n\nclass TaskRepository(ABC):\n \"\"\"任务仓储接口\"\"\"\n\n @abstractmethod\n async def find_all(self) -> List[Task]:\n \"\"\"获取所有任务\"\"\"\n pass\n\n @abstractmethod\n async def find_by_id(self, task_id: int) -> Optional[Task]:\n \"\"\"根据ID获取任务\"\"\"\n pass\n\n @abstractmethod\n async def save(self, task: Task) -> Task:\n \"\"\"保存任务（创建或更新）\"\"\"\n pass\n\n @abstractmethod\n async def delete(self, task_id: int) -> bool:\n \"\"\"删除任务\"\"\"\n pass\n" }, { "path": "src/failure_guard.py", "content": "\"\"\"Task-level failure circuit breaker.\n\n目标:\n- 当登录态失效/风控导致任务持续失败时，避免无限重试、避免高频请求。\n- 失败达到阈值后暂停任务一段时间。\n- 暂停期间最多每天通知一次，直到用户更新 cookies / 登录态文件后自动恢复。\n\n说明:\n- 仅使用标准库，既可被 API 主进程使用，也可被爬虫子进程使用。\n\"\"\"\n\nfrom __future__ import annotations\n\nimport json\nimport os\nimport time\nfrom dataclasses import dataclass\nfrom datetime import datetime, timedelta\nfrom typing import Any, Optional\n\n\ntry:\n from zoneinfo import ZoneInfo # py3.9+\n\n def _load_tz(name: str):\n return ZoneInfo(name)\n\n\nexcept Exception: # pragma: no cover\n\n def _load_tz(name: str):\n return None\n\n\ndef _as_int(value: Any, default: int) -> int:\n try:\n return int(value)\n except (TypeError, ValueError):\n return default\n\n\ndef _now(tz_name: str, now: Optional[datetime] = None) -> datetime:\n if now is not None:\n return now\n tz = _load_tz(tz_name)\n if tz is None:\n return datetime.now()\n return datetime.now(tz)\n\n\ndef _today_str(tz_name: str, now: Optional[datetime] = None) -> str:\n return _now(tz_name, now=now).date().isoformat()\n\n\ndef _dt_to_str(dt: Optional[datetime]) -> Optional[str]:\n if dt is None:\n return None\n return dt.isoformat()\n\n\ndef _str_to_dt(value: Optional[str]) -> Optional[datetime]:\n if not value:\n return None\n try:\n return datetime.fromisoformat(value)\n except ValueError:\n return None\n\n\ndef _get_mtime(path: Optional[str]) -> Optional[float]:\n if not path:\n return None\n try:\n return os.path.getmtime(path)\n except OSError:\n return None\n\n\ndef _cookie_changed(\n cookie_path: Optional[str], previous_mtime: Optional[float]\n) -> bool:\n if not cookie_path:\n return False\n current = _get_mtime(cookie_path)\n if current is None or previous_mtime is None:\n return False\n return current > (previous_mtime + 1e-6)\n\n\nclass _FileLock:\n def __init__(self, fh):\n self._fh = fh\n\n def __enter__(self):\n try:\n import fcntl\n\n fcntl.flock(self._fh.fileno(), fcntl.LOCK_EX)\n except Exception:\n pass\n return self\n\n def __exit__(self, exc_type, exc, tb):\n try:\n import fcntl\n\n fcntl.flock(self._fh.fileno(), fcntl.LOCK_UN)\n except Exception:\n pass\n return False\n\n\ndef _ensure_parent_dir(path: str) -> None:\n parent = os.path.dirname(path)\n if parent:\n os.makedirs(parent, exist_ok=True)\n\n\ndef _read_json_file(path: str) -> dict:\n try:\n with open(path, \"r\", encoding=\"utf-8\") as f:\n data = json.load(f)\n return data if isinstance(data, dict) else {}\n except FileNotFoundError:\n return {}\n except Exception:\n # 文件损坏时保留现场，避免无限解析失败。\n try:\n ts = str(int(time.time()))\n os.replace(path, f\"{path}.corrupt.{ts}\")\n except Exception:\n pass\n return {}\n\n\ndef _atomic_write_json(path: str, data: dict) -> None:\n _ensure_parent_dir(path)\n tmp = f\"{path}.tmp\"\n with open(tmp, \"w\", encoding=\"utf-8\") as f:\n json.dump(data, f, ensure_ascii=False, indent=2, sort_keys=True)\n f.flush()\n os.fsync(f.fileno())\n os.replace(tmp, path)\n\n\n@dataclass(frozen=True)\nclass SkipDecision:\n skip: bool\n should_notify: bool\n reason: str\n paused_until: Optional[datetime]\n consecutive_failures: int\n\n\nclass FailureGuard:\n def __init__(\n self,\n path: Optional[str] = None,\n *,\n threshold: Optional[int] = None,\n pause_seconds: Optional[int] = None,\n tz_name: Optional[str] = None,\n ):\n self.path = (\n path\n or os.getenv(\"TASK_FAILURE_GUARD_PATH\")\n or \"logs/task-failure-guard.json\"\n )\n self.threshold = max(\n 1, threshold or _as_int(os.getenv(\"TASK_FAILURE_THRESHOLD\"), 3)\n )\n self.pause_seconds = max(\n 60,\n pause_seconds\n or _as_int(os.getenv(\"TASK_FAILURE_PAUSE_SECONDS\"), 24 * 60 * 60),\n )\n self.tz_name = tz_name or os.getenv(\"TASK_FAILURE_TZ\") or \"Asia/Shanghai\"\n\n def _load(self) -> dict:\n data = _read_json_file(self.path)\n if \"tasks\" not in data or not isinstance(data.get(\"tasks\"), dict):\n data = {\"version\": 1, \"tasks\": {}}\n data.setdefault(\"version\", 1)\n return data\n\n def _save(self, data: dict) -> None:\n _atomic_write_json(self.path, data)\n\n def _update_task(self, task_key: str, updater) -> dict:\n _ensure_parent_dir(self.path)\n with open(self.path, \"a+\", encoding=\"utf-8\") as fh:\n with _FileLock(fh):\n fh.seek(0)\n data = self._load()\n tasks = data.setdefault(\"tasks\", {})\n entry = tasks.get(task_key) or {}\n if not isinstance(entry, dict):\n entry = {}\n entry = updater(entry) or entry\n tasks[task_key] = entry\n self._save(data)\n return entry\n\n def record_success(self, task_key: str, *, now: Optional[datetime] = None) -> None:\n def _reset(_: dict) -> dict:\n current = _now(self.tz_name, now=now)\n return {\n \"consecutive_failures\": 0,\n \"paused_until\": None,\n \"last_notified_date\": None,\n \"last_failure_reason\": None,\n \"last_failure_at\": None,\n \"last_success_at\": _dt_to_str(current),\n \"cookie_path\": None,\n \"cookie_mtime\": None,\n }\n\n self._update_task(task_key, _reset)\n\n def should_skip_start(\n self,\n task_key: str,\n *,\n cookie_path: Optional[str] = None,\n now: Optional[datetime] = None,\n ) -> SkipDecision:\n current = _now(self.tz_name, now=now)\n today = _today_str(self.tz_name, now=current)\n\n data = self._load()\n entry = (data.get(\"tasks\") or {}).get(task_key) or {}\n if not isinstance(entry, dict):\n entry = {}\n\n paused_until = _str_to_dt(entry.get(\"paused_until\"))\n consecutive = _as_int(entry.get(\"consecutive_failures\"), 0)\n last_reason = (entry.get(\"last_failure_reason\") or \"\").strip() or \"未知错误\"\n last_notified_date = entry.get(\"last_notified_date\")\n\n previous_cookie_mtime = entry.get(\"cookie_mtime\")\n if cookie_path and previous_cookie_mtime is not None:\n try:\n previous_cookie_mtime = float(previous_cookie_mtime)\n except (TypeError, ValueError):\n previous_cookie_mtime = None\n\n if (\n paused_until\n and paused_until > current\n and cookie_path\n and _cookie_changed(cookie_path, previous_cookie_mtime)\n ):\n # cookies / 登录态更新 => 自动恢复\n self.record_success(task_key, now=current)\n return SkipDecision(\n skip=False,\n should_notify=False,\n reason=\"cookie_updated\",\n paused_until=None,\n consecutive_failures=0,\n )\n\n if paused_until and current < paused_until:\n should_notify = last_notified_date != today\n\n if should_notify:\n\n def _touch(e: dict) -> dict:\n e = dict(e or {})\n e[\"last_notified_date\"] = today\n return e\n\n self._update_task(task_key, _touch)\n\n return SkipDecision(\n skip=True,\n should_notify=should_notify,\n reason=last_reason,\n paused_until=paused_until,\n consecutive_failures=consecutive,\n )\n\n return SkipDecision(\n skip=False,\n should_notify=False,\n reason=\"not_paused\",\n paused_until=None,\n consecutive_failures=consecutive,\n )\n\n def record_failure(\n self,\n task_key: str,\n reason: str,\n *,\n cookie_path: Optional[str] = None,\n min_failures_to_pause: Optional[int] = None,\n now: Optional[datetime] = None,\n ) -> dict:\n current = _now(self.tz_name, now=now)\n today = _today_str(self.tz_name, now=current)\n cookie_mtime = _get_mtime(cookie_path)\n\n effective_threshold = max(1, int(min_failures_to_pause or self.threshold))\n\n result = {\n \"should_notify\": False,\n \"opened_circuit\": False,\n \"paused_until\": None,\n \"consecutive_failures\": 0,\n }\n\n def _apply(entry: dict) -> dict:\n entry = dict(entry or {})\n previous_paused_until = _str_to_dt(entry.get(\"paused_until\"))\n was_paused = bool(previous_paused_until and current < previous_paused_until)\n\n prev_mtime = entry.get(\"cookie_mtime\")\n try:\n prev_mtime = float(prev_mtime) if prev_mtime is not None else None\n except (TypeError, ValueError):\n prev_mtime = None\n\n if cookie_path and _cookie_changed(cookie_path, prev_mtime):\n entry[\"consecutive_failures\"] = 0\n entry[\"paused_until\"] = None\n entry[\"last_notified_date\"] = None\n\n consecutive = _as_int(entry.get(\"consecutive_failures\"), 0) + 1\n entry[\"consecutive_failures\"] = consecutive\n entry[\"last_failure_reason\"] = (reason or \"未知错误\")[:1000]\n entry[\"last_failure_at\"] = _dt_to_str(current)\n if cookie_path:\n entry[\"cookie_path\"] = cookie_path\n if cookie_mtime is not None:\n entry[\"cookie_mtime\"] = cookie_mtime\n\n opened = False\n if consecutive >= effective_threshold:\n paused_until = current + timedelta(seconds=self.pause_seconds)\n entry[\"paused_until\"] = _dt_to_str(paused_until)\n opened = not was_paused\n\n if entry.get(\"last_notified_date\") != today:\n entry[\"last_notified_date\"] = today\n result[\"should_notify\"] = True\n\n result[\"paused_until\"] = paused_until\n else:\n entry[\"paused_until\"] = None\n\n result[\"opened_circuit\"] = opened\n result[\"consecutive_failures\"] = consecutive\n return entry\n\n self._update_task(task_key, _apply)\n return result\n" }, { "path": "src/infrastructure/__init__.py", "content": "" }, { "path": "src/infrastructure/config/__init__.py", "content": "from .settings import settings, AppSettings, AISettings, NotificationSettings\n\n__all__ = [\"settings\", \"AppSettings\", \"AISettings\", \"NotificationSettings\"]\n" }, { "path": "src/infrastructure/config/env_manager.py", "content": "\"\"\"\n环境变量管理器\n负责读取和更新 .env 文件，并在读取时回退到运行时环境变量\n\"\"\"\nimport os\nimport re\nfrom typing import Dict, List, Optional\nfrom pathlib import Path\n\nfrom dotenv import dotenv_values\n\n\n_PLAIN_ENV_VALUE_PATTERN = re.compile(r\"^[A-Za-z0-9_./:-]+$\")\n\n\nclass EnvManager:\n \"\"\"环境变量管理器\"\"\"\n\n def __init__(self, env_file: str = \".env\"):\n self.env_file = Path(env_file)\n self._ensure_env_file_exists()\n\n def _ensure_env_file_exists(self):\n \"\"\"确保 .env 文件存在\"\"\"\n if not self.env_file.exists():\n self.env_file.touch()\n\n def read_env(self) -> Dict[str, str]:\n \"\"\"读取所有环境变量\"\"\"\n if not self.env_file.exists():\n return {}\n\n loaded = dotenv_values(self.env_file, encoding=\"utf-8\")\n return {\n key: value\n for key, value in loaded.items()\n if key and value is not None\n }\n\n def get_value(self, key: str, default: Optional[str] = None) -> Optional[str]:\n \"\"\"获取单个环境变量的值，优先返回运行时环境变量\"\"\"\n runtime_value = os.getenv(key)\n if runtime_value is not None:\n return runtime_value\n\n env_vars = self.read_env()\n return env_vars.get(key, default)\n\n def update_values(self, updates: Dict[str, str]) -> bool:\n \"\"\"批量更新环境变量\"\"\"\n return self.apply_changes(updates=updates)\n\n def apply_changes(\n self,\n updates: Dict[str, str],\n deletions: List[str] | None = None,\n ) -> bool:\n \"\"\"批量更新并删除环境变量\"\"\"\n try:\n existing_vars = self.read_env()\n existing_vars.update(updates)\n for key in deletions or []:\n existing_vars.pop(key, None)\n return self._write_env(existing_vars)\n except Exception as e:\n print(f\"更新环境变量失败: {e}\")\n return False\n\n def set_value(self, key: str, value: str) -> bool:\n \"\"\"设置单个环境变量\"\"\"\n return self.update_values({key: value})\n\n def delete_keys(self, keys: List[str]) -> bool:\n \"\"\"删除指定的环境变量\"\"\"\n try:\n existing_vars = self.read_env()\n for key in keys:\n existing_vars.pop(key, None)\n return self._write_env(existing_vars)\n except Exception as e:\n print(f\"删除环境变量失败: {e}\")\n return False\n\n def _write_env(self, env_vars: Dict[str, str]) -> bool:\n \"\"\"写入环境变量到文件\"\"\"\n try:\n with open(self.env_file, 'w', encoding='utf-8') as f:\n for key, value in env_vars.items():\n f.write(f\"{key}={self._serialize_value(value)}\\n\")\n return True\n except Exception as e:\n print(f\"写入 .env 文件失败: {e}\")\n return False\n\n def _serialize_value(self, value: str) -> str:\n text = str(value)\n if text == \"\":\n return \"\"\n if _PLAIN_ENV_VALUE_PATTERN.fullmatch(text):\n return text\n escaped = text.replace(\"\\\\\", \"\\\\\\\\\").replace('\"', '\\\\\"').replace(\"\\n\", \"\\\\n\")\n return f'\"{escaped}\"'\n\n\n# 全局实例\nenv_manager = EnvManager()\n" }, { "path": "src/infrastructure/config/settings.py", "content": "\"\"\"\n统一配置管理模块\n使用 Pydantic 进行类型安全的配置管理\n\"\"\"\ntry:\n from pydantic_settings import BaseSettings, SettingsConfigDict\n _USING_PYDANTIC_SETTINGS = True\nexcept ImportError:\n from pydantic import BaseSettings\n _USING_PYDANTIC_SETTINGS = False\nfrom pydantic import Field\nfrom typing import Optional\nimport os\n\nDEFAULT_TELEGRAM_API_BASE_URL = \"https://api.telegram.org\"\n\n\ndef _env_field(default, env_name: str, **kwargs):\n if _USING_PYDANTIC_SETTINGS:\n return Field(default, validation_alias=env_name, **kwargs)\n return Field(default, env=env_name, **kwargs)\n\n\nif _USING_PYDANTIC_SETTINGS:\n class _EnvSettings(BaseSettings):\n model_config = SettingsConfigDict(\n env_file=\".env\",\n env_file_encoding=\"utf-8\",\n extra=\"ignore\",\n protected_namespaces=(),\n )\nelse:\n class _EnvSettings(BaseSettings):\n class Config:\n env_file = \".env\"\n env_file_encoding = \"utf-8\"\n extra = \"ignore\"\n protected_namespaces = ()\n\n\nclass AISettings(_EnvSettings):\n \"\"\"AI模型配置\"\"\"\n api_key: Optional[str] = _env_field(None, \"OPENAI_API_KEY\")\n base_url: str = _env_field(\"\", \"OPENAI_BASE_URL\")\n model_name: str = _env_field(\"\", \"OPENAI_MODEL_NAME\")\n proxy_url: Optional[str] = _env_field(None, \"PROXY_URL\")\n debug_mode: bool = _env_field(False, \"AI_DEBUG_MODE\")\n enable_response_format: bool = _env_field(True, \"ENABLE_RESPONSE_FORMAT\")\n enable_thinking: bool = _env_field(False, \"ENABLE_THINKING\")\n skip_analysis: bool = _env_field(False, \"SKIP_AI_ANALYSIS\")\n\n def is_configured(self) -> bool:\n \"\"\"检查AI是否已正确配置\"\"\"\n return bool(self.base_url and self.model_name)\n\n\nclass NotificationSettings(_EnvSettings):\n \"\"\"通知服务配置\"\"\"\n ntfy_topic_url: Optional[str] = _env_field(None, \"NTFY_TOPIC_URL\")\n gotify_url: Optional[str] = _env_field(None, \"GOTIFY_URL\")\n gotify_token: Optional[str] = _env_field(None, \"GOTIFY_TOKEN\")\n bark_url: Optional[str] = _env_field(None, \"BARK_URL\")\n wx_bot_url: Optional[str] = _env_field(None, \"WX_BOT_URL\")\n telegram_bot_token: Optional[str] = _env_field(None, \"TELEGRAM_BOT_TOKEN\")\n telegram_chat_id: Optional[str] = _env_field(None, \"TELEGRAM_CHAT_ID\")\n telegram_api_base_url: Optional[str] = _env_field(\n DEFAULT_TELEGRAM_API_BASE_URL,\n \"TELEGRAM_API_BASE_URL\",\n )\n webhook_url: Optional[str] = _env_field(None, \"WEBHOOK_URL\")\n webhook_method: str = _env_field(\"POST\", \"WEBHOOK_METHOD\")\n webhook_headers: Optional[str] = _env_field(None, \"WEBHOOK_HEADERS\")\n webhook_content_type: str = _env_field(\"JSON\", \"WEBHOOK_CONTENT_TYPE\")\n webhook_query_parameters: Optional[str] = _env_field(None, \"WEBHOOK_QUERY_PARAMETERS\")\n webhook_body: Optional[str] = _env_field(None, \"WEBHOOK_BODY\")\n pcurl_to_mobile: bool = _env_field(True, \"PCURL_TO_MOBILE\")\n\n def has_any_notification_enabled(self) -> bool:\n \"\"\"检查是否配置了任何通知服务\"\"\"\n return any([\n self.ntfy_topic_url,\n self.wx_bot_url,\n self.gotify_url and self.gotify_token,\n self.bark_url,\n self.telegram_bot_token and self.telegram_chat_id,\n self.webhook_url\n ])\n\n\nclass ScraperSettings(_EnvSettings):\n \"\"\"爬虫相关配置\"\"\"\n run_headless: bool = _env_field(True, \"RUN_HEADLESS\")\n login_is_edge: bool = _env_field(False, \"LOGIN_IS_EDGE\")\n running_in_docker: bool = _env_field(False, \"RUNNING_IN_DOCKER\")\n state_file: str = _env_field(\"xianyu_state.json\", \"STATE_FILE\")\n\n\nclass AppSettings(_EnvSettings):\n \"\"\"应用主配置\"\"\"\n server_port: int = _env_field(8000, \"SERVER_PORT\")\n web_username: str = _env_field(\"admin\", \"WEB_USERNAME\")\n web_password: str = _env_field(\"admin123\", \"WEB_PASSWORD\")\n task_log_retention_days: int = _env_field(7, \"TASK_LOG_RETENTION_DAYS\", ge=1)\n\n # 文件路径配置\n config_file: str = \"config.json\"\n image_save_dir: str = \"images\"\n task_image_dir_prefix: str = \"task_images_\"\n\n def __init__(self, **kwargs):\n super().__init__(**kwargs)\n # 创建必要的目录\n os.makedirs(self.image_save_dir, exist_ok=True)\n\n\n# 全局配置实例（单例模式）\n_settings_instance = None\n\ndef get_settings() -> AppSettings:\n \"\"\"获取全局配置实例\"\"\"\n global _settings_instance\n if _settings_instance is None:\n _settings_instance = AppSettings()\n return _settings_instance\n\n\ndef reload_settings() -> None:\n \"\"\"重新加载全局配置实例\"\"\"\n global _settings_instance, settings, ai_settings, notification_settings, scraper_settings\n from dotenv import load_dotenv\n from src.infrastructure.config.env_manager import env_manager\n\n load_dotenv(dotenv_path=env_manager.env_file, override=True)\n _settings_instance = None\n settings = get_settings()\n ai_settings = AISettings()\n notification_settings = NotificationSettings()\n scraper_settings = ScraperSettings()\n\n\n# 导出便捷访问的配置实例\nsettings = get_settings()\nai_settings = AISettings()\nnotification_settings = NotificationSettings()\nscraper_settings = ScraperSettings()\n" }, { "path": "src/infrastructure/external/__init__.py", "content": "" }, { "path": "src/infrastructure/external/ai_client.py", "content": "\"\"\"\nAI 客户端封装\n提供统一的 AI 调用接口\n\"\"\"\nimport os\nimport json\nimport base64\nfrom typing import Dict, List, Optional\nfrom datetime import datetime\nfrom dotenv import load_dotenv\nfrom openai import AsyncOpenAI\nfrom src.ai_message_builder import (\n build_analysis_text_prompt,\n build_user_message_content,\n)\nfrom src.infrastructure.config.settings import AISettings\nfrom src.infrastructure.config.env_manager import env_manager\nfrom src.services.ai_request_compat import (\n CHAT_COMPLETIONS_API_MODE,\n RESPONSES_API_MODE,\n build_ai_request_params,\n create_ai_response_async,\n is_chat_completions_api_unsupported_error,\n is_json_output_unsupported_error,\n is_responses_api_unsupported_error,\n is_temperature_unsupported_error,\n remove_temperature_param,\n)\nfrom src.services.ai_response_parser import (\n EmptyAIResponseError,\n extract_ai_response_content,\n parse_ai_response_json,\n)\n\n\nclass AIClient:\n \"\"\"AI 客户端封装\"\"\"\n\n def __init__(self):\n self.settings: Optional[AISettings] = None\n self.client: Optional[AsyncOpenAI] = None\n self.refresh()\n\n def _load_settings(self) -> None:\n load_dotenv(dotenv_path=env_manager.env_file, override=True)\n self.settings = AISettings()\n\n def refresh(self) -> None:\n self._load_settings()\n self.client = self._initialize_client()\n\n def _initialize_client(self) -> Optional[AsyncOpenAI]:\n \"\"\"初始化 OpenAI 客户端\"\"\"\n if not self.settings or not self.settings.is_configured():\n print(\"警告：AI 配置不完整，AI 功能将不可用\")\n return None\n\n try:\n if self.settings.proxy_url:\n print(f\"正在为 AI 请求使用代理: {self.settings.proxy_url}\")\n os.environ['HTTP_PROXY'] = self.settings.proxy_url\n os.environ['HTTPS_PROXY'] = self.settings.proxy_url\n\n return AsyncOpenAI(\n api_key=self.settings.api_key,\n base_url=self.settings.base_url\n )\n except Exception as e:\n print(f\"初始化 AI 客户端失败: {e}\")\n return None\n\n def is_available(self) -> bool:\n \"\"\"检查 AI 客户端是否可用\"\"\"\n return self.client is not None\n\n async def close(self) -> None:\n \"\"\"关闭底层异步客户端，避免事件循环结束后再触发清理。\"\"\"\n client = self.client\n self.client = None\n if client is None:\n return\n\n close = getattr(client, \"close\", None)\n if close is None:\n return\n await close()\n\n @staticmethod\n def encode_image(image_path: str) -> Optional[str]:\n \"\"\"将图片编码为 Base64\"\"\"\n if not image_path or not os.path.exists(image_path):\n return None\n try:\n with open(image_path, \"rb\") as f:\n return base64.b64encode(f.read()).decode('utf-8')\n except Exception as e:\n print(f\"编码图片失败: {e}\")\n return None\n\n async def analyze(\n self,\n product_data: Dict,\n image_paths: List[str],\n prompt_text: str\n ) -> Optional[Dict]:\n \"\"\"\n 分析商品数据\n\n Args:\n product_data: 商品数据\n image_paths: 图片路径列表\n prompt_text: 分析提示词\n\n Returns:\n 分析结果\n \"\"\"\n if not self.is_available():\n print(\"AI 客户端不可用\")\n return None\n\n try:\n messages = self._build_messages(product_data, image_paths, prompt_text)\n response = await self._call_ai(messages)\n return self._parse_response(response)\n except Exception as e:\n print(f\"AI 分析失败: {e}\")\n return None\n\n def _build_messages(self, product_data: Dict, image_paths: List[str], prompt_text: str) -> List[Dict]:\n \"\"\"构建 AI 消息\"\"\"\n product_json = json.dumps(product_data, ensure_ascii=False, indent=2)\n image_data_urls: List[str] = []\n for path in image_paths:\n base64_img = self.encode_image(path)\n if base64_img:\n image_data_urls.append(f\"data:image/jpeg;base64,{base64_img}\")\n\n text_prompt = build_analysis_text_prompt(\n product_json,\n prompt_text,\n include_images=bool(image_data_urls),\n )\n user_content = build_user_message_content(text_prompt, image_data_urls)\n return [{\"role\": \"user\", \"content\": user_content}]\n\n async def _call_ai(\n self,\n messages: List[Dict],\n *,\n temperature: float = 0.1,\n max_output_tokens: int = 4000,\n enable_json_output: Optional[bool] = None,\n ) -> str:\n \"\"\"调用 AI API\"\"\"\n api_mode = CHAT_COMPLETIONS_API_MODE\n use_response_format = (\n self.settings.enable_response_format\n if enable_json_output is None\n else enable_json_output\n )\n use_temperature = True\n max_attempts = 4\n\n for attempt in range(max_attempts):\n request_params = build_ai_request_params(\n api_mode,\n model=self.settings.model_name,\n messages=messages,\n temperature=temperature,\n max_output_tokens=max_output_tokens,\n enable_json_output=use_response_format,\n )\n if not use_temperature:\n request_params = remove_temperature_param(request_params)\n\n if self.settings.enable_thinking:\n request_params[\"extra_body\"] = {\"enable_thinking\": False}\n\n try:\n response = await create_ai_response_async(\n self.client,\n api_mode,\n request_params,\n )\n return extract_ai_response_content(response)\n except EmptyAIResponseError as exc:\n if attempt < max_attempts - 1:\n print(\n f\"AI响应为空，正在自动重试 ({attempt + 2}/{max_attempts})\"\n )\n continue\n raise exc\n except Exception as exc:\n changed = False\n if (\n api_mode == CHAT_COMPLETIONS_API_MODE\n and is_chat_completions_api_unsupported_error(exc)\n ):\n api_mode = RESPONSES_API_MODE\n changed = True\n print(\"当前服务未实现 Chat Completions API，正在自动回退到 Responses API\")\n elif (\n api_mode == RESPONSES_API_MODE\n and is_responses_api_unsupported_error(exc)\n ):\n api_mode = CHAT_COMPLETIONS_API_MODE\n changed = True\n print(\"当前服务未实现 Responses API，正在自动回退到 Chat Completions API\")\n if use_response_format and is_json_output_unsupported_error(exc):\n use_response_format = False\n changed = True\n print(\"当前模型不支持结构化 JSON 输出，正在自动重试并移除该参数\")\n if use_temperature and is_temperature_unsupported_error(exc):\n use_temperature = False\n changed = True\n print(\"当前模型不支持 temperature 参数，正在自动重试并移除该参数\")\n if changed and attempt < max_attempts - 1:\n continue\n raise\n\n raise RuntimeError(\"AI 调用在兼容性重试后仍未返回结果\")\n\n def _parse_response(self, response_text: str) -> Optional[Dict]:\n \"\"\"解析 AI 响应\"\"\"\n try:\n return parse_ai_response_json(response_text)\n except json.JSONDecodeError:\n print(f\"无法解析 AI 响应: {response_text[:100]}\")\n return None\n" }, { "path": "src/infrastructure/external/notification_clients/__init__.py", "content": "from .base import NotificationClient, NotificationMessage\nfrom .bark_client import BarkClient\nfrom .gotify_client import GotifyClient\nfrom .ntfy_client import NtfyClient\nfrom .telegram_client import TelegramClient\nfrom .wecom_bot_client import WeComBotClient\nfrom .webhook_client import WebhookClient\n\n__all__ = [\n \"NotificationClient\",\n \"NotificationMessage\",\n \"BarkClient\",\n \"GotifyClient\",\n \"NtfyClient\",\n \"TelegramClient\",\n \"WeComBotClient\",\n \"WebhookClient\",\n]\n" }, { "path": "src/infrastructure/external/notification_clients/bark_client.py", "content": "\"\"\"\nBark 通知客户端\n\"\"\"\nimport asyncio\nimport requests\nfrom typing import Dict\nfrom .base import NotificationClient\n\n\nclass BarkClient(NotificationClient):\n \"\"\"Bark 通知客户端\"\"\"\n\n channel_key = \"bark\"\n display_name = \"Bark\"\n\n def __init__(self, bark_url: str = None, pcurl_to_mobile: bool = True):\n super().__init__(enabled=bool(bark_url), pcurl_to_mobile=pcurl_to_mobile)\n self.bark_url = bark_url\n\n async def send(self, product_data: Dict, reason: str) -> None:\n \"\"\"发送 Bark 通知\"\"\"\n if not self.is_enabled():\n raise RuntimeError(\"Bark 未启用\")\n\n message = self._build_message(product_data, reason)\n bark_payload = {\n \"title\": message.notification_title,\n \"body\": message.content,\n \"url\": message.mobile_link or message.desktop_link,\n \"level\": \"timeSensitive\",\n \"group\": \"闲鱼监控\"\n }\n\n if message.image_url:\n bark_payload[\"icon\"] = message.image_url\n\n headers = {\"Content-Type\": \"application/json; charset=utf-8\"}\n loop = asyncio.get_running_loop()\n response = await loop.run_in_executor(\n None,\n lambda: requests.post(\n self.bark_url,\n json=bark_payload,\n headers=headers,\n timeout=10\n )\n )\n response.raise_for_status()\n" }, { "path": "src/infrastructure/external/notification_clients/base.py", "content": "\"\"\"\n通知客户端基类\n定义通知客户端的统一接口\n\"\"\"\nfrom abc import ABC, abstractmethod\nfrom dataclasses import dataclass\nfrom typing import Dict\n\nfrom src.utils import convert_goofish_link\n\n\n@dataclass(frozen=True)\nclass NotificationMessage:\n title: str\n price: str\n reason: str\n desktop_link: str\n mobile_link: str | None\n notification_title: str\n content: str\n image_url: str | None\n\n\nclass NotificationClient(ABC):\n \"\"\"通知客户端抽象基类\"\"\"\n\n channel_key = \"unknown\"\n display_name = \"未知渠道\"\n\n def __init__(self, enabled: bool = False, pcurl_to_mobile: bool = True):\n self._enabled = enabled\n self._pcurl_to_mobile = pcurl_to_mobile\n\n def is_enabled(self) -> bool:\n \"\"\"检查客户端是否启用\"\"\"\n return self._enabled\n\n @abstractmethod\n async def send(self, product_data: Dict, reason: str) -> bool:\n \"\"\"\n 发送通知\n\n Args:\n product_data: 商品数据\n reason: 推荐原因\n\n Returns:\n 是否发送成功\n \"\"\"\n raise NotImplementedError\n\n def _build_message(self, product_data: Dict, reason: str) -> NotificationMessage:\n \"\"\"格式化消息内容\"\"\"\n title = product_data.get('商品标题', 'N/A')\n price = product_data.get('当前售价', 'N/A')\n desktop_link = product_data.get('商品链接', '#')\n mobile_link = None\n\n if self._pcurl_to_mobile and desktop_link and desktop_link != \"#\":\n mobile_link = convert_goofish_link(desktop_link)\n\n content_lines = [\n f\"价格: {price}\",\n f\"原因: {reason}\",\n ]\n if mobile_link:\n content_lines.append(f\"手机端链接: {mobile_link}\")\n content_lines.append(f\"电脑端链接: {desktop_link}\")\n else:\n content_lines.append(f\"链接: {desktop_link}\")\n\n short_title = title[:30]\n suffix = \"...\" if len(title) > 30 else \"\"\n notification_title = f\"🚨 新推荐! {short_title}{suffix}\"\n\n main_image = product_data.get('商品主图链接')\n if not main_image:\n image_list = product_data.get('商品图片列表', [])\n if image_list:\n main_image = image_list[0]\n\n return NotificationMessage(\n title=title,\n price=price,\n reason=reason,\n desktop_link=desktop_link,\n mobile_link=mobile_link,\n notification_title=notification_title,\n content=\"\\n\".join(content_lines),\n image_url=main_image,\n )\n" }, { "path": "src/infrastructure/external/notification_clients/factory.py", "content": "\"\"\"\n通知客户端工厂\n\"\"\"\nfrom src.infrastructure.config.settings import NotificationSettings\n\nfrom .bark_client import BarkClient\nfrom .gotify_client import GotifyClient\nfrom .ntfy_client import NtfyClient\nfrom .telegram_client import TelegramClient\nfrom .wecom_bot_client import WeComBotClient\nfrom .webhook_client import WebhookClient\n\n\ndef build_notification_clients(settings: NotificationSettings):\n pcurl_to_mobile = settings.pcurl_to_mobile\n return [\n NtfyClient(settings.ntfy_topic_url, pcurl_to_mobile=pcurl_to_mobile),\n BarkClient(settings.bark_url, pcurl_to_mobile=pcurl_to_mobile),\n GotifyClient(\n settings.gotify_url,\n settings.gotify_token,\n pcurl_to_mobile=pcurl_to_mobile,\n ),\n WeComBotClient(settings.wx_bot_url, pcurl_to_mobile=pcurl_to_mobile),\n TelegramClient(\n settings.telegram_bot_token,\n settings.telegram_chat_id,\n settings.telegram_api_base_url,\n pcurl_to_mobile=pcurl_to_mobile,\n ),\n WebhookClient(\n settings.webhook_url,\n webhook_method=settings.webhook_method,\n webhook_headers=settings.webhook_headers,\n webhook_content_type=settings.webhook_content_type,\n webhook_query_parameters=settings.webhook_query_parameters,\n webhook_body=settings.webhook_body,\n pcurl_to_mobile=pcurl_to_mobile,\n ),\n ]\n" }, { "path": "src/infrastructure/external/notification_clients/gotify_client.py", "content": "\"\"\"\nGotify 通知客户端\n\"\"\"\nimport asyncio\nfrom typing import Dict\n\nimport requests\n\nfrom .base import NotificationClient\n\n\nclass GotifyClient(NotificationClient):\n \"\"\"Gotify 通知客户端\"\"\"\n\n channel_key = \"gotify\"\n display_name = \"Gotify\"\n\n def __init__(\n self,\n gotify_url: str | None = None,\n gotify_token: str | None = None,\n pcurl_to_mobile: bool = True,\n ):\n super().__init__(\n enabled=bool(gotify_url and gotify_token),\n pcurl_to_mobile=pcurl_to_mobile,\n )\n self.gotify_url = (gotify_url or \"\").rstrip(\"/\")\n self.gotify_token = gotify_token\n\n async def send(self, product_data: Dict, reason: str) -> None:\n if not self.is_enabled():\n raise RuntimeError(\"Gotify 未启用\")\n\n message = self._build_message(product_data, reason)\n payload = {\n \"title\": (None, message.notification_title),\n \"message\": (None, message.content),\n \"priority\": (None, \"5\"),\n }\n final_url = f\"{self.gotify_url}/message?token={self.gotify_token}\"\n loop = asyncio.get_running_loop()\n response = await loop.run_in_executor(\n None,\n lambda: requests.post(final_url, files=payload, timeout=10),\n )\n response.raise_for_status()\n" }, { "path": "src/infrastructure/external/notification_clients/ntfy_client.py", "content": "\"\"\"\nNtfy 通知客户端\n\"\"\"\nimport asyncio\nimport requests\nfrom typing import Dict\nfrom .base import NotificationClient\n\n\nclass NtfyClient(NotificationClient):\n \"\"\"Ntfy 通知客户端\"\"\"\n\n channel_key = \"ntfy\"\n display_name = \"Ntfy\"\n\n def __init__(self, topic_url: str = None, pcurl_to_mobile: bool = True):\n super().__init__(enabled=bool(topic_url), pcurl_to_mobile=pcurl_to_mobile)\n self.topic_url = topic_url\n\n async def send(self, product_data: Dict, reason: str) -> None:\n \"\"\"发送 Ntfy 通知\"\"\"\n if not self.is_enabled():\n raise RuntimeError(\"Ntfy 未启用\")\n\n message = self._build_message(product_data, reason)\n loop = asyncio.get_running_loop()\n response = await loop.run_in_executor(\n None,\n lambda: requests.post(\n self.topic_url,\n data=message.content.encode('utf-8'),\n headers={\n \"Title\": message.notification_title.encode('utf-8'),\n \"Priority\": \"urgent\",\n \"Tags\": \"bell,vibration\"\n },\n timeout=10\n )\n )\n response.raise_for_status()\n" }, { "path": "src/infrastructure/external/notification_clients/telegram_client.py", "content": "\"\"\"\nTelegram 通知客户端\n\"\"\"\nimport asyncio\nfrom typing import Dict\n\nimport requests\n\nfrom src.infrastructure.config.settings import DEFAULT_TELEGRAM_API_BASE_URL\n\nfrom .base import NotificationClient\n\n\nclass TelegramClient(NotificationClient):\n \"\"\"Telegram 通知客户端\"\"\"\n\n channel_key = \"telegram\"\n display_name = \"Telegram\"\n\n def __init__(\n self,\n bot_token: str = None,\n chat_id: str = None,\n api_base_url: str = DEFAULT_TELEGRAM_API_BASE_URL,\n pcurl_to_mobile: bool = True,\n ):\n super().__init__(enabled=bool(bot_token and chat_id), pcurl_to_mobile=pcurl_to_mobile)\n self.bot_token = bot_token\n self.chat_id = chat_id\n self.api_base_url = (\n (api_base_url or DEFAULT_TELEGRAM_API_BASE_URL).rstrip(\"/\")\n )\n\n async def send(self, product_data: Dict, reason: str) -> None:\n \"\"\"发送 Telegram 通知\"\"\"\n if not self.is_enabled():\n raise RuntimeError(\"Telegram 未启用\")\n\n message = self._build_message(product_data, reason)\n telegram_message = [\n \"🚨 新推荐!\",\n \"\",\n f\"{message.title[:50]}{'...' if len(message.title) > 50 else ''}\",\n \"\",\n f\"💰 价格: {message.price}\",\n f\"📝 原因: {message.reason}\",\n ]\n if message.mobile_link:\n telegram_message.append(f\"📱 手机端链接\")\n telegram_message.append(f\"💻 电脑端链接\")\n\n telegram_api_url = f\"{self.api_base_url}/bot{self.bot_token}/sendMessage\"\n telegram_payload = {\n \"chat_id\": self.chat_id,\n \"text\": \"\\n\".join(telegram_message),\n \"parse_mode\": \"HTML\",\n \"disable_web_page_preview\": False\n }\n\n headers = {\"Content-Type\": \"application/json\"}\n loop = asyncio.get_running_loop()\n response = await loop.run_in_executor(\n None,\n lambda: requests.post(\n telegram_api_url,\n json=telegram_payload,\n headers=headers,\n timeout=10\n )\n )\n response.raise_for_status()\n result = response.json()\n if not result.get(\"ok\"):\n raise RuntimeError(result.get(\"description\", \"Telegram 返回未知错误\"))\n" }, { "path": "src/infrastructure/external/notification_clients/webhook_client.py", "content": "\"\"\"\n通用 Webhook 通知客户端\n\"\"\"\nimport asyncio\nimport json\nfrom typing import Any, Dict\nfrom urllib.parse import parse_qsl, urlencode, urlparse, urlunparse\n\nimport requests\n\nfrom .base import NotificationClient, NotificationMessage\n\n\nclass WebhookClient(NotificationClient):\n \"\"\"通用 Webhook 通知客户端\"\"\"\n\n channel_key = \"webhook\"\n display_name = \"Webhook\"\n\n def __init__(\n self,\n webhook_url: str | None = None,\n webhook_method: str = \"POST\",\n webhook_headers: str | None = None,\n webhook_content_type: str = \"JSON\",\n webhook_query_parameters: str | None = None,\n webhook_body: str | None = None,\n pcurl_to_mobile: bool = True,\n ):\n super().__init__(enabled=bool(webhook_url), pcurl_to_mobile=pcurl_to_mobile)\n self.webhook_url = webhook_url\n self.webhook_method = (webhook_method or \"POST\").upper()\n self.webhook_headers = webhook_headers\n self.webhook_content_type = (webhook_content_type or \"JSON\").upper()\n self.webhook_query_parameters = webhook_query_parameters\n self.webhook_body = webhook_body\n\n async def send(self, product_data: Dict, reason: str) -> None:\n if not self.is_enabled():\n raise RuntimeError(\"Webhook 未启用\")\n\n message = self._build_message(product_data, reason)\n headers = self._parse_json(self.webhook_headers, \"WEBHOOK_HEADERS\", expect_dict=True) or {}\n final_url = self._build_url(message)\n loop = asyncio.get_running_loop()\n\n if self.webhook_method == \"GET\":\n response = await loop.run_in_executor(\n None,\n lambda: requests.get(final_url, headers=headers, timeout=15),\n )\n response.raise_for_status()\n return\n\n json_payload, form_payload = self._build_body(message, headers)\n response = await loop.run_in_executor(\n None,\n lambda: requests.post(\n final_url,\n headers=headers,\n json=json_payload,\n data=form_payload,\n timeout=15,\n ),\n )\n response.raise_for_status()\n\n def _build_url(self, message: NotificationMessage) -> str:\n params = self._parse_json(\n self.webhook_query_parameters,\n \"WEBHOOK_QUERY_PARAMETERS\",\n expect_dict=True,\n ) or {}\n rendered = self._render_template(params, message)\n parsed_url = list(urlparse(self.webhook_url))\n query = dict(parse_qsl(parsed_url[4]))\n query.update(rendered)\n parsed_url[4] = urlencode(query)\n return urlunparse(parsed_url)\n\n def _build_body(\n self,\n message: NotificationMessage,\n headers: Dict[str, str],\n ) -> tuple[Any | None, Any | None]:\n if not self.webhook_body:\n return None, None\n\n body_template = self._parse_json(self.webhook_body, \"WEBHOOK_BODY\")\n rendered_body = self._render_template(body_template, message)\n\n if self.webhook_content_type == \"JSON\":\n if \"Content-Type\" not in headers and \"content-type\" not in headers:\n headers[\"Content-Type\"] = \"application/json; charset=utf-8\"\n return rendered_body, None\n\n if self.webhook_content_type == \"FORM\":\n if not isinstance(rendered_body, dict):\n raise ValueError(\"WEBHOOK_BODY 在 FORM 模式下必须是 JSON 对象\")\n if \"Content-Type\" not in headers and \"content-type\" not in headers:\n headers[\"Content-Type\"] = \"application/x-www-form-urlencoded\"\n return None, rendered_body\n\n raise ValueError(f\"不支持的 WEBHOOK_CONTENT_TYPE: {self.webhook_content_type}\")\n\n def _parse_json(\n self,\n raw_value: str | None,\n field_name: str,\n expect_dict: bool = False,\n ) -> Any | None:\n if not raw_value:\n return None\n try:\n parsed = json.loads(raw_value)\n except json.JSONDecodeError as exc:\n raise ValueError(f\"{field_name} 不是合法 JSON: {exc.msg}\") from exc\n if expect_dict and not isinstance(parsed, dict):\n raise ValueError(f\"{field_name} 必须是 JSON 对象\")\n return parsed\n\n def _render_template(self, value: Any, message: NotificationMessage) -> Any:\n if isinstance(value, str):\n return self._replace_placeholders(value, message)\n if isinstance(value, list):\n return [self._render_template(item, message) for item in value]\n if isinstance(value, dict):\n return {\n key: self._render_template(item, message)\n for key, item in value.items()\n }\n return value\n\n def _replace_placeholders(self, value: str, message: NotificationMessage) -> str:\n replacements = {\n \"title\": message.notification_title,\n \"content\": message.content,\n \"price\": message.price,\n \"reason\": message.reason,\n \"desktop_link\": message.desktop_link,\n \"mobile_link\": message.mobile_link or message.desktop_link,\n }\n rendered = value\n for key, replacement in replacements.items():\n rendered = rendered.replace(f\"${{{key}}}\", replacement)\n rendered = rendered.replace(f\"{{{{{key}}}}}\", replacement)\n return rendered\n" }, { "path": "src/infrastructure/external/notification_clients/wecom_bot_client.py", "content": "\"\"\"\n企业微信机器人通知客户端\n\"\"\"\nimport asyncio\nfrom typing import Dict\n\nimport requests\n\nfrom .base import NotificationClient\n\n\nclass WeComBotClient(NotificationClient):\n \"\"\"企业微信机器人通知客户端\"\"\"\n\n channel_key = \"wecom\"\n display_name = \"企业微信\"\n\n def __init__(self, bot_url: str | None = None, pcurl_to_mobile: bool = True):\n super().__init__(enabled=bool(bot_url), pcurl_to_mobile=pcurl_to_mobile)\n self.bot_url = bot_url\n\n async def send(self, product_data: Dict, reason: str) -> None:\n if not self.is_enabled():\n raise RuntimeError(\"企业微信 未启用\")\n\n message = self._build_message(product_data, reason)\n markdown_lines = [f\"## {message.notification_title}\", \"\"]\n markdown_lines.append(f\"- 价格: {message.price}\")\n markdown_lines.append(f\"- 原因: {message.reason}\")\n if message.mobile_link:\n markdown_lines.append(f\"- 手机端链接: [{message.mobile_link}]({message.mobile_link})\")\n markdown_lines.append(f\"- 电脑端链接: [{message.desktop_link}]({message.desktop_link})\")\n payload = {\n \"msgtype\": \"markdown\",\n \"markdown\": {\"content\": \"\\n\".join(markdown_lines)},\n }\n headers = {\"Content-Type\": \"application/json\"}\n loop = asyncio.get_running_loop()\n response = await loop.run_in_executor(\n None,\n lambda: requests.post(\n self.bot_url,\n json=payload,\n headers=headers,\n timeout=10,\n ),\n )\n response.raise_for_status()\n result = response.json()\n if result.get(\"errcode\", 0) != 0:\n raise RuntimeError(result.get(\"errmsg\", \"企业微信返回未知错误\"))\n" }, { "path": "src/infrastructure/persistence/__init__.py", "content": "" }, { "path": "src/infrastructure/persistence/json_task_repository.py", "content": "\"\"\"\n基于JSON文件的任务仓储实现\n\"\"\"\nfrom typing import List, Optional\nimport json\nimport aiofiles\nfrom src.domain.models.task import Task\nfrom src.domain.repositories.task_repository import TaskRepository\n\n\nclass JsonTaskRepository(TaskRepository):\n \"\"\"基于JSON文件的任务仓储\"\"\"\n\n def __init__(self, config_file: str = \"config.json\"):\n self.config_file = config_file\n\n async def find_all(self) -> List[Task]:\n \"\"\"获取所有任务\"\"\"\n try:\n async with aiofiles.open(self.config_file, 'r', encoding='utf-8') as f:\n content = await f.read()\n if not content.strip():\n return []\n\n tasks_data = json.loads(content)\n tasks = []\n for i, task_data in enumerate(tasks_data):\n task_data['id'] = i\n tasks.append(Task(**task_data))\n return tasks\n except FileNotFoundError:\n return []\n except json.JSONDecodeError:\n print(f\"配置文件 {self.config_file} 格式错误\")\n return []\n\n async def find_by_id(self, task_id: int) -> Optional[Task]:\n \"\"\"根据ID获取任务\"\"\"\n tasks = await self.find_all()\n if 0 <= task_id < len(tasks):\n return tasks[task_id]\n return None\n\n async def save(self, task: Task) -> Task:\n \"\"\"保存任务（创建或更新）\"\"\"\n tasks = await self.find_all()\n\n if task.id is not None and 0 <= task.id < len(tasks):\n # 更新现有任务\n tasks[task.id] = task\n else:\n # 创建新任务\n task.id = len(tasks)\n tasks.append(task)\n\n await self._write_tasks(tasks)\n return task\n\n async def delete(self, task_id: int) -> bool:\n \"\"\"删除任务\"\"\"\n tasks = await self.find_all()\n if 0 <= task_id < len(tasks):\n tasks.pop(task_id)\n await self._write_tasks(tasks)\n return True\n return False\n\n async def _write_tasks(self, tasks: List[Task]):\n \"\"\"写入任务列表到文件\"\"\"\n tasks_data = [task.model_dump(exclude={'id'}) for task in tasks]\n async with aiofiles.open(self.config_file, 'w', encoding='utf-8') as f:\n await f.write(json.dumps(tasks_data, ensure_ascii=False, indent=2))\n" }, { "path": "src/infrastructure/persistence/sqlite_bootstrap.py", "content": "\"\"\"\nSQLite 启动初始化与旧文件迁移。\n\"\"\"\nfrom __future__ import annotations\n\nimport hashlib\nimport json\nimport threading\nfrom pathlib import Path\n\nfrom src.infrastructure.persistence.sqlite_connection import init_schema, sqlite_connection\nfrom src.infrastructure.persistence.storage_names import (\n build_result_filename,\n normalize_keyword_from_filename,\n normalize_keyword_slug,\n)\n\n\nBOOTSTRAP_LOCK = threading.Lock()\nLEGACY_CONFIG_FILE = \"config.json\"\nLEGACY_RESULT_DIR = \"jsonl\"\nLEGACY_PRICE_HISTORY_DIR = \"price_history\"\nTASKS_BOOTSTRAP_KEY = \"bootstrap:legacy_tasks\"\nRESULTS_BOOTSTRAP_KEY = \"bootstrap:legacy_results\"\nSNAPSHOTS_BOOTSTRAP_KEY = \"bootstrap:legacy_price_snapshots\"\n\n\ndef bootstrap_sqlite_storage(\n db_path: str | None = None,\n *,\n legacy_config_file: str | None = LEGACY_CONFIG_FILE,\n legacy_result_dir: str = LEGACY_RESULT_DIR,\n legacy_price_history_dir: str = LEGACY_PRICE_HISTORY_DIR,\n) -> None:\n with BOOTSTRAP_LOCK:\n with sqlite_connection(db_path) as conn:\n init_schema(conn)\n _import_tasks_if_needed(conn, legacy_config_file)\n _import_results_if_needed(conn, legacy_result_dir)\n _import_price_snapshots_if_needed(conn, legacy_price_history_dir)\n\n\ndef _table_is_empty(conn, table_name: str) -> bool:\n row = conn.execute(f\"SELECT COUNT(1) AS total FROM {table_name}\").fetchone()\n return row is None or int(row[\"total\"]) == 0\n\n\ndef _load_json_file(path: Path):\n if not path.exists():\n return None\n content = path.read_text(encoding=\"utf-8\").strip()\n if not content:\n return None\n return json.loads(content)\n\n\ndef _import_tasks_if_needed(conn, legacy_config_file: str | None) -> None:\n if _bootstrap_completed(conn, TASKS_BOOTSTRAP_KEY):\n return\n if not _table_is_empty(conn, \"tasks\"):\n _mark_bootstrap_completed(conn, TASKS_BOOTSTRAP_KEY)\n conn.commit()\n return\n if legacy_config_file is None:\n _mark_bootstrap_completed(conn, TASKS_BOOTSTRAP_KEY)\n conn.commit()\n return\n path = Path(legacy_config_file)\n tasks = _load_json_file(path)\n if not isinstance(tasks, list):\n _mark_bootstrap_completed(conn, TASKS_BOOTSTRAP_KEY)\n conn.commit()\n return\n\n for index, raw_task in enumerate(tasks):\n if not isinstance(raw_task, dict):\n continue\n conn.execute(\n \"\"\"\n INSERT INTO tasks (\n id, task_name, enabled, keyword, description, analyze_images,\n max_pages, personal_only, min_price, max_price, cron,\n ai_prompt_base_file, ai_prompt_criteria_file, account_state_file,\n account_strategy, free_shipping, new_publish_option, region,\n decision_mode, keyword_rules_json, is_running\n ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)\n \"\"\",\n (\n index,\n raw_task.get(\"task_name\", \"\"),\n _as_int(raw_task.get(\"enabled\", True)),\n raw_task.get(\"keyword\", \"\"),\n raw_task.get(\"description\", \"\"),\n _as_int(raw_task.get(\"analyze_images\", True)),\n int(raw_task.get(\"max_pages\", 1) or 1),\n _as_int(raw_task.get(\"personal_only\", False)),\n raw_task.get(\"min_price\"),\n raw_task.get(\"max_price\"),\n raw_task.get(\"cron\"),\n raw_task.get(\"ai_prompt_base_file\", \"prompts/base_prompt.txt\"),\n raw_task.get(\"ai_prompt_criteria_file\", \"\"),\n raw_task.get(\"account_state_file\"),\n raw_task.get(\"account_strategy\", \"auto\"),\n _as_int(raw_task.get(\"free_shipping\", True)),\n raw_task.get(\"new_publish_option\"),\n raw_task.get(\"region\"),\n raw_task.get(\"decision_mode\", \"ai\"),\n json.dumps(raw_task.get(\"keyword_rules\") or [], ensure_ascii=False),\n _as_int(raw_task.get(\"is_running\", False)),\n ),\n )\n _mark_bootstrap_completed(conn, TASKS_BOOTSTRAP_KEY)\n conn.commit()\n\n\ndef _import_results_if_needed(conn, legacy_result_dir: str) -> None:\n if _bootstrap_completed(conn, RESULTS_BOOTSTRAP_KEY):\n return\n if not _table_is_empty(conn, \"result_items\"):\n _mark_bootstrap_completed(conn, RESULTS_BOOTSTRAP_KEY)\n conn.commit()\n return\n result_dir = Path(legacy_result_dir)\n if not result_dir.exists():\n _mark_bootstrap_completed(conn, RESULTS_BOOTSTRAP_KEY)\n conn.commit()\n return\n\n for path in sorted(result_dir.glob(\"*.jsonl\")):\n filename = path.name\n keyword = normalize_keyword_from_filename(filename)\n with path.open(\"r\", encoding=\"utf-8\") as handle:\n for line in handle:\n text = line.strip()\n if not text:\n continue\n try:\n record = json.loads(text)\n except json.JSONDecodeError:\n continue\n _insert_result_record(conn, record, keyword=keyword, filename=filename)\n _mark_bootstrap_completed(conn, RESULTS_BOOTSTRAP_KEY)\n conn.commit()\n\n\ndef _import_price_snapshots_if_needed(conn, legacy_price_history_dir: str) -> None:\n if _bootstrap_completed(conn, SNAPSHOTS_BOOTSTRAP_KEY):\n return\n if not _table_is_empty(conn, \"price_snapshots\"):\n _mark_bootstrap_completed(conn, SNAPSHOTS_BOOTSTRAP_KEY)\n conn.commit()\n return\n history_dir = Path(legacy_price_history_dir)\n if not history_dir.exists():\n _mark_bootstrap_completed(conn, SNAPSHOTS_BOOTSTRAP_KEY)\n conn.commit()\n return\n\n for path in sorted(history_dir.glob(\"*_history.jsonl\")):\n with path.open(\"r\", encoding=\"utf-8\") as handle:\n for line in handle:\n text = line.strip()\n if not text:\n continue\n try:\n record = json.loads(text)\n except json.JSONDecodeError:\n continue\n _insert_price_snapshot(conn, record)\n _mark_bootstrap_completed(conn, SNAPSHOTS_BOOTSTRAP_KEY)\n conn.commit()\n\n\ndef _insert_result_record(conn, record: dict, *, keyword: str, filename: str) -> None:\n item = record.get(\"商品信息\", {}) or {}\n analysis = record.get(\"ai_analysis\", {}) or {}\n link = str(item.get(\"商品链接\") or \"\")\n if link:\n link_unique_key = link.split(\"&\", 1)[0]\n else:\n item_id = str(item.get(\"商品ID\") or \"\").strip()\n if item_id:\n link_unique_key = f\"item:{item_id}\"\n else:\n link_unique_key = \"hash:\" + hashlib.sha1(\n json.dumps(record, ensure_ascii=False, sort_keys=True).encode(\"utf-8\")\n ).hexdigest()\n final_keyword = str(record.get(\"搜索关键字\") or keyword)\n result_filename = filename or build_result_filename(final_keyword)\n keyword_hit_count = analysis.get(\"keyword_hit_count\", 0)\n try:\n keyword_hit_count = int(keyword_hit_count)\n except (TypeError, ValueError):\n keyword_hit_count = 0\n\n conn.execute(\n \"\"\"\n INSERT OR IGNORE INTO result_items (\n result_filename, keyword, task_name, crawl_time, publish_time, price,\n price_display, item_id, title, link, link_unique_key, seller_nickname,\n is_recommended, analysis_source, keyword_hit_count, raw_json\n ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)\n \"\"\",\n (\n result_filename,\n final_keyword,\n record.get(\"任务名称\", \"\"),\n record.get(\"爬取时间\", \"\"),\n item.get(\"发布时间\"),\n _parse_price(item.get(\"当前售价\")),\n item.get(\"当前售价\"),\n item.get(\"商品ID\"),\n item.get(\"商品标题\"),\n link,\n link_unique_key,\n (record.get(\"卖家信息\", {}) or {}).get(\"卖家昵称\") or item.get(\"卖家昵称\"),\n _as_int(analysis.get(\"is_recommended\", False)),\n analysis.get(\"analysis_source\"),\n keyword_hit_count,\n json.dumps(record, ensure_ascii=False),\n ),\n )\n\n\ndef _insert_price_snapshot(conn, record: dict) -> None:\n keyword = str(record.get(\"keyword\") or \"\")\n slug = str(record.get(\"keyword_slug\") or normalize_keyword_slug(keyword))\n conn.execute(\n \"\"\"\n INSERT OR IGNORE INTO price_snapshots (\n keyword_slug, keyword, task_name, snapshot_time, snapshot_day, run_id,\n item_id, title, price, price_display, tags_json, region, seller,\n publish_time, link\n ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)\n \"\"\",\n (\n slug,\n keyword,\n record.get(\"task_name\", \"\"),\n record.get(\"snapshot_time\", \"\"),\n record.get(\"snapshot_day\", \"\"),\n record.get(\"run_id\", \"\"),\n record.get(\"item_id\", \"\"),\n record.get(\"title\", \"\"),\n _parse_price(record.get(\"price\")),\n record.get(\"price_display\"),\n json.dumps(record.get(\"tags\") or [], ensure_ascii=False),\n record.get(\"region\"),\n record.get(\"seller\"),\n record.get(\"publish_time\"),\n record.get(\"link\"),\n ),\n )\n\n\ndef _as_int(value) -> int:\n if isinstance(value, bool):\n return 1 if value else 0\n if value is None:\n return 0\n return 1 if str(value).strip().lower() in {\"1\", \"true\", \"yes\", \"on\"} else 0\n\n\ndef _parse_price(value):\n if value is None:\n return None\n if isinstance(value, (int, float)):\n return round(float(value), 2)\n\n text = str(value).strip().replace(\"¥\", \"\").replace(\",\", \"\")\n if not text or text in {\"价格异常\", \"暂无\", \"-\", \"N/A\"}:\n return None\n if text.endswith(\"万\"):\n text = str(float(text[:-1]) * 10000)\n try:\n return round(float(text), 2)\n except (TypeError, ValueError):\n return None\n\n\ndef _bootstrap_completed(conn, key: str) -> bool:\n row = conn.execute(\n \"SELECT value FROM app_metadata WHERE key = ?\",\n (key,),\n ).fetchone()\n return row is not None\n\n\ndef _mark_bootstrap_completed(conn, key: str) -> None:\n conn.execute(\n \"\"\"\n INSERT OR REPLACE INTO app_metadata(key, value)\n VALUES (?, 'done')\n \"\"\",\n (key,),\n )\n" }, { "path": "src/infrastructure/persistence/sqlite_connection.py", "content": "\"\"\"\nSQLite 连接与 schema 初始化。\n\"\"\"\nfrom __future__ import annotations\n\nimport os\nimport sqlite3\nfrom contextlib import contextmanager\nfrom pathlib import Path\nfrom typing import Iterator\n\nfrom src.infrastructure.persistence.storage_names import DEFAULT_DATABASE_PATH\n\n\nBUSY_TIMEOUT_MS = 5000\n\nSCHEMA_STATEMENTS = (\n \"\"\"\n CREATE TABLE IF NOT EXISTS app_metadata (\n key TEXT PRIMARY KEY,\n value TEXT NOT NULL\n )\n \"\"\",\n \"\"\"\n CREATE TABLE IF NOT EXISTS tasks (\n id INTEGER PRIMARY KEY,\n task_name TEXT NOT NULL,\n enabled INTEGER NOT NULL,\n keyword TEXT NOT NULL,\n description TEXT,\n analyze_images INTEGER NOT NULL,\n max_pages INTEGER NOT NULL,\n personal_only INTEGER NOT NULL,\n min_price TEXT,\n max_price TEXT,\n cron TEXT,\n ai_prompt_base_file TEXT NOT NULL,\n ai_prompt_criteria_file TEXT NOT NULL,\n account_state_file TEXT,\n account_strategy TEXT NOT NULL,\n free_shipping INTEGER NOT NULL,\n new_publish_option TEXT,\n region TEXT,\n decision_mode TEXT NOT NULL,\n keyword_rules_json TEXT NOT NULL,\n is_running INTEGER NOT NULL\n )\n \"\"\",\n \"\"\"\n CREATE TABLE IF NOT EXISTS result_items (\n id INTEGER PRIMARY KEY AUTOINCREMENT,\n result_filename TEXT NOT NULL,\n keyword TEXT NOT NULL,\n task_name TEXT NOT NULL,\n crawl_time TEXT NOT NULL,\n publish_time TEXT,\n price REAL,\n price_display TEXT,\n item_id TEXT,\n title TEXT,\n link TEXT,\n link_unique_key TEXT NOT NULL,\n seller_nickname TEXT,\n is_recommended INTEGER NOT NULL,\n analysis_source TEXT,\n keyword_hit_count INTEGER NOT NULL,\n raw_json TEXT NOT NULL,\n UNIQUE(result_filename, link_unique_key)\n )\n \"\"\",\n \"\"\"\n CREATE TABLE IF NOT EXISTS price_snapshots (\n id INTEGER PRIMARY KEY AUTOINCREMENT,\n keyword_slug TEXT NOT NULL,\n keyword TEXT NOT NULL,\n task_name TEXT NOT NULL,\n snapshot_time TEXT NOT NULL,\n snapshot_day TEXT NOT NULL,\n run_id TEXT NOT NULL,\n item_id TEXT NOT NULL,\n title TEXT,\n price REAL NOT NULL,\n price_display TEXT,\n tags_json TEXT NOT NULL,\n region TEXT,\n seller TEXT,\n publish_time TEXT,\n link TEXT,\n UNIQUE(keyword_slug, run_id, item_id)\n )\n \"\"\",\n \"CREATE INDEX IF NOT EXISTS idx_tasks_name ON tasks(task_name)\",\n \"\"\"\n CREATE INDEX IF NOT EXISTS idx_results_filename_crawl\n ON result_items(result_filename, crawl_time DESC)\n \"\"\",\n \"\"\"\n CREATE INDEX IF NOT EXISTS idx_results_filename_publish\n ON result_items(result_filename, publish_time DESC)\n \"\"\",\n \"\"\"\n CREATE INDEX IF NOT EXISTS idx_results_filename_price\n ON result_items(result_filename, price DESC)\n \"\"\",\n \"\"\"\n CREATE INDEX IF NOT EXISTS idx_results_filename_recommended\n ON result_items(result_filename, is_recommended, analysis_source, crawl_time DESC)\n \"\"\",\n \"\"\"\n CREATE INDEX IF NOT EXISTS idx_snapshots_keyword_time\n ON price_snapshots(keyword_slug, snapshot_time DESC)\n \"\"\",\n \"\"\"\n CREATE INDEX IF NOT EXISTS idx_snapshots_keyword_item_time\n ON price_snapshots(keyword_slug, item_id, snapshot_time DESC)\n \"\"\",\n)\n\n\ndef get_database_path() -> str:\n return os.getenv(\"APP_DATABASE_FILE\", DEFAULT_DATABASE_PATH)\n\n\ndef _prepare_database_file(path: str) -> None:\n Path(path).parent.mkdir(parents=True, exist_ok=True)\n\n\ndef _apply_pragmas(conn: sqlite3.Connection) -> None:\n conn.execute(\"PRAGMA journal_mode=WAL\")\n conn.execute(\"PRAGMA foreign_keys=ON\")\n conn.execute(f\"PRAGMA busy_timeout={BUSY_TIMEOUT_MS}\")\n\n\ndef init_schema(conn: sqlite3.Connection) -> None:\n for statement in SCHEMA_STATEMENTS:\n conn.execute(statement)\n conn.commit()\n\n\n@contextmanager\ndef sqlite_connection(\n db_path: str | None = None,\n) -> Iterator[sqlite3.Connection]:\n path = db_path or get_database_path()\n _prepare_database_file(path)\n conn = sqlite3.connect(path)\n conn.row_factory = sqlite3.Row\n try:\n _apply_pragmas(conn)\n yield conn\n finally:\n conn.close()\n" }, { "path": "src/infrastructure/persistence/sqlite_task_repository.py", "content": "\"\"\"\n基于 SQLite 的任务仓储实现。\n\"\"\"\nfrom __future__ import annotations\n\nimport asyncio\nimport json\nfrom typing import List, Optional\n\nfrom src.domain.models.task import Task\nfrom src.domain.repositories.task_repository import TaskRepository\nfrom src.infrastructure.persistence.sqlite_bootstrap import bootstrap_sqlite_storage\nfrom src.infrastructure.persistence.sqlite_connection import sqlite_connection\n\n\ndef _row_to_task(row) -> Task:\n payload = dict(row)\n payload[\"enabled\"] = bool(payload[\"enabled\"])\n payload[\"analyze_images\"] = bool(payload[\"analyze_images\"])\n payload[\"personal_only\"] = bool(payload[\"personal_only\"])\n payload[\"free_shipping\"] = bool(payload[\"free_shipping\"])\n payload[\"is_running\"] = bool(payload[\"is_running\"])\n payload[\"keyword_rules\"] = json.loads(payload.pop(\"keyword_rules_json\") or \"[]\")\n return Task(**payload)\n\n\ndef find_task_by_name_sync(task_name: str) -> Task | None:\n bootstrap_sqlite_storage()\n with sqlite_connection() as conn:\n row = conn.execute(\n \"SELECT * FROM tasks WHERE task_name = ? ORDER BY id ASC LIMIT 1\",\n (task_name,),\n ).fetchone()\n return _row_to_task(row) if row else None\n\n\nclass SqliteTaskRepository(TaskRepository):\n \"\"\"基于 SQLite 的任务仓储\"\"\"\n\n def __init__(\n self,\n db_path: str | None = None,\n legacy_config_file: str | None = \"config.json\",\n ):\n self.db_path = db_path\n self.legacy_config_file = legacy_config_file\n\n async def find_all(self) -> List[Task]:\n return await asyncio.to_thread(self._find_all_sync)\n\n async def find_by_id(self, task_id: int) -> Optional[Task]:\n return await asyncio.to_thread(self._find_by_id_sync, task_id)\n\n async def save(self, task: Task) -> Task:\n return await asyncio.to_thread(self._save_sync, task)\n\n async def delete(self, task_id: int) -> bool:\n return await asyncio.to_thread(self._delete_sync, task_id)\n\n def _find_all_sync(self) -> List[Task]:\n bootstrap_sqlite_storage(\n self.db_path,\n legacy_config_file=self.legacy_config_file,\n )\n with sqlite_connection(self.db_path) as conn:\n rows = conn.execute(\"SELECT * FROM tasks ORDER BY id ASC\").fetchall()\n return [_row_to_task(row) for row in rows]\n\n def _find_by_id_sync(self, task_id: int) -> Optional[Task]:\n bootstrap_sqlite_storage(\n self.db_path,\n legacy_config_file=self.legacy_config_file,\n )\n with sqlite_connection(self.db_path) as conn:\n row = conn.execute(\"SELECT * FROM tasks WHERE id = ?\", (task_id,)).fetchone()\n return _row_to_task(row) if row else None\n\n def _save_sync(self, task: Task) -> Task:\n bootstrap_sqlite_storage(\n self.db_path,\n legacy_config_file=self.legacy_config_file,\n )\n with sqlite_connection(self.db_path) as conn:\n task_id = task.id\n if task_id is None:\n task_id = self._next_task_id(conn)\n payload = self._task_values(task.model_copy(update={\"id\": task_id}))\n conn.execute(\n \"\"\"\n INSERT OR REPLACE INTO tasks (\n id, task_name, enabled, keyword, description, analyze_images,\n max_pages, personal_only, min_price, max_price, cron,\n ai_prompt_base_file, ai_prompt_criteria_file, account_state_file,\n account_strategy, free_shipping, new_publish_option, region,\n decision_mode, keyword_rules_json, is_running\n ) VALUES (\n :id, :task_name, :enabled, :keyword, :description, :analyze_images,\n :max_pages, :personal_only, :min_price, :max_price, :cron,\n :ai_prompt_base_file, :ai_prompt_criteria_file, :account_state_file,\n :account_strategy, :free_shipping, :new_publish_option, :region,\n :decision_mode, :keyword_rules_json, :is_running\n )\n \"\"\",\n payload,\n )\n conn.commit()\n return task.model_copy(update={\"id\": task_id})\n\n def _delete_sync(self, task_id: int) -> bool:\n bootstrap_sqlite_storage(\n self.db_path,\n legacy_config_file=self.legacy_config_file,\n )\n with sqlite_connection(self.db_path) as conn:\n cursor = conn.execute(\"DELETE FROM tasks WHERE id = ?\", (task_id,))\n conn.commit()\n return cursor.rowcount > 0\n\n def _next_task_id(self, conn) -> int:\n row = conn.execute(\"SELECT COALESCE(MAX(id), -1) AS max_id FROM tasks\").fetchone()\n return int(row[\"max_id\"]) + 1\n\n def _task_values(self, task: Task) -> dict:\n values = task.model_dump()\n values[\"enabled\"] = int(task.enabled)\n values[\"analyze_images\"] = int(task.analyze_images)\n values[\"personal_only\"] = int(task.personal_only)\n values[\"free_shipping\"] = int(task.free_shipping)\n values[\"is_running\"] = int(task.is_running)\n values[\"keyword_rules_json\"] = json.dumps(task.keyword_rules or [], ensure_ascii=False)\n values.pop(\"keyword_rules\", None)\n return values\n" }, { "path": "src/infrastructure/persistence/storage_names.py", "content": "\"\"\"\nSQLite 持久化相关的统一命名规则。\n\"\"\"\nfrom __future__ import annotations\n\n\nDEFAULT_DATABASE_PATH = \"data/app.sqlite3\"\nRESULT_FILE_SUFFIX = \"_full_data.jsonl\"\n\n\ndef build_result_filename(keyword: str) -> str:\n return f\"{str(keyword or '').replace(' ', '_')}{RESULT_FILE_SUFFIX}\"\n\n\ndef normalize_keyword_from_filename(filename: str) -> str:\n return str(filename or \"\").replace(RESULT_FILE_SUFFIX, \"\")\n\n\ndef normalize_keyword_slug(keyword: str) -> str:\n text = \"\".join(\n char for char in str(keyword or \"\").lower().replace(\" \", \"_\")\n if char.isalnum() or char in \"_-\"\n ).rstrip(\"_\")\n return text or \"unknown\"\n" }, { "path": "src/keyword_rule_engine.py", "content": "\"\"\"\n关键词判断引擎：单组 OR 逻辑，命中任意关键词即推荐。\n纯英数字关键词按完整词匹配，避免 Q1 误命中 Q1R5。\n\"\"\"\nimport re\nfrom typing import Any, Dict, Iterable, List\n\n\n_ASCII_TOKEN_KEYWORD_PATTERN = re.compile(r\"^[a-z0-9 ]+$\")\n_ASCII_TOKEN_BOUNDARY = r\"[a-z0-9]\"\n\n\ndef normalize_text(value: str) -> str:\n return \" \".join((value or \"\").lower().split())\n\n\ndef _collect_text_fragments(value: Any, bucket: List[str]) -> None:\n if value is None:\n return\n if isinstance(value, str):\n text = value.strip()\n if text:\n bucket.append(text)\n return\n if isinstance(value, (int, float, bool)):\n bucket.append(str(value))\n return\n if isinstance(value, dict):\n for item in value.values():\n _collect_text_fragments(item, bucket)\n return\n if isinstance(value, list):\n for item in value:\n _collect_text_fragments(item, bucket)\n\n\ndef build_search_text(record: Dict[str, Any]) -> str:\n fragments: List[str] = []\n product_info = record.get(\"商品信息\", {})\n seller_info = record.get(\"卖家信息\", {})\n\n _collect_text_fragments(product_info.get(\"商品标题\"), fragments)\n _collect_text_fragments(product_info, fragments)\n _collect_text_fragments(seller_info, fragments)\n\n return normalize_text(\" \".join(fragments))\n\n\ndef _normalize_keywords(values: Iterable[str]) -> List[str]:\n normalized: List[str] = []\n seen = set()\n for raw in values or []:\n text = normalize_text(str(raw).strip())\n if not text or text in seen:\n continue\n seen.add(text)\n normalized.append(text)\n return normalized\n\n\ndef _uses_ascii_token_match(keyword: str) -> bool:\n return bool(keyword) and _ASCII_TOKEN_KEYWORD_PATTERN.fullmatch(keyword) is not None\n\n\ndef _keyword_matches(keyword: str, normalized_text: str) -> bool:\n if not _uses_ascii_token_match(keyword):\n return keyword in normalized_text\n pattern = rf\"(? Dict[str, Any]:\n normalized_text = normalize_text(search_text)\n normalized_keywords = _normalize_keywords(keywords)\n\n if not normalized_text:\n return {\n \"analysis_source\": \"keyword\",\n \"is_recommended\": False,\n \"reason\": \"可匹配文本为空，关键词规则无法执行。\",\n \"matched_keywords\": [],\n \"keyword_hit_count\": 0,\n }\n\n if not normalized_keywords:\n return {\n \"analysis_source\": \"keyword\",\n \"is_recommended\": False,\n \"reason\": \"未配置关键词规则。\",\n \"matched_keywords\": [],\n \"keyword_hit_count\": 0,\n }\n\n matched_keywords = [kw for kw in normalized_keywords if _keyword_matches(kw, normalized_text)]\n hit_count = len(matched_keywords)\n is_recommended = hit_count > 0\n\n if is_recommended:\n reason = f\"命中 {hit_count} 个关键词：{', '.join(matched_keywords)}\"\n else:\n reason = \"未命中任何关键词。\"\n\n return {\n \"analysis_source\": \"keyword\",\n \"is_recommended\": is_recommended,\n \"reason\": reason,\n \"matched_keywords\": matched_keywords,\n \"keyword_hit_count\": hit_count,\n }\n" }, { "path": "src/parsers.py", "content": "import json\nfrom datetime import datetime\n\nfrom src.config import AI_DEBUG_MODE\nfrom src.utils import safe_get\n\n\nasync def _parse_search_results_json(json_data: dict, source: str) -> list:\n \"\"\"解析搜索API的JSON数据，返回基础商品信息列表。\"\"\"\n page_data = []\n try:\n items = await safe_get(json_data, \"data\", \"resultList\", default=[])\n if not items:\n print(f\"LOG: ({source}) API响应中未找到商品列表 (resultList)。\")\n if AI_DEBUG_MODE:\n print(f\"--- [SEARCH DEBUG] RAW JSON RESPONSE from {source} ---\")\n print(json.dumps(json_data, ensure_ascii=False, indent=2))\n print(\"----------------------------------------------------\")\n return []\n\n for item in items:\n main_data = await safe_get(item, \"data\", \"item\", \"main\", \"exContent\", default={})\n click_params = await safe_get(item, \"data\", \"item\", \"main\", \"clickParam\", \"args\", default={})\n\n title = await safe_get(main_data, \"title\", default=\"未知标题\")\n price_parts = await safe_get(main_data, \"price\", default=[])\n price = \"\".join([str(p.get(\"text\", \"\")) for p in price_parts if isinstance(p, dict)]).replace(\"当前价\", \"\").strip() if isinstance(price_parts, list) else \"价格异常\"\n if \"万\" in price: price = f\"¥{float(price.replace('¥', '').replace('万', '')) * 10000:.0f}\"\n area = await safe_get(main_data, \"area\", default=\"地区未知\")\n seller = await safe_get(main_data, \"userNickName\", default=\"匿名卖家\")\n raw_link = await safe_get(item, \"data\", \"item\", \"main\", \"targetUrl\", default=\"\")\n image_url = await safe_get(main_data, \"picUrl\", default=\"\")\n pub_time_ts = click_params.get(\"publishTime\", \"\")\n item_id = await safe_get(main_data, \"itemId\", default=\"未知ID\")\n original_price = await safe_get(main_data, \"oriPrice\", default=\"暂无\")\n wants_count = await safe_get(click_params, \"wantNum\", default='NaN')\n\n\n tags = []\n if await safe_get(click_params, \"tag\") == \"freeship\":\n tags.append(\"包邮\")\n r1_tags = await safe_get(main_data, \"fishTags\", \"r1\", \"tagList\", default=[])\n for tag_item in r1_tags:\n content = await safe_get(tag_item, \"data\", \"content\", default=\"\")\n if \"验货宝\" in content:\n tags.append(\"验货宝\")\n\n page_data.append({\n \"商品标题\": title,\n \"当前售价\": price,\n \"商品原价\": original_price,\n \"“想要”人数\": wants_count,\n \"商品标签\": tags,\n \"发货地区\": area,\n \"卖家昵称\": seller,\n \"商品链接\": raw_link.replace(\"fleamarket://\", \"https://www.goofish.com/\"),\n \"发布时间\": datetime.fromtimestamp(int(pub_time_ts)/1000).strftime(\"%Y-%m-%d %H:%M\") if pub_time_ts.isdigit() else \"未知时间\",\n \"商品ID\": item_id\n })\n print(f\"LOG: ({source}) 成功解析到 {len(page_data)} 条商品基础信息。\")\n return page_data\n except Exception as e:\n print(f\"LOG: ({source}) JSON数据处理异常: {str(e)}\")\n return []\n\n\nasync def calculate_reputation_from_ratings(ratings_json: list) -> dict:\n \"\"\"从原始评价API数据列表中，计算作为卖家和买家的好评数与好评率。\"\"\"\n seller_total = 0\n seller_positive = 0\n buyer_total = 0\n buyer_positive = 0\n\n for card in ratings_json:\n # 使用 safe_get 保证安全访问\n data = await safe_get(card, 'cardData', default={})\n role_tag = await safe_get(data, 'rateTagList', 0, 'text', default='')\n rate_type = await safe_get(data, 'rate') # 1=好评, 0=中评, -1=差评\n\n if \"卖家\" in role_tag:\n seller_total += 1\n if rate_type == 1:\n seller_positive += 1\n elif \"买家\" in role_tag:\n buyer_total += 1\n if rate_type == 1:\n buyer_positive += 1\n\n # 计算比率，并处理除以零的情况\n seller_rate = f\"{(seller_positive / seller_total * 100):.2f}%\" if seller_total > 0 else \"N/A\"\n buyer_rate = f\"{(buyer_positive / buyer_total * 100):.2f}%\" if buyer_total > 0 else \"N/A\"\n\n return {\n \"作为卖家的好评数\": f\"{seller_positive}/{seller_total}\",\n \"作为卖家的好评率\": seller_rate,\n \"作为买家的好评数\": f\"{buyer_positive}/{buyer_total}\",\n \"作为买家的好评率\": buyer_rate\n }\n\n\nasync def _parse_user_items_data(items_json: list) -> list:\n \"\"\"解析用户主页的商品列表API的JSON数据。\"\"\"\n parsed_list = []\n for card in items_json:\n data = card.get('cardData', {})\n status_code = data.get('itemStatus')\n if status_code == 0:\n status_text = \"在售\"\n elif status_code == 1:\n status_text = \"已售\"\n else:\n status_text = f\"未知状态 ({status_code})\"\n\n parsed_list.append({\n \"商品ID\": data.get('id'),\n \"商品标题\": data.get('title'),\n \"商品价格\": data.get('priceInfo', {}).get('price'),\n \"商品主图\": data.get('picInfo', {}).get('picUrl'),\n \"商品状态\": status_text\n })\n return parsed_list\n\n\nasync def parse_user_head_data(head_json: dict) -> dict:\n \"\"\"解析用户头部API的JSON数据。\"\"\"\n data = head_json.get('data', {})\n ylz_tags = await safe_get(data, 'module', 'base', 'ylzTags', default=[])\n seller_credit, buyer_credit = {}, {}\n for tag in ylz_tags:\n if await safe_get(tag, 'attributes', 'role') == 'seller':\n seller_credit = {'level': await safe_get(tag, 'attributes', 'level'), 'text': tag.get('text')}\n elif await safe_get(tag, 'attributes', 'role') == 'buyer':\n buyer_credit = {'level': await safe_get(tag, 'attributes', 'level'), 'text': tag.get('text')}\n return {\n \"卖家昵称\": await safe_get(data, 'module', 'base', 'displayName'),\n \"卖家头像链接\": await safe_get(data, 'module', 'base', 'avatar', 'avatar'),\n \"卖家个性签名\": await safe_get(data, 'module', 'base', 'introduction', default=''),\n \"卖家在售/已售商品数\": await safe_get(data, 'module', 'tabs', 'item', 'number'),\n \"卖家收到的评价总数\": await safe_get(data, 'module', 'tabs', 'rate', 'number'),\n \"卖家信用等级\": seller_credit.get('text', '暂无'),\n \"买家信用等级\": buyer_credit.get('text', '暂无')\n }\n\n\nasync def parse_ratings_data(ratings_json: list) -> list:\n \"\"\"解析评价列表API的JSON数据。\"\"\"\n parsed_list = []\n for card in ratings_json:\n data = await safe_get(card, 'cardData', default={})\n rate_tag = await safe_get(data, 'rateTagList', 0, 'text', default='未知角色')\n rate_type = await safe_get(data, 'rate')\n if rate_type == 1: rate_text = \"好评\"\n elif rate_type == 0: rate_text = \"中评\"\n elif rate_type == -1: rate_text = \"差评\"\n else: rate_text = \"未知\"\n parsed_list.append({\n \"评价ID\": data.get('rateId'),\n \"评价内容\": data.get('feedback'),\n \"评价类型\": rate_text,\n \"评价来源角色\": rate_tag,\n \"评价者昵称\": data.get('raterUserNick'),\n \"评价时间\": data.get('gmtCreate'),\n \"评价图片\": await safe_get(data, 'pictCdnUrlList', default=[])\n })\n return parsed_list\n" }, { "path": "src/prompt_utils.py", "content": "import json\nimport os\nimport sys\nfrom typing import Awaitable, Callable, Optional\n\nimport aiofiles\n\nfrom src.infrastructure.external.ai_client import AIClient\n\n# The meta-prompt to instruct the AI\nMETA_PROMPT_TEMPLATE = \"\"\"\n你是一位世界级的AI提示词工程大师。你的任务是根据用户提供的【购买需求】，模仿一个【参考范例】，为闲鱼监控机器人的AI分析模块（代号 EagleEye）生成一份全新的【分析标准】文本。\n\n你的输出必须严格遵循【参考范例】的结构、语气和核心原则，但内容要完全针对用户的【购买需求】进行定制。最终生成的文本将作为AI分析模块的思考指南。\n\n---\n这是【参考范例】（`macbook_criteria.txt`）：\n```text\n{reference_text}\n```\n---\n\n这是用户的【购买需求】：\n```text\n{user_description}\n```\n---\n\n请现在开始生成全新的【分析标准】文本。请注意：\n1. **只输出新生成的文本内容**，不要包含任何额外的解释、标题或代码块标记。\n2. 保留范例中的 `[V6.3 核心升级]`、`[V6.4 逻辑修正]` 等版本标记，这有助于保持格式一致性。\n3. 将范例中所有与 \"MacBook\" 相关的内容，替换为与用户需求商品相关的内容。\n4. 思考并生成针对新商品类型的“一票否决硬性原则”和“危险信号清单”。\n\"\"\"\n\nProgressCallback = Callable[[str, str], Awaitable[None]]\n\n\nasync def _report_progress(\n progress_callback: Optional[ProgressCallback],\n step_key: str,\n message: str,\n) -> None:\n if progress_callback:\n await progress_callback(step_key, message)\n\n\ndef _read_reference_text(reference_file_path: str) -> str:\n try:\n with open(reference_file_path, \"r\", encoding=\"utf-8\") as file:\n return file.read()\n except FileNotFoundError:\n raise FileNotFoundError(f\"参考文件未找到: {reference_file_path}\")\n except IOError as exc:\n raise IOError(f\"读取参考文件失败: {exc}\")\n\n\nasync def _request_generated_text(ai_client: AIClient, prompt: str) -> str:\n print(\"正在调用AI生成新的分析标准，请稍候...\")\n try:\n generated_text = await ai_client._call_ai(\n [{\"role\": \"user\", \"content\": prompt}],\n temperature=0.5,\n max_output_tokens=800,\n enable_json_output=False,\n )\n except Exception as exc:\n print(f\"调用 OpenAI API 时出错: {exc}\")\n raise\n\n print(\"AI已成功生成内容。\")\n return generated_text.strip()\n\n\nasync def _close_ai_client(\n ai_client: AIClient,\n active_error: BaseException | None,\n) -> None:\n try:\n await ai_client.close()\n except Exception as close_error:\n print(f\"关闭 AI 客户端时出错: {close_error}\")\n if active_error is None:\n raise\n\n\nasync def generate_criteria(\n user_description: str,\n reference_file_path: str,\n progress_callback: Optional[ProgressCallback] = None,\n) -> str:\n \"\"\"\n Generates a new criteria file content using AI.\n \"\"\"\n ai_client = AIClient()\n active_error: BaseException | None = None\n try:\n if not ai_client.is_available():\n ai_client.refresh()\n if not ai_client.is_available():\n raise RuntimeError(\"AI客户端未初始化，无法生成分析标准。请检查.env配置。\")\n\n await _report_progress(progress_callback, \"reference\", \"正在读取参考文件。\")\n print(f\"正在读取参考文件: {reference_file_path}\")\n reference_text = _read_reference_text(reference_file_path)\n\n await _report_progress(progress_callback, \"prompt\", \"正在构建发送给 AI 的指令。\")\n print(\"正在构建发送给AI的指令...\")\n prompt = META_PROMPT_TEMPLATE.format(\n reference_text=reference_text,\n user_description=user_description,\n )\n\n await _report_progress(progress_callback, \"llm\", \"正在调用 AI 生成分析标准。\")\n return await _request_generated_text(ai_client, prompt)\n except Exception as exc:\n active_error = exc\n raise\n finally:\n await _close_ai_client(ai_client, active_error)\n\n\nasync def update_config_with_new_task(new_task: dict, config_file: str = \"config.json\"):\n \"\"\"\n 将一个新任务添加到指定的JSON配置文件中。\n \"\"\"\n print(f\"正在更新配置文件: {config_file}\")\n try:\n # 读取现有配置\n config_data = []\n if os.path.exists(config_file):\n async with aiofiles.open(config_file, 'r', encoding='utf-8') as f:\n content = await f.read()\n # 处理空文件的情况\n if content.strip():\n try:\n config_data = json.loads(content)\n print(f\"成功读取现有配置，当前任务数量: {len(config_data)}\")\n except json.JSONDecodeError as e:\n print(f\"解析配置文件失败，将创建新配置: {e}\")\n config_data = []\n else:\n print(f\"配置文件不存在，将创建新文件: {config_file}\")\n\n # 追加新任务\n config_data.append(new_task)\n\n # 写回配置文件\n async with aiofiles.open(config_file, 'w', encoding='utf-8') as f:\n await f.write(json.dumps(config_data, ensure_ascii=False, indent=2))\n print(f\"配置文件写入完成\")\n\n print(f\"成功！新任务 '{new_task.get('task_name')}' 已添加到 {config_file} 并已启用。\")\n return True\n except json.JSONDecodeError as e:\n error_msg = f\"错误: 配置文件 {config_file} 格式错误，无法解析: {e}\"\n sys.stderr.write(error_msg + \"\\n\")\n print(error_msg)\n return False\n except IOError as e:\n error_msg = f\"错误: 读写配置文件失败: {e}\"\n sys.stderr.write(error_msg + \"\\n\")\n print(error_msg)\n return False\n except Exception as e:\n error_msg = f\"错误: 更新配置文件时发生未知错误: {e}\"\n sys.stderr.write(error_msg + \"\\n\")\n print(error_msg)\n import traceback\n print(traceback.format_exc())\n return False\n" }, { "path": "src/rotation.py", "content": "import os\nimport random\nimport time\nfrom dataclasses import dataclass\nfrom typing import Dict, List, Optional\n\n\n@dataclass\nclass RotationItem:\n value: str\n last_error: Optional[str] = None\n\n\nclass RotationPool:\n def __init__(self, items: List[str], blacklist_ttl: int = 300, name: str = \"\"):\n self.items = [RotationItem(value=item) for item in items if item]\n self.blacklist_ttl = max(0, int(blacklist_ttl))\n self.name = name or \"rotation\"\n self._blacklist: Dict[str, float] = {}\n\n def _cleanup_blacklist(self) -> None:\n now = time.time()\n expired = [key for key, ts in self._blacklist.items() if ts <= now]\n for key in expired:\n self._blacklist.pop(key, None)\n\n def available_items(self) -> List[RotationItem]:\n self._cleanup_blacklist()\n return [item for item in self.items if item.value not in self._blacklist]\n\n def pick_random(self) -> Optional[RotationItem]:\n candidates = self.available_items()\n if not candidates:\n return None\n return random.choice(candidates)\n\n def mark_bad(self, item: Optional[RotationItem], reason: str = \"\") -> None:\n if not item:\n return\n item.last_error = reason\n if self.blacklist_ttl <= 0:\n return\n self._blacklist[item.value] = time.time() + self.blacklist_ttl\n\n\ndef parse_proxy_pool(value: Optional[str]) -> List[str]:\n if not value:\n return []\n if isinstance(value, list):\n return [str(item).strip() for item in value if str(item).strip()]\n return [entry.strip() for entry in str(value).split(\",\") if entry.strip()]\n\n\ndef load_state_files(state_dir: str) -> List[str]:\n if not state_dir:\n return []\n if not os.path.isdir(state_dir):\n return []\n files = []\n for name in os.listdir(state_dir):\n if name.endswith(\".json\"):\n files.append(os.path.join(state_dir, name))\n return sorted(files)\n" }, { "path": "src/scraper.py", "content": "import asyncio\nimport json\nimport os\nimport random\nfrom datetime import datetime\nfrom typing import Optional\nfrom urllib.parse import urlencode\n\nfrom playwright.async_api import (\n Response,\n TimeoutError as PlaywrightTimeoutError,\n async_playwright,\n)\n\nfrom src.ai_handler import (\n download_all_images,\n get_ai_analysis,\n send_ntfy_notification,\n cleanup_task_images,\n)\nfrom src.config import (\n AI_DEBUG_MODE,\n DETAIL_API_URL_PATTERN,\n LOGIN_IS_EDGE,\n RUN_HEADLESS,\n RUNNING_IN_DOCKER,\n SKIP_AI_ANALYSIS,\n STATE_FILE,\n)\nfrom src.parsers import (\n _parse_search_results_json,\n _parse_user_items_data,\n calculate_reputation_from_ratings,\n parse_ratings_data,\n parse_user_head_data,\n)\nfrom src.utils import (\n format_registration_days,\n get_link_unique_key,\n log_time,\n random_sleep,\n safe_get,\n save_to_jsonl,\n)\nfrom src.rotation import RotationPool, load_state_files, parse_proxy_pool, RotationItem\nfrom src.failure_guard import FailureGuard\nfrom src.services.account_strategy_service import resolve_account_runtime_plan\nfrom src.infrastructure.persistence.storage_names import build_result_filename\nfrom src.services.item_analysis_dispatcher import (\n ItemAnalysisDispatcher,\n ItemAnalysisJob,\n)\nfrom src.services.price_history_service import (\n build_market_reference,\n load_price_snapshots,\n record_market_snapshots,\n)\nfrom src.services.result_storage_service import load_processed_link_keys\nfrom src.services.seller_profile_cache import SellerProfileCache\nfrom src.services.search_pagination import (\n advance_search_page,\n is_search_results_response,\n)\n\n\nclass RiskControlError(Exception):\n pass\n\n\nclass LoginRequiredError(Exception):\n \"\"\"Raised when Goofish redirects to the passport/mini_login flow.\"\"\"\n\n\nFAILURE_GUARD = FailureGuard()\nEDGE_DOCKER_WARNING_PRINTED = False\n\n\ndef _is_login_url(url: str) -> bool:\n if not url:\n return False\n lowered = url.lower()\n return \"passport.goofish.com\" in lowered or \"mini_login\" in lowered\n\n\ndef _resolve_browser_channel() -> str:\n global EDGE_DOCKER_WARNING_PRINTED\n if RUNNING_IN_DOCKER:\n if LOGIN_IS_EDGE and not EDGE_DOCKER_WARNING_PRINTED:\n print(\n \"检测到 LOGIN_IS_EDGE=true，但 Docker 镜像未内置 Edge，\"\n \"任务运行时将改用 Chromium。\"\n )\n EDGE_DOCKER_WARNING_PRINTED = True\n return \"chromium\"\n return \"msedge\" if LOGIN_IS_EDGE else \"chrome\"\n\n\ndef _should_analyze_images(task_config: dict) -> bool:\n raw_value = task_config.get(\"analyze_images\", True)\n if isinstance(raw_value, bool):\n return raw_value\n return str(raw_value).strip().lower() not in {\"false\", \"0\", \"no\", \"off\"}\n\n\ndef _format_failure_reason(reason: str, limit: int = 500) -> str:\n if not reason:\n return \"未知错误\"\n cleaned = \" \".join(str(reason).split())\n if len(cleaned) <= limit:\n return cleaned\n return cleaned[: limit - 3] + \"...\"\n\n\nasync def _notify_task_failure(\n task_config: dict, reason: str, *, cookie_path: Optional[str]\n) -> None:\n task_name = task_config.get(\"task_name\", \"未命名任务\")\n keyword = task_config.get(\"keyword\", \"\")\n formatted_reason = _format_failure_reason(reason)\n\n # Some failures are deterministic misconfiguration and should pause/notify immediately.\n pause_immediately = any(\n marker in formatted_reason\n for marker in (\n \"未找到可用的代理地址\",\n \"未找到可用的登录状态文件\",\n )\n )\n\n guard_result = FAILURE_GUARD.record_failure(\n task_name,\n formatted_reason,\n cookie_path=cookie_path,\n min_failures_to_pause=1 if pause_immediately else None,\n )\n\n if not guard_result.get(\"should_notify\"):\n print(\n f\"[FailureGuard] 任务 '{task_name}' 失败计数 {guard_result.get('consecutive_failures')}/{FAILURE_GUARD.threshold}，暂不通知。\"\n )\n return\n\n paused_until = guard_result.get(\"paused_until\")\n paused_until_str = (\n paused_until.strftime(\"%Y-%m-%d %H:%M:%S\") if paused_until else \"N/A\"\n )\n\n product_data = {\n \"商品标题\": f\"[任务异常] {task_name}\",\n \"当前售价\": \"N/A\",\n \"商品链接\": \"#\",\n }\n notify_reason = (\n f\"任务运行失败(已连续 {guard_result.get('consecutive_failures')}/{FAILURE_GUARD.threshold} 次): {formatted_reason}\"\n f\"\\n任务: {task_name}\"\n f\"\\n关键词: {keyword or 'N/A'}\"\n f\"\\n已自动暂停重试，暂停到: {paused_until_str}\"\n f\"\\n修复后(更新登录态/cookies文件)将自动恢复。\"\n )\n\n try:\n await send_ntfy_notification(product_data, notify_reason)\n except Exception as e:\n print(f\"发送任务异常通知失败: {e}\")\n\n\ndef _as_bool(value, default: bool = False) -> bool:\n if value is None:\n return default\n if isinstance(value, bool):\n return value\n return str(value).strip().lower() in {\"1\", \"true\", \"yes\", \"y\", \"on\"}\n\n\ndef _as_int(value, default: int) -> int:\n if value is None:\n return default\n try:\n return int(value)\n except (TypeError, ValueError):\n return default\n\n\ndef _get_rotation_settings(task_config: dict) -> dict:\n account_cfg = task_config.get(\"account_rotation\") or {}\n proxy_cfg = task_config.get(\"proxy_rotation\") or {}\n\n account_enabled = _as_bool(\n account_cfg.get(\"enabled\"),\n _as_bool(os.getenv(\"ACCOUNT_ROTATION_ENABLED\"), False),\n )\n account_mode = (\n account_cfg.get(\"mode\") or os.getenv(\"ACCOUNT_ROTATION_MODE\", \"per_task\")\n ).lower()\n account_state_dir = account_cfg.get(\"state_dir\") or os.getenv(\n \"ACCOUNT_STATE_DIR\", \"state\"\n )\n account_retry_limit = _as_int(\n account_cfg.get(\"retry_limit\"),\n _as_int(os.getenv(\"ACCOUNT_ROTATION_RETRY_LIMIT\"), 2),\n )\n account_blacklist_ttl = _as_int(\n account_cfg.get(\"blacklist_ttl_sec\"),\n _as_int(os.getenv(\"ACCOUNT_BLACKLIST_TTL\"), 300),\n )\n\n proxy_enabled = _as_bool(\n proxy_cfg.get(\"enabled\"), _as_bool(os.getenv(\"PROXY_ROTATION_ENABLED\"), False)\n )\n proxy_mode = (\n proxy_cfg.get(\"mode\") or os.getenv(\"PROXY_ROTATION_MODE\", \"per_task\")\n ).lower()\n proxy_pool = proxy_cfg.get(\"proxy_pool\") or os.getenv(\"PROXY_POOL\", \"\")\n proxy_retry_limit = _as_int(\n proxy_cfg.get(\"retry_limit\"),\n _as_int(os.getenv(\"PROXY_ROTATION_RETRY_LIMIT\"), 2),\n )\n proxy_blacklist_ttl = _as_int(\n proxy_cfg.get(\"blacklist_ttl_sec\"),\n _as_int(os.getenv(\"PROXY_BLACKLIST_TTL\"), 300),\n )\n\n return {\n \"account_enabled\": account_enabled,\n \"account_mode\": account_mode,\n \"account_state_dir\": account_state_dir,\n \"account_retry_limit\": max(1, account_retry_limit),\n \"account_blacklist_ttl\": max(0, account_blacklist_ttl),\n \"proxy_enabled\": proxy_enabled,\n \"proxy_mode\": proxy_mode,\n \"proxy_pool\": proxy_pool,\n \"proxy_retry_limit\": max(1, proxy_retry_limit),\n \"proxy_blacklist_ttl\": max(0, proxy_blacklist_ttl),\n }\n\n\ndef _get_ai_analysis_concurrency(task_config: dict) -> int:\n configured = task_config.get(\"ai_analysis_concurrency\")\n default = _as_int(os.getenv(\"AI_ANALYSIS_CONCURRENCY\"), 2)\n return max(1, _as_int(configured, default))\n\n\ndef _get_seller_profile_cache_ttl(task_config: dict) -> int:\n configured = task_config.get(\"seller_profile_cache_ttl\")\n default = _as_int(os.getenv(\"SELLER_PROFILE_CACHE_TTL\"), 1800)\n return max(0, _as_int(configured, default))\n\n\ndef _default_context_options() -> dict:\n return {\n \"user_agent\": \"Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Mobile Safari/537.36\",\n \"viewport\": {\"width\": 412, \"height\": 915},\n \"device_scale_factor\": 2.625,\n \"is_mobile\": True,\n \"has_touch\": True,\n \"locale\": \"zh-CN\",\n \"timezone_id\": \"Asia/Shanghai\",\n \"permissions\": [\"geolocation\"],\n \"geolocation\": {\"longitude\": 121.4737, \"latitude\": 31.2304},\n \"color_scheme\": \"light\",\n }\n\n\ndef _clean_kwargs(options: dict) -> dict:\n return {k: v for k, v in options.items() if v is not None}\n\n\ndef _looks_like_mobile(ua: str) -> Optional[bool]:\n if not ua:\n return None\n ua_lower = ua.lower()\n if \"mobile\" in ua_lower or \"android\" in ua_lower or \"iphone\" in ua_lower:\n return True\n if \"windows\" in ua_lower or \"macintosh\" in ua_lower:\n return False\n return None\n\n\ndef _build_context_overrides(snapshot: dict) -> dict:\n env = snapshot.get(\"env\") or {}\n headers = snapshot.get(\"headers\") or {}\n navigator = env.get(\"navigator\") or {}\n screen = env.get(\"screen\") or {}\n intl = env.get(\"intl\") or {}\n\n overrides = {}\n\n ua = (\n headers.get(\"User-Agent\")\n or headers.get(\"user-agent\")\n or navigator.get(\"userAgent\")\n )\n if ua:\n overrides[\"user_agent\"] = ua\n\n accept_language = headers.get(\"Accept-Language\") or headers.get(\"accept-language\")\n locale = None\n if accept_language:\n locale = accept_language.split(\",\")[0].strip()\n elif navigator.get(\"language\"):\n locale = navigator[\"language\"]\n if locale:\n overrides[\"locale\"] = locale\n\n tz = intl.get(\"timeZone\")\n if tz:\n overrides[\"timezone_id\"] = tz\n\n width = screen.get(\"width\")\n height = screen.get(\"height\")\n if isinstance(width, (int, float)) and isinstance(height, (int, float)):\n overrides[\"viewport\"] = {\"width\": int(width), \"height\": int(height)}\n\n dpr = screen.get(\"devicePixelRatio\")\n if isinstance(dpr, (int, float)):\n overrides[\"device_scale_factor\"] = float(dpr)\n\n touch_points = navigator.get(\"maxTouchPoints\")\n if isinstance(touch_points, (int, float)):\n overrides[\"has_touch\"] = touch_points > 0\n\n mobile_flag = _looks_like_mobile(ua or \"\")\n if mobile_flag is not None:\n overrides[\"is_mobile\"] = mobile_flag\n\n return _clean_kwargs(overrides)\n\n\ndef _build_extra_headers(raw_headers: Optional[dict]) -> dict:\n if not raw_headers:\n return {}\n excluded = {\"cookie\", \"content-length\"}\n headers = {}\n for key, value in raw_headers.items():\n if not key or key.lower() in excluded or value is None:\n continue\n headers[key] = value\n return headers\n\n\nasync def scrape_user_profile(context, user_id: str) -> dict:\n \"\"\"\n 【新版】访问指定用户的个人主页，按顺序采集其摘要信息、完整的商品列表和完整的评价列表。\n \"\"\"\n print(f\" -> 开始采集用户ID: {user_id} 的完整信息...\")\n profile_data = {}\n page = await context.new_page()\n\n # 为各项异步任务准备Future和数据容器\n head_api_future = asyncio.get_event_loop().create_future()\n\n all_items, all_ratings = [], []\n stop_item_scrolling, stop_rating_scrolling = asyncio.Event(), asyncio.Event()\n\n async def handle_response(response: Response):\n # 捕获头部摘要API\n if (\n \"mtop.idle.web.user.page.head\" in response.url\n and not head_api_future.done()\n ):\n try:\n head_api_future.set_result(await response.json())\n print(f\" [API捕获] 用户头部信息... 成功\")\n except Exception as e:\n if not head_api_future.done():\n head_api_future.set_exception(e)\n\n # 捕获商品列表API\n elif \"mtop.idle.web.xyh.item.list\" in response.url:\n try:\n data = await response.json()\n all_items.extend(data.get(\"data\", {}).get(\"cardList\", []))\n print(f\" [API捕获] 商品列表... 当前已捕获 {len(all_items)} 件\")\n if not data.get(\"data\", {}).get(\"nextPage\", True):\n stop_item_scrolling.set()\n except Exception as e:\n stop_item_scrolling.set()\n\n # 捕获评价列表API\n elif \"mtop.idle.web.trade.rate.list\" in response.url:\n try:\n data = await response.json()\n all_ratings.extend(data.get(\"data\", {}).get(\"cardList\", []))\n print(f\" [API捕获] 评价列表... 当前已捕获 {len(all_ratings)} 条\")\n if not data.get(\"data\", {}).get(\"nextPage\", True):\n stop_rating_scrolling.set()\n except Exception as e:\n stop_rating_scrolling.set()\n\n page.on(\"response\", handle_response)\n\n try:\n # --- 任务1: 导航并采集头部信息 ---\n await page.goto(\n f\"https://www.goofish.com/personal?userId={user_id}\",\n wait_until=\"domcontentloaded\",\n timeout=20000,\n )\n head_data = await asyncio.wait_for(head_api_future, timeout=15)\n profile_data = await parse_user_head_data(head_data)\n\n # --- 任务2: 滚动加载所有商品 (默认页面) ---\n print(\" [采集阶段] 开始采集该用户的商品列表...\")\n await random_sleep(2, 4) # 等待第一页商品API完成\n while not stop_item_scrolling.is_set():\n await page.evaluate(\"window.scrollTo(0, document.body.scrollHeight)\")\n try:\n await asyncio.wait_for(stop_item_scrolling.wait(), timeout=8)\n except asyncio.TimeoutError:\n print(\" [滚动超时] 商品列表可能已加载完毕。\")\n break\n profile_data[\"卖家发布的商品列表\"] = await _parse_user_items_data(all_items)\n\n # --- 任务3: 点击并采集所有评价 ---\n print(\" [采集阶段] 开始采集该用户的评价列表...\")\n rating_tab_locator = page.locator(\"//div[text()='信用及评价']/ancestor::li\")\n if await rating_tab_locator.count() > 0:\n await rating_tab_locator.click()\n await random_sleep(3, 5) # 等待第一页评价API完成\n\n while not stop_rating_scrolling.is_set():\n await page.evaluate(\"window.scrollTo(0, document.body.scrollHeight)\")\n try:\n await asyncio.wait_for(stop_rating_scrolling.wait(), timeout=8)\n except asyncio.TimeoutError:\n print(\" [滚动超时] 评价列表可能已加载完毕。\")\n break\n\n profile_data[\"卖家收到的评价列表\"] = await parse_ratings_data(all_ratings)\n reputation_stats = await calculate_reputation_from_ratings(all_ratings)\n profile_data.update(reputation_stats)\n else:\n print(\" [警告] 未找到评价选项卡，跳过评价采集。\")\n\n except Exception as e:\n print(f\" [错误] 采集用户 {user_id} 信息时发生错误: {e}\")\n finally:\n page.remove_listener(\"response\", handle_response)\n await page.close()\n print(f\" -> 用户 {user_id} 信息采集完成。\")\n\n return profile_data\n\n\nasync def scrape_xianyu(task_config: dict, debug_limit: int = 0):\n \"\"\"\n 【核心执行器】\n 根据单个任务配置，异步爬取闲鱼商品数据，并对每个新发现的商品进行实时的、独立的AI分析和通知。\n \"\"\"\n keyword = task_config[\"keyword\"]\n max_pages = task_config.get(\"max_pages\", 1)\n personal_only = task_config.get(\"personal_only\", False)\n min_price = task_config.get(\"min_price\")\n max_price = task_config.get(\"max_price\")\n ai_prompt_text = task_config.get(\"ai_prompt_text\", \"\")\n analyze_images = _should_analyze_images(task_config)\n decision_mode = str(task_config.get(\"decision_mode\", \"ai\")).strip().lower()\n if decision_mode not in {\"ai\", \"keyword\"}:\n decision_mode = \"ai\"\n keyword_rules = task_config.get(\"keyword_rules\") or []\n free_shipping = task_config.get(\"free_shipping\", False)\n raw_new_publish = task_config.get(\"new_publish_option\") or \"\"\n new_publish_option = raw_new_publish.strip()\n if new_publish_option == \"__none__\":\n new_publish_option = \"\"\n region_filter = (task_config.get(\"region\") or \"\").strip()\n\n processed_links = set()\n history_run_id = datetime.now().strftime(\"%Y%m%d%H%M%S\")\n history_seen_item_ids: set[str] = set()\n historical_snapshots = load_price_snapshots(keyword)\n result_filename = build_result_filename(keyword)\n processed_links = load_processed_link_keys(keyword)\n if processed_links:\n print(f\"LOG: 发现已存在结果集 {result_filename}，已加载 {len(processed_links)} 个历史商品用于去重。\")\n else:\n print(f\"LOG: 结果集 {result_filename} 当前为空，将写入新记录。\")\n\n rotation_settings = _get_rotation_settings(task_config)\n account_items = load_state_files(rotation_settings[\"account_state_dir\"])\n runtime_plan = resolve_account_runtime_plan(\n strategy=task_config.get(\"account_strategy\"),\n account_state_file=task_config.get(\"account_state_file\"),\n has_root_state_file=os.path.exists(STATE_FILE),\n available_account_files=account_items,\n )\n forced_account = runtime_plan[\"forced_account\"]\n if runtime_plan[\"prefer_root_state\"]:\n account_items = [STATE_FILE]\n rotation_settings[\"account_enabled\"] = False\n elif runtime_plan[\"use_account_pool\"]:\n rotation_settings[\"account_enabled\"] = True\n else:\n rotation_settings[\"account_enabled\"] = False\n\n account_pool = RotationPool(\n account_items, rotation_settings[\"account_blacklist_ttl\"], \"account\"\n )\n proxy_pool = RotationPool(\n parse_proxy_pool(rotation_settings[\"proxy_pool\"]),\n rotation_settings[\"proxy_blacklist_ttl\"],\n \"proxy\",\n )\n\n selected_account: Optional[RotationItem] = None\n selected_proxy: Optional[RotationItem] = None\n\n def _select_account(force_new: bool = False) -> Optional[RotationItem]:\n nonlocal selected_account\n if forced_account:\n return RotationItem(value=forced_account)\n if not rotation_settings[\"account_enabled\"]:\n if os.path.exists(STATE_FILE):\n return RotationItem(value=STATE_FILE)\n return None\n if (\n rotation_settings[\"account_mode\"] == \"per_task\"\n and selected_account\n and not force_new\n ):\n return selected_account\n picked = account_pool.pick_random()\n return picked or selected_account\n\n def _select_proxy(force_new: bool = False) -> Optional[RotationItem]:\n nonlocal selected_proxy\n if not rotation_settings[\"proxy_enabled\"]:\n return None\n if (\n rotation_settings[\"proxy_mode\"] == \"per_task\"\n and selected_proxy\n and not force_new\n ):\n return selected_proxy\n picked = proxy_pool.pick_random()\n return picked or selected_proxy\n\n async def _run_scrape_attempt(state_file: str, proxy_server: Optional[str]) -> int:\n processed_item_count = 0\n stop_scraping = False\n\n if not os.path.exists(state_file):\n raise FileNotFoundError(f\"登录状态文件不存在: {state_file}\")\n\n snapshot_data = None\n try:\n with open(state_file, \"r\", encoding=\"utf-8\") as f:\n snapshot_data = json.load(f)\n except Exception as e:\n print(f\"警告：读取登录状态文件失败，将直接按路径使用: {e}\")\n\n async with async_playwright() as p:\n # 反检测启动参数\n launch_args = [\n \"--disable-blink-features=AutomationControlled\",\n \"--disable-dev-shm-usage\",\n \"--no-sandbox\",\n \"--disable-setuid-sandbox\",\n \"--disable-web-security\",\n \"--disable-features=IsolateOrigins,site-per-process\",\n ]\n\n launch_kwargs = {\"headless\": RUN_HEADLESS, \"args\": launch_args}\n if proxy_server:\n launch_kwargs[\"proxy\"] = {\"server\": proxy_server}\n\n launch_kwargs[\"channel\"] = _resolve_browser_channel()\n\n browser = await p.chromium.launch(**launch_kwargs)\n\n context_kwargs = _default_context_options()\n storage_state_arg = state_file\n analysis_dispatcher: Optional[ItemAnalysisDispatcher] = None\n\n if isinstance(snapshot_data, dict):\n # 新版扩展导出的增强快照，包含环境和Header\n if any(\n key in snapshot_data\n for key in (\"env\", \"headers\", \"page\", \"storage\")\n ):\n print(f\"检测到增强浏览器快照，应用环境参数: {state_file}\")\n storage_state_arg = {\"cookies\": snapshot_data.get(\"cookies\", [])}\n context_kwargs.update(_build_context_overrides(snapshot_data))\n extra_headers = _build_extra_headers(snapshot_data.get(\"headers\"))\n if extra_headers:\n context_kwargs[\"extra_http_headers\"] = extra_headers\n else:\n storage_state_arg = snapshot_data\n\n context_kwargs = _clean_kwargs(context_kwargs)\n context = await browser.new_context(\n storage_state=storage_state_arg, **context_kwargs\n )\n seller_profile_cache = SellerProfileCache(\n ttl_seconds=_get_seller_profile_cache_ttl(task_config)\n )\n analysis_dispatcher = ItemAnalysisDispatcher(\n concurrency=_get_ai_analysis_concurrency(task_config),\n skip_ai_analysis=SKIP_AI_ANALYSIS,\n seller_loader=lambda user_id: seller_profile_cache.get_or_load(\n str(user_id),\n lambda seller_key: scrape_user_profile(context, seller_key),\n ),\n image_downloader=download_all_images,\n ai_analyzer=get_ai_analysis,\n notifier=send_ntfy_notification,\n saver=save_to_jsonl,\n )\n\n # 增强反检测脚本（模拟真实移动设备）\n await context.add_init_script(\"\"\"\n // 移除webdriver标识\n Object.defineProperty(navigator, 'webdriver', {get: () => undefined});\n\n // 模拟真实移动设备的navigator属性\n Object.defineProperty(navigator, 'plugins', {get: () => [1, 2, 3, 4, 5]});\n Object.defineProperty(navigator, 'languages', {get: () => ['zh-CN', 'zh', 'en-US', 'en']});\n\n // 添加chrome对象\n window.chrome = {runtime: {}, loadTimes: function() {}, csi: function() {}};\n\n // 模拟触摸支持\n Object.defineProperty(navigator, 'maxTouchPoints', {get: () => 5});\n\n // 覆盖permissions查询（避免暴露自动化）\n const originalQuery = window.navigator.permissions.query;\n window.navigator.permissions.query = (parameters) => (\n parameters.name === 'notifications' ?\n Promise.resolve({state: Notification.permission}) :\n originalQuery(parameters)\n );\n \"\"\")\n\n page = await context.new_page()\n\n try:\n # 步骤 0 - 模拟真实用户：先访问首页（重要的反检测措施）\n log_time(\"步骤 0 - 模拟真实用户访问首页...\")\n await page.goto(\n \"https://www.goofish.com/\",\n wait_until=\"domcontentloaded\",\n timeout=30000,\n )\n log_time(\"[反爬] 在首页停留，模拟浏览...\")\n await random_sleep(1, 2)\n\n # 模拟随机滚动（移动设备的触摸滚动）\n await page.evaluate(\"window.scrollBy(0, Math.random() * 500 + 200)\")\n await random_sleep(1, 2)\n\n log_time(\"步骤 1 - 导航到搜索结果页...\")\n # 使用 'q' 参数构建正确的搜索URL，并进行URL编码\n params = {\"q\": keyword}\n search_url = f\"https://www.goofish.com/search?{urlencode(params)}\"\n log_time(f\"目标URL: {search_url}\")\n\n # 先监听搜索接口响应，再执行导航，避免错过首次请求\n async with page.expect_response(\n is_search_results_response, timeout=30000\n ) as initial_response_info:\n await page.goto(\n search_url, wait_until=\"domcontentloaded\", timeout=60000\n )\n if _is_login_url(page.url):\n raise LoginRequiredError(\n f\"Login required: redirected to {page.url} (cookies/state likely expired)\"\n )\n\n # 捕获初始搜索的API数据\n initial_response = await initial_response_info.value\n\n # 等待页面加载出关键筛选元素，以确认已成功进入搜索结果页\n try:\n await page.wait_for_selector(\"text=新发布\", timeout=15000)\n except PlaywrightTimeoutError as e:\n if _is_login_url(page.url):\n raise LoginRequiredError(\n f\"Login required: redirected to {page.url} (cookies/state likely expired)\"\n ) from e\n raise\n\n # 模拟真实用户行为：页面加载后的初始停留和浏览\n log_time(\"[反爬] 模拟用户查看页面...\")\n await random_sleep(1, 3)\n\n # --- 新增：检查是否存在验证弹窗 ---\n baxia_dialog = page.locator(\"div.baxia-dialog-mask\")\n middleware_widget = page.locator(\"div.J_MIDDLEWARE_FRAME_WIDGET\")\n try:\n # 等待弹窗在2秒内出现。如果出现，则执行块内代码。\n await baxia_dialog.wait_for(state=\"visible\", timeout=2000)\n print(\n \"\\n==================== CRITICAL BLOCK DETECTED ====================\"\n )\n print(\"检测到闲鱼反爬虫验证弹窗 (baxia-dialog)，无法继续操作。\")\n print(\"这通常是因为操作过于频繁或被识别为机器人。\")\n print(\"建议：\")\n print(\"1. 停止脚本一段时间再试。\")\n print(\n \"2. (推荐) 在 .env 文件中设置 RUN_HEADLESS=false，以非无头模式运行，这有助于绕过检测。\"\n )\n print(f\"任务 '{keyword}' 将在此处中止。\")\n print(\n \"===================================================================\"\n )\n raise RiskControlError(\"baxia-dialog\")\n except PlaywrightTimeoutError:\n # 2秒内弹窗未出现，这是正常情况，继续执行\n pass\n\n # 检查是否有J_MIDDLEWARE_FRAME_WIDGET覆盖层\n try:\n await middleware_widget.wait_for(state=\"visible\", timeout=2000)\n print(\n \"\\n==================== CRITICAL BLOCK DETECTED ====================\"\n )\n print(\n \"检测到闲鱼反爬虫验证弹窗 (J_MIDDLEWARE_FRAME_WIDGET)，无法继续操作。\"\n )\n print(\"这通常是因为操作过于频繁或被识别为机器人。\")\n print(\"建议：\")\n print(\"1. 停止脚本一段时间再试。\")\n print(\"2. (推荐) 更新登录状态文件，确保登录状态有效。\")\n print(\"3. 降低任务执行频率，避免被识别为机器人。\")\n print(f\"任务 '{keyword}' 将在此处中止。\")\n print(\n \"===================================================================\"\n )\n raise RiskControlError(\"J_MIDDLEWARE_FRAME_WIDGET\")\n except PlaywrightTimeoutError:\n # 2秒内弹窗未出现，这是正常情况，继续执行\n pass\n # --- 结束新增 ---\n\n try:\n await page.click(\"div[class*='closeIconBg']\", timeout=3000)\n print(\"LOG: 已关闭广告弹窗。\")\n except PlaywrightTimeoutError:\n print(\"LOG: 未检测到广告弹窗。\")\n\n final_response = None\n log_time(\"步骤 2 - 应用筛选条件...\")\n if new_publish_option:\n try:\n await page.click(\"text=新发布\")\n await random_sleep(1, 2) # 原来是 (1.5, 2.5)\n async with page.expect_response(\n is_search_results_response, timeout=20000\n ) as response_info:\n await page.click(f\"text={new_publish_option}\")\n # --- 修改: 增加排序后的等待时间 ---\n await random_sleep(2, 4) # 原来是 (3, 5)\n final_response = await response_info.value\n except PlaywrightTimeoutError:\n log_time(\n f\"新发布筛选 '{new_publish_option}' 请求超时，继续执行。\"\n )\n except Exception as e:\n print(f\"LOG: 应用新发布筛选失败: {e}\")\n\n if personal_only:\n async with page.expect_response(\n is_search_results_response, timeout=20000\n ) as response_info:\n await page.click(\"text=个人闲置\")\n # --- 修改: 将固定等待改为随机等待，并加长 ---\n await random_sleep(2, 4) # 原来是 asyncio.sleep(5)\n final_response = await response_info.value\n\n if free_shipping:\n try:\n async with page.expect_response(\n is_search_results_response, timeout=20000\n ) as response_info:\n await page.click(\"text=包邮\")\n await random_sleep(2, 4)\n final_response = await response_info.value\n except PlaywrightTimeoutError:\n log_time(\"包邮筛选请求超时，继续执行。\")\n except Exception as e:\n print(f\"LOG: 应用包邮筛选失败: {e}\")\n\n if region_filter:\n try:\n area_trigger = page.get_by_text(\"区域\", exact=True)\n if await area_trigger.count():\n await area_trigger.first.click()\n await random_sleep(1.5, 2)\n popover_candidates = page.locator(\"div.ant-popover\")\n popover = popover_candidates.filter(\n has=page.locator(\n \".areaWrap--FaZHsn8E, [class*='areaWrap']\"\n )\n ).last\n if not await popover.count():\n popover = popover_candidates.filter(\n has=page.get_by_text(\"重新定位\")\n ).last\n if not await popover.count():\n popover = popover_candidates.filter(\n has=page.get_by_text(\"查看\")\n ).last\n if not await popover.count():\n print(\"LOG: 未找到区域弹窗，跳过区域筛选。\")\n raise PlaywrightTimeoutError(\"region-popover-not-found\")\n await popover.wait_for(state=\"visible\", timeout=5000)\n\n # 列表容器：第一层 children 即省/市/区三列，不再强依赖具体类名，提升鲁棒性\n area_wrap = popover.locator(\n \".areaWrap--FaZHsn8E, [class*='areaWrap']\"\n ).first\n await area_wrap.wait_for(state=\"visible\", timeout=3000)\n columns = area_wrap.locator(\":scope > div\")\n col_prov = columns.nth(0)\n col_city = columns.nth(1)\n col_dist = columns.nth(2)\n\n region_parts = [\n p.strip() for p in region_filter.split(\"/\") if p.strip()\n ]\n\n async def _click_in_column(\n column_locator, text_value: str, desc: str\n ) -> None:\n option = column_locator.locator(\n \".provItem--QAdOx8nD\", has_text=text_value\n ).first\n if await option.count():\n await option.click()\n await random_sleep(1.5, 2)\n try:\n await option.wait_for(\n state=\"attached\", timeout=1500\n )\n await option.wait_for(\n state=\"visible\", timeout=1500\n )\n except PlaywrightTimeoutError:\n pass\n else:\n print(f\"LOG: 未找到{desc} '{text_value}'，跳过。\")\n\n if len(region_parts) >= 1:\n await _click_in_column(\n col_prov, region_parts[0], \"省份\"\n )\n await random_sleep(1, 2)\n if len(region_parts) >= 2:\n await _click_in_column(\n col_city, region_parts[1], \"城市\"\n )\n await random_sleep(1, 2)\n if len(region_parts) >= 3:\n await _click_in_column(\n col_dist, region_parts[2], \"区/县\"\n )\n await random_sleep(1, 2)\n\n search_btn = popover.locator(\n \"div.searchBtn--Ic6RKcAb\"\n ).first\n if await search_btn.count():\n try:\n async with page.expect_response(\n is_search_results_response,\n timeout=20000,\n ) as response_info:\n await search_btn.click()\n await random_sleep(2, 3)\n final_response = await response_info.value\n except PlaywrightTimeoutError:\n log_time(\"区域筛选提交超时，继续执行。\")\n else:\n print(\n \"LOG: 未找到区域弹窗的“查看XX件宝贝”按钮，跳过提交。\"\n )\n else:\n print(\"LOG: 未找到区域筛选触发器。\")\n except PlaywrightTimeoutError:\n log_time(f\"区域筛选 '{region_filter}' 请求超时，继续执行。\")\n except Exception as e:\n print(f\"LOG: 应用区域筛选 '{region_filter}' 失败: {e}\")\n\n if min_price or max_price:\n price_container = page.locator(\n 'div[class*=\"search-price-input-container\"]'\n ).first\n if await price_container.is_visible():\n if min_price:\n await price_container.get_by_placeholder(\"¥\").first.fill(\n min_price\n )\n # --- 修改: 将固定等待改为随机等待 ---\n await random_sleep(1, 2.5) # 原来是 asyncio.sleep(5)\n if max_price:\n await (\n price_container.get_by_placeholder(\"¥\")\n .nth(1)\n .fill(max_price)\n )\n # --- 修改: 将固定等待改为随机等待 ---\n await random_sleep(1, 2.5) # 原来是 asyncio.sleep(5)\n\n async with page.expect_response(\n is_search_results_response, timeout=20000\n ) as response_info:\n await page.keyboard.press(\"Tab\")\n # --- 修改: 增加确认价格后的等待时间 ---\n await random_sleep(2, 4) # 原来是 asyncio.sleep(5)\n final_response = await response_info.value\n else:\n print(\"LOG: 警告 - 未找到价格输入容器。\")\n\n log_time(\"所有筛选已完成，开始处理商品列表...\")\n\n current_response = (\n final_response\n if final_response and final_response.ok\n else initial_response\n )\n for page_num in range(1, max_pages + 1):\n if stop_scraping:\n break\n log_time(f\"开始处理第 {page_num}/{max_pages} 页 ...\")\n\n if page_num > 1:\n page_advance_result = await advance_search_page(\n page=page,\n page_num=page_num,\n )\n if not page_advance_result.advanced:\n break\n current_response = page_advance_result.response\n\n if not (current_response and current_response.ok):\n log_time(f\"第 {page_num} 页响应无效，跳过。\")\n continue\n\n basic_items = await _parse_search_results_json(\n await current_response.json(), f\"第 {page_num} 页\"\n )\n if not basic_items:\n break\n historical_snapshots.extend(\n record_market_snapshots(\n keyword=keyword,\n task_name=task_config.get(\"task_name\", \"Untitled Task\"),\n items=basic_items,\n run_id=history_run_id,\n snapshot_time=datetime.now().isoformat(),\n seen_item_ids=history_seen_item_ids,\n )\n )\n\n total_items_on_page = len(basic_items)\n for i, item_data in enumerate(basic_items, 1):\n if debug_limit > 0 and processed_item_count >= debug_limit:\n log_time(\n f\"已达到调试上限 ({debug_limit})，停止获取新商品。\"\n )\n stop_scraping = True\n break\n\n unique_key = get_link_unique_key(item_data[\"商品链接\"])\n if unique_key in processed_links:\n log_time(\n f\"[页内进度 {i}/{total_items_on_page}] 商品 '{item_data['商品标题'][:20]}...' 已存在，跳过。\"\n )\n continue\n\n log_time(\n f\"[页内进度 {i}/{total_items_on_page}] 发现新商品，获取详情: {item_data['商品标题'][:30]}...\"\n )\n # --- 修改: 访问详情页前的等待时间，模拟用户在列表页上看了一会儿 ---\n await random_sleep(2, 4) # 原来是 (2, 4)\n\n detail_page = await context.new_page()\n try:\n async with detail_page.expect_response(\n lambda r: DETAIL_API_URL_PATTERN in r.url, timeout=25000\n ) as detail_info:\n await detail_page.goto(\n item_data[\"商品链接\"],\n wait_until=\"domcontentloaded\",\n timeout=25000,\n )\n\n detail_response = await detail_info.value\n if detail_response.ok:\n detail_json = await detail_response.json()\n\n ret_string = str(\n await safe_get(detail_json, \"ret\", default=[])\n )\n if \"FAIL_SYS_USER_VALIDATE\" in ret_string:\n print(\n \"\\n==================== CRITICAL BLOCK DETECTED ====================\"\n )\n print(\n \"检测到闲鱼反爬虫验证 (FAIL_SYS_USER_VALIDATE)，程序将终止。\"\n )\n long_sleep_duration = random.randint(3, 60)\n print(\n f\"为避免账户风险，将执行一次长时间休眠 ({long_sleep_duration} 秒) 后再退出...\"\n )\n await asyncio.sleep(long_sleep_duration)\n print(\"长时间休眠结束，现在将安全退出。\")\n print(\n \"===================================================================\"\n )\n raise RiskControlError(\"FAIL_SYS_USER_VALIDATE\")\n\n # 解析商品详情数据并更新 item_data\n item_do = await safe_get(\n detail_json, \"data\", \"itemDO\", default={}\n )\n seller_do = await safe_get(\n detail_json, \"data\", \"sellerDO\", default={}\n )\n\n reg_days_raw = await safe_get(\n seller_do, \"userRegDay\", default=0\n )\n registration_duration_text = format_registration_days(\n reg_days_raw\n )\n\n # --- START: 新增代码块 ---\n\n # 1. 提取卖家的芝麻信用信息\n zhima_credit_text = await safe_get(\n seller_do, \"zhimaLevelInfo\", \"levelName\"\n )\n\n # 2. 提取该商品的完整图片列表\n image_infos = await safe_get(\n item_do, \"imageInfos\", default=[]\n )\n if image_infos:\n # 使用列表推导式获取所有有效的图片URL\n all_image_urls = [\n img.get(\"url\")\n for img in image_infos\n if img.get(\"url\")\n ]\n if all_image_urls:\n # 用新的字段存储图片列表，替换掉旧的单个链接\n item_data[\"商品图片列表\"] = all_image_urls\n # (可选) 仍然保留主图链接，以防万一\n item_data[\"商品主图链接\"] = all_image_urls[0]\n\n # --- END: 新增代码块 ---\n item_data[\"“想要”人数\"] = await safe_get(\n item_do,\n \"wantCnt\",\n default=item_data.get(\"“想要”人数\", \"NaN\"),\n )\n item_data[\"浏览量\"] = await safe_get(\n item_do, \"browseCnt\", default=\"-\"\n )\n # ...[此处可添加更多从详情页解析出的商品信息]...\n\n user_id = await safe_get(seller_do, \"sellerId\")\n\n # 构建基础记录\n final_record = {\n \"爬取时间\": datetime.now().isoformat(),\n \"搜索关键字\": keyword,\n \"任务名称\": task_config.get(\n \"task_name\", \"Untitled Task\"\n ),\n \"商品信息\": item_data,\n \"卖家信息\": {},\n }\n price_reference = build_market_reference(\n keyword=keyword,\n item=item_data,\n current_market_items=basic_items,\n historical_snapshots=historical_snapshots,\n )\n final_record[\"价格参考\"] = price_reference\n final_record[\"price_insight\"] = price_reference.get(\n \"本商品价格位置\", {}\n )\n\n analysis_dispatcher.submit(\n ItemAnalysisJob(\n keyword=keyword,\n task_name=task_config.get(\n \"task_name\", \"Untitled Task\"\n ),\n decision_mode=decision_mode,\n analyze_images=analyze_images,\n prompt_text=ai_prompt_text,\n keyword_rules=tuple(keyword_rules or []),\n final_record=final_record,\n seller_id=str(user_id) if user_id else None,\n zhima_credit_text=zhima_credit_text,\n registration_duration_text=registration_duration_text,\n )\n )\n\n processed_links.add(unique_key)\n processed_item_count += 1\n log_time(\n f\"商品已提交后台分析。累计处理 {processed_item_count} 个新商品。\"\n )\n\n # --- 修改: 增加单个商品处理后的主要延迟 ---\n log_time(\n \"[反爬] 执行一次主要的随机延迟以模拟用户浏览间隔...\"\n )\n await random_sleep(5, 10)\n else:\n print(\n f\" 错误: 获取商品详情API响应失败，状态码: {detail_response.status}\"\n )\n if AI_DEBUG_MODE:\n print(\n f\"--- [DETAIL DEBUG] FAILED RESPONSE from {item_data['商品链接']} ---\"\n )\n try:\n print(await detail_response.text())\n except Exception as e:\n print(f\"无法读取响应内容: {e}\")\n print(\n \"----------------------------------------------------\"\n )\n\n except PlaywrightTimeoutError:\n print(f\" 错误: 访问商品详情页或等待API响应超时。\")\n except Exception as e:\n print(f\" 错误: 处理商品详情时发生未知错误: {e}\")\n finally:\n await detail_page.close()\n # --- 修改: 增加关闭页面后的短暂整理时间 ---\n await random_sleep(2, 4) # 原来是 (1, 2.5)\n\n # --- 新增: 在处理完一页所有商品后，翻页前，增加一个更长的“休息”时间 ---\n if not stop_scraping and page_num < max_pages:\n print(\n f\"--- 第 {page_num} 页处理完毕，准备翻页。执行一次页面间的长时休息... ---\"\n )\n await random_sleep(10, 15)\n\n except PlaywrightTimeoutError as e:\n if _is_login_url(page.url):\n raise LoginRequiredError(\n f\"Login required: redirected to {page.url} (cookies/state likely expired)\"\n ) from e\n print(f\"\\n操作超时错误: 页面元素或网络响应未在规定时间内出现。\\n{e}\")\n raise\n except asyncio.CancelledError:\n log_time(\"收到取消信号，正在终止当前爬虫任务...\")\n raise\n except Exception as e:\n if type(e).__name__ == \"TargetClosedError\":\n log_time(\"浏览器已关闭，忽略后续异常（可能是任务被停止）。\")\n return processed_item_count\n if \"passport.goofish.com\" in str(e):\n raise LoginRequiredError(\n f\"Login required: redirected to passport flow ({e})\"\n ) from e\n print(f\"\\n爬取过程中发生未知错误: {e}\")\n raise\n finally:\n if analysis_dispatcher is not None:\n log_time(\"等待后台分析任务完成...\")\n await analysis_dispatcher.join()\n log_time(\"任务执行完毕，浏览器将在5秒后自动关闭...\")\n await asyncio.sleep(5)\n if debug_limit:\n input(\"按回车键关闭浏览器...\")\n await browser.close()\n\n return processed_item_count\n\n processed_item_count = 0\n attempt_limit = max(\n rotation_settings[\"account_retry_limit\"],\n rotation_settings[\"proxy_retry_limit\"],\n 1,\n )\n last_error = \"\"\n last_state_path: Optional[str] = None\n\n # If this task is already in a paused state, skip immediately.\n task_name_for_guard = task_config.get(\"task_name\", \"未命名任务\")\n pause_cookie_path = None\n if (\n isinstance(task_config.get(\"account_state_file\"), str)\n and task_config.get(\"account_state_file\").strip()\n ):\n pause_cookie_path = task_config.get(\"account_state_file\").strip()\n elif os.path.exists(STATE_FILE):\n pause_cookie_path = STATE_FILE\n\n decision = FAILURE_GUARD.should_skip_start(\n task_name_for_guard, cookie_path=pause_cookie_path\n )\n if decision.skip:\n print(\n f\"[FailureGuard] 任务 '{task_name_for_guard}' 已暂停重试 (连续失败 {decision.consecutive_failures}/{FAILURE_GUARD.threshold})\"\n )\n if decision.should_notify:\n try:\n await send_ntfy_notification(\n {\n \"商品标题\": f\"[任务暂停] {task_name_for_guard}\",\n \"当前售价\": \"N/A\",\n \"商品链接\": \"#\",\n },\n \"任务处于暂停状态，将跳过执行。\\n\"\n f\"原因: {decision.reason}\\n\"\n f\"连续失败: {decision.consecutive_failures}/{FAILURE_GUARD.threshold}\\n\"\n f\"暂停到: {decision.paused_until.strftime('%Y-%m-%d %H:%M:%S') if decision.paused_until else 'N/A'}\\n\"\n \"修复方法: 更新登录态/cookies文件后会自动恢复。\",\n )\n except Exception as e:\n print(f\"发送任务暂停通知失败: {e}\")\n\n cleanup_task_images(task_config.get(\"task_name\", \"default\"))\n return 0\n\n for attempt in range(1, attempt_limit + 1):\n if attempt == 1:\n selected_account = _select_account()\n selected_proxy = _select_proxy()\n else:\n if (\n rotation_settings[\"account_enabled\"]\n and rotation_settings[\"account_mode\"] == \"on_failure\"\n ):\n account_pool.mark_bad(selected_account, last_error)\n selected_account = _select_account(force_new=True)\n if (\n rotation_settings[\"proxy_enabled\"]\n and rotation_settings[\"proxy_mode\"] == \"on_failure\"\n ):\n proxy_pool.mark_bad(selected_proxy, last_error)\n selected_proxy = _select_proxy(force_new=True)\n\n if rotation_settings[\"account_enabled\"] and not selected_account:\n last_error = \"未找到可用的登录状态文件，无法继续执行任务。\"\n print(last_error)\n break\n if not rotation_settings[\"account_enabled\"] and not selected_account:\n last_error = \"未找到可用的登录状态文件，无法继续执行任务。\"\n print(last_error)\n break\n if rotation_settings[\"proxy_enabled\"] and not selected_proxy:\n last_error = \"未找到可用的代理地址，无法继续执行任务。\"\n print(last_error)\n break\n\n state_path = selected_account.value if selected_account else STATE_FILE\n last_state_path = state_path\n proxy_server = selected_proxy.value if selected_proxy else None\n if rotation_settings[\"account_enabled\"]:\n print(f\"账号轮换：使用登录状态 {state_path}\")\n if rotation_settings[\"proxy_enabled\"] and proxy_server:\n print(f\"IP 轮换：使用代理 {proxy_server}\")\n\n try:\n processed_item_count += await _run_scrape_attempt(state_path, proxy_server)\n last_error = \"\"\n FAILURE_GUARD.record_success(task_name_for_guard)\n break\n except LoginRequiredError as e:\n last_error = str(e)\n print(f\"检测到登录失效/重定向: {e}\")\n break\n except RiskControlError as e:\n last_error = str(e)\n print(f\"检测到风控或验证触发: {e}\")\n # 风控验证通常不是简单轮换能解决的，避免无意义重试。\n break\n except Exception as e:\n last_error = f\"{type(e).__name__}: {e}\"\n print(f\"本次尝试失败: {last_error}\")\n if attempt < attempt_limit:\n print(\"将尝试轮换账号/IP 后重试...\")\n\n if last_error:\n await _notify_task_failure(task_config, last_error, cookie_path=last_state_path)\n\n # 清理任务图片目录\n cleanup_task_images(task_config.get(\"task_name\", \"default\"))\n\n return processed_item_count\n" }, { "path": "src/services/__init__.py", "content": "" }, { "path": "src/services/account_strategy_service.py", "content": "\"\"\"\n账号策略辅助函数\n\"\"\"\nfrom typing import Optional\n\n\nACCOUNT_STRATEGIES = {\"auto\", \"fixed\", \"rotate\"}\n\n\ndef clean_account_state_file(value: Optional[str]) -> Optional[str]:\n if value is None:\n return None\n text = str(value).strip()\n if not text or text in {\"null\", \"undefined\"}:\n return None\n return text\n\n\ndef normalize_account_strategy(\n strategy: Optional[str],\n account_state_file: Optional[str] = None,\n) -> str:\n raw = str(strategy or \"\").strip().lower()\n if raw in ACCOUNT_STRATEGIES:\n return raw\n if clean_account_state_file(account_state_file):\n return \"fixed\"\n return \"auto\"\n\n\ndef resolve_account_runtime_plan(\n *,\n strategy: Optional[str],\n account_state_file: Optional[str],\n has_root_state_file: bool,\n available_account_files: list[str],\n) -> dict:\n normalized_strategy = normalize_account_strategy(strategy, account_state_file)\n cleaned_account = clean_account_state_file(account_state_file)\n has_account_pool = len(available_account_files) > 0\n\n if normalized_strategy == \"fixed\":\n return {\n \"strategy\": normalized_strategy,\n \"forced_account\": cleaned_account,\n \"use_account_pool\": False,\n \"prefer_root_state\": False,\n }\n\n if normalized_strategy == \"rotate\":\n return {\n \"strategy\": normalized_strategy,\n \"forced_account\": None,\n \"use_account_pool\": has_account_pool,\n \"prefer_root_state\": False,\n }\n\n return {\n \"strategy\": normalized_strategy,\n \"forced_account\": None,\n \"use_account_pool\": (not has_root_state_file) and has_account_pool,\n \"prefer_root_state\": has_root_state_file,\n }\n" }, { "path": "src/services/ai_request_compat.py", "content": "\"\"\"AI 请求兼容性辅助逻辑。\"\"\"\n\nimport copy\nfrom typing import Any, Dict, Iterable, List\n\n\nRESPONSES_API_MODE = \"responses\"\nCHAT_COMPLETIONS_API_MODE = \"chat_completions\"\nINPUT_TEXT_TYPE = \"input_text\"\nINPUT_IMAGE_TYPE = \"input_image\"\nIMAGE_DETAIL_AUTO = \"auto\"\nJSON_OUTPUT_TYPE = \"json_object\"\nUNSUPPORTED_JSON_OUTPUT_MARKERS = (\n \"not supported by this model\",\n \"json_object\",\n \"json_schema\",\n \"text.format\",\n \"response_format.type\",\n)\nRESPONSES_API_UNSUPPORTED_MARKERS = (\n \"404 page not found\",\n \"page not found\",\n \"/responses\",\n \"/v1/responses\",\n)\nCHAT_COMPLETIONS_API_UNSUPPORTED_MARKERS = (\n \"404 page not found\",\n \"page not found\",\n \"/chat/completions\",\n \"/v1/chat/completions\",\n)\nUNSUPPORTED_TEMPERATURE_MARKERS = (\n \"temperature\",\n \"sampling temperature\",\n)\n\n\ndef build_responses_input(messages: Iterable[Dict[str, Any]]) -> List[Dict[str, Any]]:\n \"\"\"将 Chat Completions 风格的消息转换为 Responses API 输入。\"\"\"\n input_items: List[Dict[str, Any]] = []\n for message in messages:\n role = str(message.get(\"role\") or \"user\")\n input_items.append(\n {\n \"role\": role,\n \"content\": _build_input_content(message.get(\"content\")),\n }\n )\n return input_items\n\n\ndef add_json_text_format(\n request_params: Dict[str, Any],\n enabled: bool,\n) -> Dict[str, Any]:\n \"\"\"按需附加 Responses API 的结构化 JSON 输出参数。\"\"\"\n next_params = dict(request_params)\n if not enabled:\n return next_params\n\n text_config = dict(next_params.get(\"text\") or {})\n text_config[\"format\"] = {\"type\": JSON_OUTPUT_TYPE}\n next_params[\"text\"] = text_config\n return next_params\n\n\ndef add_json_response_format(\n request_params: Dict[str, Any],\n enabled: bool,\n) -> Dict[str, Any]:\n \"\"\"按需附加 Chat Completions 的 JSON 输出参数。\"\"\"\n next_params = dict(request_params)\n if enabled:\n next_params[\"response_format\"] = {\"type\": JSON_OUTPUT_TYPE}\n return next_params\n\n\ndef is_json_output_unsupported_error(error: Exception) -> bool:\n \"\"\"识别模型不支持结构化 JSON 输出参数的错误。\"\"\"\n message = str(error)\n return (\n \"not supported\" in message.lower()\n and any(marker in message for marker in UNSUPPORTED_JSON_OUTPUT_MARKERS)\n )\n\n\ndef is_responses_api_unsupported_error(error: Exception) -> bool:\n \"\"\"识别 OpenAI 兼容服务未实现 Responses API 的错误。\"\"\"\n return _is_api_unsupported_error(error, RESPONSES_API_UNSUPPORTED_MARKERS)\n\n\ndef is_chat_completions_api_unsupported_error(error: Exception) -> bool:\n \"\"\"识别 OpenAI 兼容服务未实现 Chat Completions API 的错误。\"\"\"\n return _is_api_unsupported_error(error, CHAT_COMPLETIONS_API_UNSUPPORTED_MARKERS)\n\n\ndef build_ai_request_params(\n api_mode: str,\n *,\n model: str,\n messages: Iterable[Dict[str, Any]],\n temperature: float | None = None,\n max_output_tokens: int | None = None,\n enable_json_output: bool = False,\n) -> Dict[str, Any]:\n \"\"\"根据 API 模式构建请求参数。\"\"\"\n request_params = {\"model\": model}\n if api_mode == RESPONSES_API_MODE:\n request_params[\"input\"] = build_responses_input(messages)\n if max_output_tokens is not None:\n request_params[\"max_output_tokens\"] = max_output_tokens\n if temperature is not None:\n request_params[\"temperature\"] = temperature\n return add_json_text_format(request_params, enable_json_output)\n\n if api_mode == CHAT_COMPLETIONS_API_MODE:\n request_params[\"messages\"] = copy.deepcopy(list(messages))\n if max_output_tokens is not None:\n request_params[\"max_tokens\"] = max_output_tokens\n if temperature is not None:\n request_params[\"temperature\"] = temperature\n return add_json_response_format(request_params, enable_json_output)\n\n raise ValueError(f\"不支持的 AI API 模式: {api_mode}\")\n\n\nasync def create_ai_response_async(\n client: Any,\n api_mode: str,\n request_params: Dict[str, Any],\n) -> Any:\n \"\"\"根据 API 模式发起异步请求。\"\"\"\n if api_mode == RESPONSES_API_MODE:\n return await client.responses.create(**request_params)\n if api_mode == CHAT_COMPLETIONS_API_MODE:\n return await client.chat.completions.create(**request_params)\n raise ValueError(f\"不支持的 AI API 模式: {api_mode}\")\n\n\ndef create_ai_response_sync(\n client: Any,\n api_mode: str,\n request_params: Dict[str, Any],\n) -> Any:\n \"\"\"根据 API 模式发起同步请求。\"\"\"\n if api_mode == RESPONSES_API_MODE:\n return client.responses.create(**request_params)\n if api_mode == CHAT_COMPLETIONS_API_MODE:\n return client.chat.completions.create(**request_params)\n raise ValueError(f\"不支持的 AI API 模式: {api_mode}\")\n\n\ndef is_temperature_unsupported_error(error: Exception) -> bool:\n \"\"\"识别模型或中转站不支持 temperature 参数的错误。\"\"\"\n message = str(error).lower()\n return (\n \"not supported\" in message\n or \"unsupported\" in message\n or \"invalid\" in message\n or \"参数错误\" in message\n ) and any(marker in message for marker in UNSUPPORTED_TEMPERATURE_MARKERS)\n\n\ndef remove_temperature_param(request_params: Dict[str, Any]) -> Dict[str, Any]:\n \"\"\"移除 temperature 参数，适配不支持采样温度的模型网关。\"\"\"\n next_params = dict(request_params)\n next_params.pop(\"temperature\", None)\n return next_params\n\n\ndef _is_api_unsupported_error(\n error: Exception,\n markers: tuple[str, ...],\n) -> bool:\n message = str(error).lower()\n if any(marker in message for marker in markers):\n return True\n\n status_code = getattr(error, \"status_code\", None)\n body = getattr(error, \"body\", None)\n response = getattr(error, \"response\", None)\n response_text = getattr(response, \"text\", None) if response else None\n return (\n status_code == 404\n and message.strip() == \"error code: 404\"\n and not body\n and not response_text\n )\n\n\ndef _build_input_content(content: Any) -> List[Dict[str, Any]]:\n if isinstance(content, str):\n return [{\"type\": INPUT_TEXT_TYPE, \"text\": content}]\n if not isinstance(content, list):\n raise ValueError(f\"AI消息内容类型不受支持: {type(content).__name__}\")\n\n return [_coerce_content_item(item) for item in content]\n\n\ndef _coerce_content_item(item: Any) -> Dict[str, Any]:\n if not isinstance(item, dict):\n raise ValueError(f\"AI消息片段类型不受支持: {type(item).__name__}\")\n\n item_type = item.get(\"type\")\n if item_type in {\"text\", INPUT_TEXT_TYPE}:\n text = item.get(\"text\")\n if not isinstance(text, str):\n raise ValueError(\"文本消息片段缺少 text 字段。\")\n return {\"type\": INPUT_TEXT_TYPE, \"text\": text}\n\n if item_type in {\"image_url\", INPUT_IMAGE_TYPE}:\n return _build_image_input_item(item)\n\n raise ValueError(f\"不支持的 AI 消息片段类型: {item_type}\")\n\n\ndef _build_image_input_item(item: Dict[str, Any]) -> Dict[str, Any]:\n raw_image = item.get(\"image_url\")\n if isinstance(raw_image, dict):\n image_url = raw_image.get(\"url\")\n else:\n image_url = raw_image\n\n if not isinstance(image_url, str) or not image_url.strip():\n raise ValueError(\"图片消息片段缺少有效的 image_url。\")\n\n return {\n \"type\": INPUT_IMAGE_TYPE,\n \"image_url\": image_url,\n \"detail\": item.get(\"detail\", IMAGE_DETAIL_AUTO),\n }\n" }, { "path": "src/services/ai_response_parser.py", "content": "\"\"\"\nAI 响应解析工具\n\"\"\"\nimport json\nfrom typing import Any\n\n\nclass EmptyAIResponseError(ValueError):\n \"\"\"AI 返回了空内容。\"\"\"\n\n\ndef extract_ai_response_content(response: Any) -> str:\n \"\"\"从不同形态的 AI 响应中提取文本内容。\"\"\"\n if response is None:\n raise EmptyAIResponseError(\"AI响应对象为空。\")\n\n if isinstance(response, (bytes, bytearray)):\n text = response.decode(\"utf-8\", errors=\"replace\")\n return _normalize_text_content(text)\n\n if isinstance(response, str):\n return _normalize_text_content(response)\n\n output_text = getattr(response, \"output_text\", None)\n if isinstance(output_text, str):\n return _normalize_text_content(output_text)\n\n choices = getattr(response, \"choices\", None)\n if choices:\n message = getattr(choices[0], \"message\", None)\n if message is None:\n raise EmptyAIResponseError(\"AI响应缺少 message。\")\n content = getattr(message, \"content\", None)\n return _normalize_text_content(_coerce_content_parts(content))\n\n raise ValueError(f\"无法识别的AI响应类型: {type(response).__name__}\")\n\n\ndef parse_ai_response_json(content: str) -> dict:\n \"\"\"解析 AI 文本响应中的 JSON。\"\"\"\n cleaned = _strip_code_fences(content)\n try:\n return json.loads(cleaned)\n except json.JSONDecodeError as exc:\n return _extract_first_json_value(cleaned, exc)\n\n\ndef _coerce_content_parts(content: Any) -> str:\n if content is None:\n return \"\"\n if isinstance(content, str):\n return content\n if isinstance(content, (bytes, bytearray)):\n return content.decode(\"utf-8\", errors=\"replace\")\n if not isinstance(content, list):\n raise ValueError(f\"AI响应内容类型不受支持: {type(content).__name__}\")\n\n parts: list[str] = []\n for item in content:\n if isinstance(item, str):\n parts.append(item)\n continue\n if isinstance(item, dict):\n text = item.get(\"text\")\n if isinstance(text, str):\n parts.append(text)\n continue\n text = getattr(item, \"text\", None)\n if isinstance(text, str):\n parts.append(text)\n return \"\".join(parts)\n\n\ndef _normalize_text_content(content: str) -> str:\n text = str(content).strip()\n if not text:\n raise EmptyAIResponseError(\"AI响应内容为空。\")\n return text\n\n\ndef _strip_code_fences(content: str) -> str:\n cleaned = content.strip()\n if cleaned.startswith(\"```json\"):\n cleaned = cleaned[7:]\n if cleaned.startswith(\"```\"):\n cleaned = cleaned[3:]\n if cleaned.endswith(\"```\"):\n cleaned = cleaned[:-3]\n return cleaned.strip()\n\n\ndef _extract_first_json_value(\n content: str,\n fallback_error: json.JSONDecodeError,\n):\n decoder = json.JSONDecoder()\n last_error: json.JSONDecodeError | None = None\n\n for start_index, char in enumerate(content):\n if char not in \"{[\":\n continue\n try:\n parsed, _ = decoder.raw_decode(content[start_index:])\n return parsed\n except json.JSONDecodeError as exc:\n last_error = exc\n\n if last_error is not None:\n raise last_error\n raise fallback_error\n" }, { "path": "src/services/ai_service.py", "content": "\"\"\"\nAI 分析服务\n封装 AI 分析相关的业务逻辑\n\"\"\"\nfrom typing import Dict, List, Optional\nfrom src.infrastructure.external.ai_client import AIClient\n\n\nclass AIAnalysisService:\n \"\"\"AI 分析服务\"\"\"\n\n def __init__(self, ai_client: AIClient):\n self.ai_client = ai_client\n\n async def analyze_product(\n self,\n product_data: Dict,\n image_paths: List[str],\n prompt_text: str\n ) -> Optional[Dict]:\n \"\"\"\n 分析商品\n\n Args:\n product_data: 商品数据\n image_paths: 图片路径列表\n prompt_text: 分析提示词\n\n Returns:\n 分析结果\n \"\"\"\n if not self.ai_client.is_available():\n print(\"AI 客户端不可用，跳过分析\")\n return None\n\n try:\n result = await self.ai_client.analyze(product_data, image_paths, prompt_text)\n\n if result and self._validate_result(result):\n return result\n else:\n print(\"AI 分析结果验证失败\")\n return None\n except Exception as e:\n print(f\"AI 分析服务出错: {e}\")\n return None\n\n def _validate_result(self, result: Dict) -> bool:\n \"\"\"验证 AI 分析结果的格式\"\"\"\n required_fields = [\n \"prompt_version\",\n \"is_recommended\",\n \"reason\",\n \"risk_tags\",\n \"criteria_analysis\"\n ]\n\n # 检查必需字段\n for field in required_fields:\n if field not in result:\n print(f\"AI 响应缺少必需字段: {field}\")\n return False\n\n # 检查数据类型\n if not isinstance(result.get(\"is_recommended\"), bool):\n print(\"is_recommended 字段不是布尔类型\")\n return False\n\n if not isinstance(result.get(\"risk_tags\"), list):\n print(\"risk_tags 字段不是列表类型\")\n return False\n\n criteria_analysis = result.get(\"criteria_analysis\", {})\n if not isinstance(criteria_analysis, dict) or not criteria_analysis:\n print(\"criteria_analysis 必须是非空字典\")\n return False\n\n return True\n" }, { "path": "src/services/dashboard_payloads.py", "content": "\"\"\"\nDashboard 数据拼装辅助函数\n\"\"\"\nfrom __future__ import annotations\n\nfrom datetime import datetime\nfrom typing import Any\n\nfrom src.domain.models.task import Task\nfrom src.services.price_history_service import parse_price_value\nfrom src.services.result_file_service import (\n normalize_keyword_from_filename,\n)\nfrom src.services.result_storage_service import load_result_summary\n\n\ndef normalize_text(value: str | None) -> str:\n return (value or \"\").strip().lower()\n\n\ndef parse_timestamp(value: str | None) -> datetime | None:\n if not value:\n return None\n normalized = value.strip()\n for candidate in (normalized, normalized.replace(\"Z\", \"+00:00\"), normalized.replace(\" \", \"T\")):\n try:\n return datetime.fromisoformat(candidate)\n except ValueError:\n continue\n return None\n\n\ndef serialize_timestamp(value: datetime | None) -> str | None:\n return value.isoformat() if value else None\n\n\ndef build_empty_summary(task: Task) -> dict[str, Any]:\n return {\n \"task_id\": task.id,\n \"task_name\": task.task_name,\n \"keyword\": task.keyword,\n \"filename\": None,\n \"enabled\": task.enabled,\n \"is_running\": task.is_running,\n \"account_strategy\": task.account_strategy,\n \"cron\": task.cron,\n \"region\": task.region,\n \"total_items\": 0,\n \"recommended_items\": 0,\n \"ai_recommended_items\": 0,\n \"keyword_recommended_items\": 0,\n \"latest_crawl_time\": None,\n \"latest_recommended_title\": None,\n \"latest_recommended_price\": None,\n }\n\n\ndef build_activity(\n *,\n activity_id: str,\n activity_type: str,\n task_name: str,\n keyword: str,\n title: str,\n status: str,\n timestamp: datetime | None,\n detail: str | None = None,\n filename: str | None = None,\n) -> dict[str, Any]:\n return {\n \"id\": activity_id,\n \"type\": activity_type,\n \"task_name\": task_name,\n \"keyword\": keyword,\n \"title\": title,\n \"status\": status,\n \"detail\": detail,\n \"filename\": filename,\n \"timestamp\": serialize_timestamp(timestamp),\n }\n\n\ndef sort_key_by_latest_time(item: dict[str, Any]) -> tuple[float, str]:\n timestamp = parse_timestamp(item.get(\"latest_crawl_time\"))\n return (timestamp.timestamp() if timestamp else 0.0, item.get(\"task_name\", \"\"))\n\n\ndef sort_key_by_activity_time(item: dict[str, Any]) -> tuple[float, str]:\n timestamp = parse_timestamp(item.get(\"timestamp\"))\n return (timestamp.timestamp() if timestamp else 0.0, item.get(\"id\", \"\"))\n\n\ndef _build_fallback_summary(task_name: str, keyword: str) -> dict[str, Any]:\n return {\n \"task_id\": None,\n \"task_name\": task_name,\n \"keyword\": keyword,\n \"filename\": None,\n \"enabled\": False,\n \"is_running\": False,\n \"account_strategy\": \"auto\",\n \"cron\": None,\n \"region\": None,\n \"total_items\": 0,\n \"recommended_items\": 0,\n \"ai_recommended_items\": 0,\n \"keyword_recommended_items\": 0,\n \"latest_crawl_time\": None,\n \"latest_recommended_title\": None,\n \"latest_recommended_price\": None,\n }\n\n\ndef _resolve_task(\n task_lookup: dict[str, Task],\n latest_record: dict[str, Any] | None,\n keyword: str,\n) -> Task | None:\n task = task_lookup.get(normalize_text(keyword))\n if task is not None or latest_record is None:\n return task\n fallback_name = str(latest_record.get(\"任务名称\") or \"\")\n return next(\n (candidate for candidate in task_lookup.values() if candidate.task_name == fallback_name),\n None,\n )\n\n\ndef _collect_record_metrics(records: list[dict[str, Any]]) -> dict[str, Any]:\n latest_crawl_time: datetime | None = None\n latest_record: dict[str, Any] | None = None\n latest_recommendation: dict[str, Any] | None = None\n recommended_items = 0\n ai_recommended_items = 0\n keyword_recommended_items = 0\n\n for record in records:\n crawl_time = parse_timestamp(record.get(\"爬取时间\"))\n if crawl_time and (latest_crawl_time is None or crawl_time > latest_crawl_time):\n latest_crawl_time = crawl_time\n latest_record = record\n\n analysis = record.get(\"ai_analysis\", {}) or {}\n if analysis.get(\"is_recommended\") is not True:\n continue\n\n recommended_items += 1\n source = analysis.get(\"analysis_source\")\n if source == \"ai\":\n ai_recommended_items += 1\n elif source == \"keyword\":\n keyword_recommended_items += 1\n\n recommendation_time = parse_timestamp(\n latest_recommendation.get(\"爬取时间\") if latest_recommendation else None\n )\n if latest_recommendation is None or (crawl_time and recommendation_time and crawl_time > recommendation_time):\n latest_recommendation = record\n elif latest_recommendation is None and crawl_time:\n latest_recommendation = record\n\n return {\n \"latest_crawl_time\": latest_crawl_time,\n \"latest_record\": latest_record,\n \"latest_recommendation\": latest_recommendation,\n \"recommended_items\": recommended_items,\n \"ai_recommended_items\": ai_recommended_items,\n \"keyword_recommended_items\": keyword_recommended_items,\n }\n\n\ndef _build_recommendation_activity(\n *,\n filename: str,\n task_name: str,\n keyword: str,\n latest_recommendation: dict[str, Any] | None,\n) -> tuple[dict[str, Any] | None, str | None, float | None]:\n if not latest_recommendation:\n return None, None, None\n\n product = latest_recommendation.get(\"商品信息\", {}) or {}\n analysis = latest_recommendation.get(\"ai_analysis\", {}) or {}\n title = str(product.get(\"商品标题\") or \"发现推荐商品\")\n price = parse_price_value(product.get(\"当前售价\"))\n status = \"AI 推荐\" if analysis.get(\"analysis_source\") == \"ai\" else \"关键词命中\"\n detail = f\"当前价 ¥{price:.0f}\" if isinstance(price, (int, float)) else None\n activity = build_activity(\n activity_id=f\"{filename}:recommended\",\n activity_type=\"recommendation\",\n task_name=task_name,\n keyword=keyword,\n title=title,\n status=status,\n timestamp=parse_timestamp(latest_recommendation.get(\"爬取时间\")),\n detail=detail,\n filename=filename,\n )\n return activity, title, price\n\n\ndef _build_scan_activity(\n *,\n filename: str,\n task_name: str,\n keyword: str,\n latest_record: dict[str, Any] | None,\n total_items: int,\n) -> dict[str, Any] | None:\n if not latest_record:\n return None\n product = latest_record.get(\"商品信息\", {}) or {}\n title = str(product.get(\"商品标题\") or task_name)\n return build_activity(\n activity_id=f\"{filename}:scan\",\n activity_type=\"scan\",\n task_name=task_name,\n keyword=keyword,\n title=title,\n status=\"结果已更新\",\n timestamp=parse_timestamp(latest_record.get(\"爬取时间\")),\n detail=f\"已累计 {total_items} 条样本\",\n filename=filename,\n )\n\n\nasync def summarize_result_file(\n filename: str,\n task_lookup: dict[str, Task],\n) -> tuple[dict[str, Any] | None, list[dict[str, Any]], datetime | None]:\n metrics = await load_result_summary(filename)\n if not metrics:\n return None, [], None\n\n latest_record = metrics[\"latest_record\"]\n latest_crawl_time = parse_timestamp(metrics[\"latest_crawl_time\"])\n keyword = str((latest_record or {}).get(\"搜索关键字\") or \"\") or normalize_keyword_from_filename(filename)\n task = _resolve_task(task_lookup, latest_record, keyword)\n task_name = task.task_name if task else str((latest_record or {}).get(\"任务名称\") or keyword)\n summary = build_empty_summary(task) if task else _build_fallback_summary(task_name, keyword)\n\n activities: list[dict[str, Any]] = []\n recommendation, title, price = _build_recommendation_activity(\n filename=filename,\n task_name=task_name,\n keyword=keyword,\n latest_recommendation=metrics[\"latest_recommendation\"],\n )\n if recommendation:\n activities.append(recommendation)\n\n scan_activity = _build_scan_activity(\n filename=filename,\n task_name=task_name,\n keyword=keyword,\n latest_record=latest_record,\n total_items=metrics[\"total_items\"],\n )\n if scan_activity:\n activities.append(scan_activity)\n\n summary.update(\n {\n \"filename\": filename,\n \"total_items\": metrics[\"total_items\"],\n \"recommended_items\": metrics[\"recommended_items\"],\n \"ai_recommended_items\": metrics[\"ai_recommended_items\"],\n \"keyword_recommended_items\": metrics[\"keyword_recommended_items\"],\n \"latest_crawl_time\": serialize_timestamp(latest_crawl_time),\n \"latest_recommended_title\": title,\n \"latest_recommended_price\": price,\n }\n )\n return summary, activities, latest_crawl_time\n\n\ndef build_task_state_activities(tasks: list[Task]) -> list[dict[str, Any]]:\n activities: list[dict[str, Any]] = []\n for task in tasks:\n status = \"运行中\" if task.is_running else \"已启用\"\n detail = \"任务正在轮询闲鱼结果\" if task.is_running else \"等待下一次调度执行\"\n if not task.is_running and not task.enabled:\n continue\n activities.append(\n build_activity(\n activity_id=f\"task:{task.id}:{'running' if task.is_running else 'ready'}\",\n activity_type=\"task\",\n task_name=task.task_name,\n keyword=task.keyword,\n title=task.task_name,\n status=status,\n timestamp=None,\n detail=detail,\n )\n )\n return activities\n" }, { "path": "src/services/dashboard_service.py", "content": "\"\"\"\nDashboard 聚合服务\n统一汇总任务、结果文件和最近活动，供首页概览使用。\n\"\"\"\nfrom __future__ import annotations\n\nfrom typing import Any\n\nfrom src.domain.models.task import Task\nfrom src.services.dashboard_payloads import (\n build_empty_summary,\n build_task_state_activities,\n normalize_text,\n serialize_timestamp,\n sort_key_by_activity_time,\n sort_key_by_latest_time,\n summarize_result_file,\n)\nfrom src.services.result_storage_service import list_result_filenames\n\nMAX_RECENT_ACTIVITIES = 8\n\n\ndef _build_summary_metrics(tasks: list[Task], summary_list: list[dict[str, Any]], last_updated_at: Any) -> dict[str, Any]:\n return {\n \"enabled_tasks\": sum(1 for task in tasks if task.enabled),\n \"running_tasks\": sum(1 for task in tasks if task.is_running),\n \"result_files\": sum(1 for item in summary_list if item.get(\"filename\")),\n \"scanned_items\": sum(int(item[\"total_items\"]) for item in summary_list),\n \"recommended_items\": sum(int(item[\"recommended_items\"]) for item in summary_list),\n \"ai_recommended_items\": sum(int(item[\"ai_recommended_items\"]) for item in summary_list),\n \"keyword_recommended_items\": sum(int(item[\"keyword_recommended_items\"]) for item in summary_list),\n \"last_updated_at\": serialize_timestamp(last_updated_at),\n }\n\n\nasync def build_dashboard_snapshot(tasks: list[Task]) -> dict[str, Any]:\n task_lookup = {normalize_text(task.keyword): task for task in tasks}\n task_summaries: dict[str, dict[str, Any]] = {\n task.task_name: build_empty_summary(task) for task in tasks\n }\n recent_activities = build_task_state_activities(tasks)\n latest_updated_at = None\n\n for filename in await list_result_filenames():\n summary, activities, file_latest_time = await summarize_result_file(filename, task_lookup)\n if summary:\n task_summaries[summary[\"task_name\"]] = summary\n recent_activities.extend(activities)\n if file_latest_time and (latest_updated_at is None or file_latest_time > latest_updated_at):\n latest_updated_at = file_latest_time\n\n summary_list = sorted(task_summaries.values(), key=sort_key_by_latest_time, reverse=True)\n focus_file = next((item[\"filename\"] for item in summary_list if item.get(\"filename\")), None)\n return {\n \"summary\": _build_summary_metrics(tasks, summary_list, latest_updated_at),\n \"task_summaries\": summary_list,\n \"recent_activities\": sorted(\n recent_activities,\n key=sort_key_by_activity_time,\n reverse=True,\n )[:MAX_RECENT_ACTIVITIES],\n \"focus_file\": focus_file,\n }\n" }, { "path": "src/services/item_analysis_dispatcher.py", "content": "\"\"\"\n商品分析分发器\n将卖家资料采集、图片下载、AI 分析和结果保存移出主抓取链路。\n\"\"\"\nimport asyncio\nimport copy\nimport os\nfrom dataclasses import dataclass\nfrom typing import Awaitable, Callable, Optional\n\nfrom src.keyword_rule_engine import build_search_text, evaluate_keyword_rules\n\n\nSellerLoader = Callable[[str], Awaitable[dict]]\nImageDownloader = Callable[[str, list[str], str], Awaitable[list[str]]]\nAIAnalyzer = Callable[[dict, list[str], str], Awaitable[Optional[dict]]]\nNotifier = Callable[[dict, str], Awaitable[None]]\nSaver = Callable[[dict, str], Awaitable[bool]]\n\n\n@dataclass(frozen=True)\nclass ItemAnalysisJob:\n keyword: str\n task_name: str\n decision_mode: str\n analyze_images: bool\n prompt_text: str\n keyword_rules: tuple[str, ...]\n final_record: dict\n seller_id: Optional[str]\n zhima_credit_text: Optional[str]\n registration_duration_text: str\n\n\nclass ItemAnalysisDispatcher:\n \"\"\"用受控并发处理商品分析和落盘。\"\"\"\n\n def __init__(\n self,\n *,\n concurrency: int,\n skip_ai_analysis: bool,\n seller_loader: SellerLoader,\n image_downloader: ImageDownloader,\n ai_analyzer: AIAnalyzer,\n notifier: Notifier,\n saver: Saver,\n ) -> None:\n self._semaphore = asyncio.Semaphore(max(1, concurrency))\n self._skip_ai_analysis = skip_ai_analysis\n self._seller_loader = seller_loader\n self._image_downloader = image_downloader\n self._ai_analyzer = ai_analyzer\n self._notifier = notifier\n self._saver = saver\n self._tasks: set[asyncio.Task] = set()\n self.completed_count = 0\n\n def submit(self, job: ItemAnalysisJob) -> None:\n task = asyncio.create_task(self._process_with_limit(job))\n self._tasks.add(task)\n task.add_done_callback(self._tasks.discard)\n\n async def join(self) -> None:\n while self._tasks:\n await asyncio.gather(*tuple(self._tasks))\n\n async def _process_with_limit(self, job: ItemAnalysisJob) -> None:\n async with self._semaphore:\n await self._process_job(job)\n\n async def _process_job(self, job: ItemAnalysisJob) -> None:\n record = copy.deepcopy(job.final_record)\n item_data = record.get(\"商品信息\", {}) or {}\n record[\"卖家信息\"] = await self._load_seller_info(job)\n record[\"ai_analysis\"] = await self._build_analysis_result(job, record)\n if await self._saver(record, job.keyword):\n self.completed_count += 1\n await self._notify_if_recommended(item_data, record[\"ai_analysis\"])\n\n async def _load_seller_info(self, job: ItemAnalysisJob) -> dict:\n seller_info = {}\n if job.seller_id:\n try:\n seller_info = await self._seller_loader(job.seller_id)\n except Exception as exc:\n print(f\" [卖家] 采集卖家 {job.seller_id} 信息失败: {exc}\")\n merged = copy.deepcopy(seller_info or {})\n merged[\"卖家芝麻信用\"] = job.zhima_credit_text\n merged[\"卖家注册时长\"] = job.registration_duration_text\n return merged\n\n async def _build_analysis_result(self, job: ItemAnalysisJob, record: dict) -> dict:\n if job.decision_mode == \"keyword\":\n return self._build_keyword_result(job, record)\n if self._skip_ai_analysis:\n return self._build_skip_ai_result()\n return await self._run_ai_analysis(job, record)\n\n def _build_keyword_result(self, job: ItemAnalysisJob, record: dict) -> dict:\n search_text = build_search_text(record)\n return evaluate_keyword_rules(list(job.keyword_rules), search_text)\n\n def _build_skip_ai_result(self) -> dict:\n return {\n \"analysis_source\": \"ai\",\n \"is_recommended\": True,\n \"reason\": \"商品已跳过AI分析，直接通知\",\n \"keyword_hit_count\": 0,\n }\n\n def _build_ai_error_result(self, reason: str, *, error: str = \"\") -> dict:\n payload = {\n \"analysis_source\": \"ai\",\n \"is_recommended\": False,\n \"reason\": reason,\n \"keyword_hit_count\": 0,\n }\n if error:\n payload[\"error\"] = error\n return payload\n\n async def _run_ai_analysis(self, job: ItemAnalysisJob, record: dict) -> dict:\n image_paths: list[str] = []\n try:\n image_paths = await self._download_images(job, record)\n if not job.prompt_text:\n return self._build_ai_error_result(\"任务未配置AI prompt，跳过分析。\")\n ai_result = await self._ai_analyzer(record, image_paths, job.prompt_text)\n if not ai_result:\n return self._build_ai_error_result(\n \"AI analysis returned None after retries.\",\n error=\"AI analysis returned None after retries.\",\n )\n ai_result.setdefault(\"analysis_source\", \"ai\")\n ai_result.setdefault(\"keyword_hit_count\", 0)\n return ai_result\n except Exception as exc:\n return self._build_ai_error_result(\n f\"AI分析异常: {exc}\",\n error=str(exc),\n )\n finally:\n self._cleanup_images(image_paths)\n\n async def _download_images(self, job: ItemAnalysisJob, record: dict) -> list[str]:\n if not job.analyze_images:\n return []\n item_data = record.get(\"商品信息\", {}) or {}\n image_urls = item_data.get(\"商品图片列表\", [])\n if not image_urls:\n return []\n return await self._image_downloader(\n item_data[\"商品ID\"],\n image_urls,\n job.task_name,\n )\n\n def _cleanup_images(self, image_paths: list[str]) -> None:\n for img_path in image_paths:\n try:\n if os.path.exists(img_path):\n os.remove(img_path)\n except Exception as exc:\n print(f\" [图片] 删除图片文件时出错: {exc}\")\n\n async def _notify_if_recommended(self, item_data: dict, analysis_result: dict) -> None:\n if not analysis_result.get(\"is_recommended\"):\n return\n try:\n await self._notifier(item_data, analysis_result.get(\"reason\", \"无\"))\n except Exception as exc:\n print(f\" [通知] 发送推荐通知失败: {exc}\")\n" }, { "path": "src/services/notification_config_service.py", "content": "\"\"\"\n通知配置读写与校验服务\n\"\"\"\nimport json\nfrom urllib.parse import urlparse\n\nfrom src.infrastructure.config.env_manager import env_manager\nfrom src.infrastructure.config.settings import (\n DEFAULT_TELEGRAM_API_BASE_URL,\n NotificationSettings,\n)\n\n\nNOTIFICATION_FIELD_MAP = {\n \"NTFY_TOPIC_URL\": \"ntfy_topic_url\",\n \"GOTIFY_URL\": \"gotify_url\",\n \"GOTIFY_TOKEN\": \"gotify_token\",\n \"BARK_URL\": \"bark_url\",\n \"WX_BOT_URL\": \"wx_bot_url\",\n \"TELEGRAM_BOT_TOKEN\": \"telegram_bot_token\",\n \"TELEGRAM_CHAT_ID\": \"telegram_chat_id\",\n \"TELEGRAM_API_BASE_URL\": \"telegram_api_base_url\",\n \"WEBHOOK_URL\": \"webhook_url\",\n \"WEBHOOK_METHOD\": \"webhook_method\",\n \"WEBHOOK_HEADERS\": \"webhook_headers\",\n \"WEBHOOK_CONTENT_TYPE\": \"webhook_content_type\",\n \"WEBHOOK_QUERY_PARAMETERS\": \"webhook_query_parameters\",\n \"WEBHOOK_BODY\": \"webhook_body\",\n \"PCURL_TO_MOBILE\": \"pcurl_to_mobile\",\n}\n\nSECRET_NOTIFICATION_FIELDS = {\n \"BARK_URL\",\n \"GOTIFY_TOKEN\",\n \"WX_BOT_URL\",\n \"TELEGRAM_BOT_TOKEN\",\n \"WEBHOOK_URL\",\n \"WEBHOOK_HEADERS\",\n}\n\nJSON_NOTIFICATION_FIELDS = {\n \"WEBHOOK_HEADERS\": True,\n \"WEBHOOK_QUERY_PARAMETERS\": True,\n \"WEBHOOK_BODY\": False,\n}\n\nURL_FIELDS = {\n \"NTFY_TOPIC_URL\",\n \"GOTIFY_URL\",\n \"BARK_URL\",\n \"WX_BOT_URL\",\n \"TELEGRAM_API_BASE_URL\",\n \"WEBHOOK_URL\",\n}\n\nALLOWED_WEBHOOK_METHODS = {\"GET\", \"POST\"}\nALLOWED_WEBHOOK_CONTENT_TYPES = {\"JSON\", \"FORM\"}\n\n\nclass NotificationSettingsValidationError(ValueError):\n \"\"\"通知配置校验错误\"\"\"\n\n\ndef model_dump(model, *, exclude_unset: bool = False) -> dict:\n if hasattr(model, \"model_dump\"):\n return model.model_dump(exclude_unset=exclude_unset)\n return model.dict(exclude_unset=exclude_unset)\n\n\ndef build_notification_settings_response(\n settings: NotificationSettings | None = None,\n) -> dict:\n notification_settings = settings or load_notification_settings()\n response = {\n \"NTFY_TOPIC_URL\": notification_settings.ntfy_topic_url or \"\",\n \"GOTIFY_URL\": notification_settings.gotify_url or \"\",\n \"GOTIFY_TOKEN\": \"\",\n \"BARK_URL\": \"\",\n \"WX_BOT_URL\": \"\",\n \"TELEGRAM_BOT_TOKEN\": \"\",\n \"TELEGRAM_CHAT_ID\": notification_settings.telegram_chat_id or \"\",\n \"TELEGRAM_API_BASE_URL\": (\n notification_settings.telegram_api_base_url\n or DEFAULT_TELEGRAM_API_BASE_URL\n ),\n \"WEBHOOK_URL\": \"\",\n \"WEBHOOK_METHOD\": notification_settings.webhook_method,\n \"WEBHOOK_HEADERS\": \"\",\n \"WEBHOOK_CONTENT_TYPE\": notification_settings.webhook_content_type,\n \"WEBHOOK_QUERY_PARAMETERS\": notification_settings.webhook_query_parameters or \"\",\n \"WEBHOOK_BODY\": notification_settings.webhook_body or \"\",\n \"PCURL_TO_MOBILE\": notification_settings.pcurl_to_mobile,\n }\n for field in SECRET_NOTIFICATION_FIELDS:\n attr_name = NOTIFICATION_FIELD_MAP[field]\n response[f\"{field}_SET\"] = bool(getattr(notification_settings, attr_name))\n response[\"CONFIGURED_CHANNELS\"] = build_configured_channels(notification_settings)\n return response\n\n\ndef build_notification_status_flags(\n settings: NotificationSettings | None = None,\n) -> dict:\n notification_settings = settings or load_notification_settings()\n return {\n \"ntfy_topic_url_set\": bool(notification_settings.ntfy_topic_url),\n \"gotify_url_set\": bool(notification_settings.gotify_url),\n \"gotify_token_set\": bool(notification_settings.gotify_token),\n \"bark_url_set\": bool(notification_settings.bark_url),\n \"wx_bot_url_set\": bool(notification_settings.wx_bot_url),\n \"telegram_bot_token_set\": bool(notification_settings.telegram_bot_token),\n \"telegram_chat_id_set\": bool(notification_settings.telegram_chat_id),\n \"webhook_url_set\": bool(notification_settings.webhook_url),\n \"webhook_headers_set\": bool(notification_settings.webhook_headers),\n }\n\n\ndef build_configured_channels(\n settings: NotificationSettings | None = None,\n) -> list[str]:\n notification_settings = settings or load_notification_settings()\n channels = []\n if notification_settings.ntfy_topic_url:\n channels.append(\"ntfy\")\n if notification_settings.bark_url:\n channels.append(\"bark\")\n if notification_settings.gotify_url and notification_settings.gotify_token:\n channels.append(\"gotify\")\n if notification_settings.wx_bot_url:\n channels.append(\"wecom\")\n if notification_settings.telegram_bot_token and notification_settings.telegram_chat_id:\n channels.append(\"telegram\")\n if notification_settings.webhook_url:\n channels.append(\"webhook\")\n return channels\n\n\ndef prepare_notification_settings_update(\n patch_payload: dict,\n existing_settings: NotificationSettings | None = None,\n) -> tuple[dict[str, str], list[str], NotificationSettings]:\n current_settings = existing_settings or load_notification_settings()\n merged_values = _notification_settings_to_values(current_settings)\n\n for env_name, raw_value in patch_payload.items():\n attr_name = NOTIFICATION_FIELD_MAP.get(env_name)\n if attr_name is None:\n continue\n merged_values[attr_name] = _normalize_patch_value(env_name, raw_value)\n\n normalized_values = _normalize_notification_values(merged_values)\n candidate_settings = _build_notification_settings_model(normalized_values)\n _validate_notification_settings(candidate_settings)\n\n updates = {}\n deletions = []\n for env_name, raw_value in patch_payload.items():\n attr_name = NOTIFICATION_FIELD_MAP.get(env_name)\n if attr_name is None:\n continue\n value = normalized_values[attr_name]\n if isinstance(value, bool):\n updates[env_name] = \"true\" if value else \"false\"\n continue\n if value is None:\n deletions.append(env_name)\n continue\n updates[env_name] = value\n return updates, deletions, candidate_settings\n\n\ndef _notification_settings_to_values(settings: NotificationSettings) -> dict:\n return {\n attr_name: getattr(settings, attr_name)\n for attr_name in NOTIFICATION_FIELD_MAP.values()\n }\n\n\ndef load_notification_settings() -> NotificationSettings:\n return _build_notification_settings_model(\n {\n \"ntfy_topic_url\": _normalize_existing_text(env_manager.get_value(\"NTFY_TOPIC_URL\")),\n \"gotify_url\": _normalize_existing_text(env_manager.get_value(\"GOTIFY_URL\")),\n \"gotify_token\": _normalize_existing_text(env_manager.get_value(\"GOTIFY_TOKEN\")),\n \"bark_url\": _normalize_existing_text(env_manager.get_value(\"BARK_URL\")),\n \"wx_bot_url\": _normalize_existing_text(env_manager.get_value(\"WX_BOT_URL\")),\n \"telegram_bot_token\": _normalize_existing_text(env_manager.get_value(\"TELEGRAM_BOT_TOKEN\")),\n \"telegram_chat_id\": _normalize_existing_text(env_manager.get_value(\"TELEGRAM_CHAT_ID\")),\n \"telegram_api_base_url\": (\n _normalize_existing_text(env_manager.get_value(\"TELEGRAM_API_BASE_URL\"))\n or DEFAULT_TELEGRAM_API_BASE_URL\n ),\n \"webhook_url\": _normalize_existing_text(env_manager.get_value(\"WEBHOOK_URL\")),\n \"webhook_method\": _normalize_existing_text(env_manager.get_value(\"WEBHOOK_METHOD\")) or \"POST\",\n \"webhook_headers\": _normalize_existing_text(env_manager.get_value(\"WEBHOOK_HEADERS\")),\n \"webhook_content_type\": _normalize_existing_text(env_manager.get_value(\"WEBHOOK_CONTENT_TYPE\")) or \"JSON\",\n \"webhook_query_parameters\": _normalize_existing_text(env_manager.get_value(\"WEBHOOK_QUERY_PARAMETERS\")),\n \"webhook_body\": _normalize_existing_text(env_manager.get_value(\"WEBHOOK_BODY\")),\n \"pcurl_to_mobile\": _env_bool(env_manager.get_value(\"PCURL_TO_MOBILE\"), True),\n }\n )\n\n\ndef _build_notification_settings_model(values: dict) -> NotificationSettings:\n if hasattr(NotificationSettings, \"model_construct\"):\n return NotificationSettings.model_construct(**values)\n return NotificationSettings.construct(**values)\n\n\ndef _normalize_patch_value(env_name: str, value):\n if env_name == \"PCURL_TO_MOBILE\":\n return bool(value)\n if value is None:\n return None\n text = str(value).strip()\n return text or None\n\n\ndef _normalize_existing_text(value: str | None) -> str | None:\n if value is None:\n return None\n text = str(value).strip()\n return text or None\n\n\ndef _env_bool(value: str | None, default: bool) -> bool:\n if value is None:\n return default\n return str(value).strip().lower() in {\"1\", \"true\", \"yes\", \"y\", \"on\"}\n\n\ndef _normalize_notification_values(values: dict) -> dict:\n normalized = dict(values)\n normalized[\"webhook_method\"] = (\n (normalized.get(\"webhook_method\") or \"POST\").strip().upper()\n )\n normalized[\"webhook_content_type\"] = (\n (normalized.get(\"webhook_content_type\") or \"JSON\").strip().upper()\n )\n\n for env_name, expect_dict in JSON_NOTIFICATION_FIELDS.items():\n attr_name = NOTIFICATION_FIELD_MAP[env_name]\n raw_value = normalized.get(attr_name)\n if raw_value is None:\n continue\n parsed = _parse_json_field(env_name, raw_value, expect_dict=expect_dict)\n normalized[attr_name] = json.dumps(\n parsed,\n ensure_ascii=False,\n separators=(\",\", \":\"),\n )\n return normalized\n\n\ndef _validate_notification_settings(settings: NotificationSettings) -> None:\n for field_name in URL_FIELDS:\n value = getattr(settings, NOTIFICATION_FIELD_MAP[field_name])\n if value is not None:\n _validate_http_url(field_name, value)\n\n _validate_pair(\n \"GOTIFY_URL\",\n settings.gotify_url,\n \"GOTIFY_TOKEN\",\n settings.gotify_token,\n )\n _validate_pair(\n \"TELEGRAM_BOT_TOKEN\",\n settings.telegram_bot_token,\n \"TELEGRAM_CHAT_ID\",\n settings.telegram_chat_id,\n )\n\n if settings.webhook_method not in ALLOWED_WEBHOOK_METHODS:\n allowed = \", \".join(sorted(ALLOWED_WEBHOOK_METHODS))\n raise NotificationSettingsValidationError(\n f\"WEBHOOK_METHOD 仅支持: {allowed}\"\n )\n if settings.webhook_content_type not in ALLOWED_WEBHOOK_CONTENT_TYPES:\n allowed = \", \".join(sorted(ALLOWED_WEBHOOK_CONTENT_TYPES))\n raise NotificationSettingsValidationError(\n f\"WEBHOOK_CONTENT_TYPE 仅支持: {allowed}\"\n )\n\n has_webhook_extras = any(\n [\n settings.webhook_headers,\n settings.webhook_query_parameters,\n settings.webhook_body,\n ]\n )\n if has_webhook_extras and not settings.webhook_url:\n raise NotificationSettingsValidationError(\n \"配置 Webhook 高级参数前必须先填写 WEBHOOK_URL\"\n )\n\n if settings.webhook_content_type == \"FORM\" and settings.webhook_body:\n parsed_body = json.loads(settings.webhook_body)\n if not isinstance(parsed_body, dict):\n raise NotificationSettingsValidationError(\n \"WEBHOOK_BODY 在 FORM 模式下必须是 JSON 对象\"\n )\n\n\ndef _validate_http_url(field_name: str, value: str) -> None:\n parsed = urlparse(value)\n if parsed.scheme not in {\"http\", \"https\"} or not parsed.netloc:\n raise NotificationSettingsValidationError(\n f\"{field_name} 必须是合法的 HTTP/HTTPS URL\"\n )\n\n\ndef _validate_pair(\n left_name: str,\n left_value: str | None,\n right_name: str,\n right_value: str | None,\n) -> None:\n if bool(left_value) == bool(right_value):\n return\n raise NotificationSettingsValidationError(\n f\"{left_name} 与 {right_name} 必须成对配置\"\n )\n\n\ndef _parse_json_field(\n field_name: str,\n raw_value: str,\n expect_dict: bool,\n):\n try:\n parsed = json.loads(raw_value)\n except json.JSONDecodeError as exc:\n raise NotificationSettingsValidationError(\n f\"{field_name} 不是合法 JSON: {exc.msg}\"\n ) from exc\n if expect_dict and not isinstance(parsed, dict):\n raise NotificationSettingsValidationError(\n f\"{field_name} 必须是 JSON 对象\"\n )\n return parsed\n" }, { "path": "src/services/notification_service.py", "content": "\"\"\"\n通知服务\n统一管理所有通知渠道\n\"\"\"\nimport asyncio\nfrom typing import Dict, List\n\nfrom src.infrastructure.external.notification_clients.base import NotificationClient\nfrom src.infrastructure.external.notification_clients.factory import build_notification_clients\nfrom src.services.notification_config_service import load_notification_settings\nfrom src.infrastructure.config.settings import NotificationSettings\n\n\nclass NotificationService:\n \"\"\"通知服务\"\"\"\n\n def __init__(self, clients: List[NotificationClient]):\n self.clients = [client for client in clients if client.is_enabled()]\n\n async def send_notification(\n self,\n product_data: Dict,\n reason: str,\n ) -> Dict[str, Dict[str, str | bool]]:\n \"\"\"\n 发送通知到所有启用的渠道\n\n Returns:\n 各渠道发送结果，包含成功状态和消息\n \"\"\"\n if not self.clients:\n return {}\n\n tasks = [\n self._send_with_result(client, product_data, reason)\n for client in self.clients\n ]\n results = await asyncio.gather(*tasks)\n return {result[\"channel\"]: result for result in results}\n\n async def send_test_notification(self) -> Dict[str, Dict[str, str | bool]]:\n test_product = {\n \"商品标题\": \"[测试通知] 闲鱼智能监控\",\n \"当前售价\": \"0\",\n \"商品链接\": \"https://www.goofish.com/\",\n }\n return await self.send_notification(\n test_product,\n \"这是一条测试通知，用于验证推送渠道是否可用。\",\n )\n\n async def _send_with_result(\n self,\n client: NotificationClient,\n product_data: Dict,\n reason: str,\n ) -> Dict[str, str | bool]:\n try:\n await client.send(product_data, reason)\n return {\n \"channel\": client.channel_key,\n \"label\": client.display_name,\n \"success\": True,\n \"message\": \"发送成功\",\n }\n except Exception as exc:\n return {\n \"channel\": client.channel_key,\n \"label\": client.display_name,\n \"success\": False,\n \"message\": str(exc),\n }\n\n\ndef build_notification_service(\n settings: NotificationSettings | None = None,\n) -> NotificationService:\n notification_settings = settings or load_notification_settings()\n return NotificationService(build_notification_clients(notification_settings))\n" }, { "path": "src/services/price_history_service.py", "content": "\"\"\"\n价格历史记录与聚合服务\n\"\"\"\nfrom __future__ import annotations\n\nimport json\nimport math\nimport os\nfrom collections import defaultdict\nfrom datetime import datetime\nfrom statistics import median\nfrom typing import Any, Iterable, Optional\n\nfrom src.infrastructure.persistence.sqlite_bootstrap import bootstrap_sqlite_storage\nfrom src.infrastructure.persistence.sqlite_connection import sqlite_connection\n\nPRICE_HISTORY_DIR = \"price_history\"\nDEFAULT_HISTORY_WINDOW_DAYS = 30\n\n\ndef normalize_keyword_slug(keyword: str) -> str:\n text = \"\".join(\n char for char in str(keyword or \"\").lower().replace(\" \", \"_\")\n if char.isalnum() or char in \"_-\"\n ).rstrip(\"_\")\n return text or \"unknown\"\n\n\ndef build_price_history_path(keyword: str) -> str:\n return os.path.join(\n PRICE_HISTORY_DIR,\n f\"{normalize_keyword_slug(keyword)}_history.jsonl\",\n )\n\n\ndef parse_price_value(value: Any) -> Optional[float]:\n if value is None:\n return None\n if isinstance(value, (int, float)):\n return round(float(value), 2)\n\n text = str(value).strip().replace(\"¥\", \"\").replace(\",\", \"\")\n if not text or text in {\"价格异常\", \"暂无\", \"-\", \"N/A\"}:\n return None\n if text.endswith(\"万\"):\n text = str(float(text[:-1]) * 10000)\n try:\n return round(float(text), 2)\n except (TypeError, ValueError):\n return None\n\n\ndef _safe_iso_datetime(value: Optional[str]) -> str:\n if value:\n return value\n return datetime.now().isoformat()\n\n\ndef _to_day(iso_text: str) -> str:\n return iso_text[:10]\n\n\ndef _build_snapshot_record(\n *,\n keyword: str,\n task_name: str,\n item: dict,\n run_id: str,\n snapshot_time: str,\n) -> Optional[dict]:\n item_id = str(item.get(\"商品ID\") or \"\").strip()\n link = str(item.get(\"商品链接\") or \"\").strip()\n unique_id = item_id or link\n price_value = parse_price_value(item.get(\"当前售价\"))\n if not unique_id or price_value is None:\n return None\n\n return {\n \"snapshot_time\": snapshot_time,\n \"snapshot_day\": _to_day(snapshot_time),\n \"run_id\": run_id,\n \"task_name\": task_name,\n \"keyword\": keyword,\n \"item_id\": unique_id,\n \"title\": item.get(\"商品标题\") or \"\",\n \"price\": price_value,\n \"price_display\": item.get(\"当前售价\") or \"\",\n \"tags\": item.get(\"商品标签\") or [],\n \"region\": item.get(\"发货地区\") or \"\",\n \"seller\": item.get(\"卖家昵称\") or \"\",\n \"publish_time\": item.get(\"发布时间\") or \"\",\n \"link\": link,\n }\n\n\ndef record_market_snapshots(\n *,\n keyword: str,\n task_name: str,\n items: Iterable[dict],\n run_id: str,\n snapshot_time: Optional[str] = None,\n seen_item_ids: Optional[set[str]] = None,\n) -> list[dict]:\n snapshot_time = _safe_iso_datetime(snapshot_time)\n seen = seen_item_ids if seen_item_ids is not None else set()\n records: list[dict] = []\n\n for item in items:\n record = _build_snapshot_record(\n keyword=keyword,\n task_name=task_name,\n item=item,\n run_id=run_id,\n snapshot_time=snapshot_time,\n )\n if record is None or record[\"item_id\"] in seen:\n continue\n seen.add(record[\"item_id\"])\n records.append(record)\n\n if not records:\n return []\n\n bootstrap_sqlite_storage()\n keyword_slug = normalize_keyword_slug(keyword)\n with sqlite_connection() as conn:\n for record in records:\n conn.execute(\n \"\"\"\n INSERT OR IGNORE INTO price_snapshots (\n keyword_slug, keyword, task_name, snapshot_time, snapshot_day,\n run_id, item_id, title, price, price_display, tags_json, region,\n seller, publish_time, link\n ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)\n \"\"\",\n (\n keyword_slug,\n record.get(\"keyword\", keyword),\n record.get(\"task_name\", task_name),\n record.get(\"snapshot_time\", snapshot_time),\n record.get(\"snapshot_day\", _to_day(snapshot_time)),\n record.get(\"run_id\", run_id),\n record.get(\"item_id\", \"\"),\n record.get(\"title\", \"\"),\n record.get(\"price\"),\n record.get(\"price_display\", \"\"),\n json.dumps(record.get(\"tags\") or [], ensure_ascii=False),\n record.get(\"region\", \"\"),\n record.get(\"seller\", \"\"),\n record.get(\"publish_time\", \"\"),\n record.get(\"link\", \"\"),\n ),\n )\n conn.commit()\n return records\n\n\ndef load_price_snapshots(keyword: str) -> list[dict]:\n bootstrap_sqlite_storage()\n with sqlite_connection() as conn:\n rows = conn.execute(\n \"\"\"\n SELECT *\n FROM price_snapshots\n WHERE keyword_slug = ?\n ORDER BY snapshot_time ASC, id ASC\n \"\"\",\n (normalize_keyword_slug(keyword),),\n ).fetchall()\n snapshots: list[dict] = []\n for row in rows:\n snapshots.append(\n {\n \"snapshot_time\": row[\"snapshot_time\"],\n \"snapshot_day\": row[\"snapshot_day\"],\n \"run_id\": row[\"run_id\"],\n \"task_name\": row[\"task_name\"],\n \"keyword\": row[\"keyword\"],\n \"item_id\": row[\"item_id\"],\n \"title\": row[\"title\"],\n \"price\": row[\"price\"],\n \"price_display\": row[\"price_display\"],\n \"tags\": json.loads(row[\"tags_json\"] or \"[]\"),\n \"region\": row[\"region\"],\n \"seller\": row[\"seller\"],\n \"publish_time\": row[\"publish_time\"],\n \"link\": row[\"link\"],\n }\n )\n return snapshots\n\n\ndef delete_price_snapshots(keyword: str) -> int:\n bootstrap_sqlite_storage()\n with sqlite_connection() as conn:\n cursor = conn.execute(\n \"DELETE FROM price_snapshots WHERE keyword_slug = ?\",\n (normalize_keyword_slug(keyword),),\n )\n conn.commit()\n return int(cursor.rowcount or 0)\n\n\ndef _dedupe_latest(records: Iterable[dict], group_key: str) -> list[dict]:\n latest_by_key: dict[str, dict] = {}\n for record in records:\n key = str(record.get(group_key) or \"\").strip()\n if not key:\n continue\n latest_by_key[key] = record\n return list(latest_by_key.values())\n\n\ndef _summarize_prices(records: Iterable[dict]) -> dict:\n entries = [record for record in records if parse_price_value(record.get(\"price\")) is not None]\n prices = [float(record[\"price\"]) for record in entries]\n if not prices:\n return {\n \"sample_count\": 0,\n \"avg_price\": None,\n \"median_price\": None,\n \"min_price\": None,\n \"max_price\": None,\n }\n\n return {\n \"sample_count\": len(prices),\n \"avg_price\": round(sum(prices) / len(prices), 2),\n \"median_price\": round(float(median(prices)), 2),\n \"min_price\": round(min(prices), 2),\n \"max_price\": round(max(prices), 2),\n }\n\n\ndef _build_daily_trend(snapshots: list[dict]) -> list[dict]:\n grouped: dict[str, list[dict]] = defaultdict(list)\n for snapshot in snapshots:\n grouped[str(snapshot.get(\"snapshot_day\") or \"\")].append(snapshot)\n\n points: list[dict] = []\n for day in sorted(grouped.keys()):\n day_records = _dedupe_latest(grouped[day], \"item_id\")\n summary = _summarize_prices(day_records)\n summary[\"day\"] = day\n points.append(summary)\n return points\n\n\ndef _recent_window_snapshots(snapshots: list[dict], window_days: int) -> list[dict]:\n if not snapshots:\n return []\n latest_time = max(str(record.get(\"snapshot_time\") or \"\") for record in snapshots)\n latest_dt = datetime.fromisoformat(latest_time)\n filtered = []\n for record in snapshots:\n current_time = datetime.fromisoformat(str(record.get(\"snapshot_time\") or latest_time))\n if (latest_dt - current_time).days <= max(0, window_days):\n filtered.append(record)\n return filtered\n\n\ndef _resolve_deal_label(score: int) -> str:\n if score >= 65:\n return \"高性价比\"\n if score >= 50:\n return \"值得关注\"\n if score >= 40:\n return \"价格正常\"\n return \"价格偏高\"\n\n\ndef build_item_price_context(\n snapshots: list[dict],\n *,\n item_id: str,\n current_price: Optional[float],\n) -> dict:\n if not item_id:\n return {\"observation_count\": 0, \"deal_score\": None, \"deal_label\": \"暂无数据\"}\n\n item_snapshots = [record for record in snapshots if str(record.get(\"item_id\")) == str(item_id)]\n if not item_snapshots:\n return {\"observation_count\": 0, \"deal_score\": None, \"deal_label\": \"暂无数据\"}\n\n latest_item_snapshot = item_snapshots[-1]\n price_now = current_price if current_price is not None else parse_price_value(latest_item_snapshot.get(\"price\"))\n historical_prices = [float(record[\"price\"]) for record in item_snapshots if parse_price_value(record.get(\"price\")) is not None]\n latest_run_id = str(snapshots[-1].get(\"run_id\") or \"\")\n latest_market = _dedupe_latest(\n [record for record in snapshots if str(record.get(\"run_id\") or \"\") == latest_run_id],\n \"item_id\",\n )\n market_summary = _summarize_prices(latest_market)\n market_avg = market_summary.get(\"avg_price\")\n market_median = market_summary.get(\"median_price\")\n\n score = 50\n if price_now is not None and market_avg:\n score += int(((market_avg - price_now) / market_avg) * 60)\n if price_now is not None and historical_prices:\n historical_max = max(historical_prices)\n if historical_max > 0:\n score += int(((historical_max - price_now) / historical_max) * 20)\n if math.isclose(price_now, min(historical_prices), rel_tol=0.001):\n score += 8\n score = max(0, min(100, score))\n\n previous_price = historical_prices[-2] if len(historical_prices) >= 2 else None\n change_amount = None if previous_price is None or price_now is None else round(price_now - previous_price, 2)\n change_percent = None\n if change_amount is not None and previous_price:\n change_percent = round(change_amount / previous_price * 100, 2)\n\n return {\n \"observation_count\": len(historical_prices),\n \"current_price\": price_now,\n \"avg_price\": round(sum(historical_prices) / len(historical_prices), 2),\n \"median_price\": round(float(median(historical_prices)), 2),\n \"min_price\": round(min(historical_prices), 2),\n \"max_price\": round(max(historical_prices), 2),\n \"first_seen_at\": item_snapshots[0].get(\"snapshot_time\"),\n \"last_seen_at\": latest_item_snapshot.get(\"snapshot_time\"),\n \"market_avg_price\": market_avg,\n \"market_median_price\": market_median,\n \"price_change_amount\": change_amount,\n \"price_change_percent\": change_percent,\n \"deal_score\": score,\n \"deal_label\": _resolve_deal_label(score),\n }\n\n\ndef build_market_reference(\n *,\n keyword: str,\n item: dict,\n current_market_items: list[dict],\n historical_snapshots: list[dict],\n) -> dict:\n current_market_records = []\n for market_item in current_market_items:\n price = parse_price_value(market_item.get(\"当前售价\"))\n if price is None:\n continue\n current_market_records.append({\"price\": price})\n\n market_snapshot = _summarize_prices(current_market_records)\n history_summary = _summarize_prices(_dedupe_latest(historical_snapshots, \"item_id\"))\n item_context = build_item_price_context(\n historical_snapshots,\n item_id=str(item.get(\"商品ID\") or \"\"),\n current_price=parse_price_value(item.get(\"当前售价\")),\n )\n return {\n \"当前搜索样本\": market_snapshot,\n \"历史价格概览\": history_summary,\n \"本商品价格位置\": item_context,\n \"关键词\": keyword,\n }\n\n\ndef build_price_history_insights(\n keyword: str,\n *,\n window_days: int = DEFAULT_HISTORY_WINDOW_DAYS,\n) -> dict:\n snapshots = load_price_snapshots(keyword)\n if not snapshots:\n return {\n \"market_summary\": _summarize_prices([]),\n \"history_summary\": {\"unique_items\": 0, **_summarize_prices([])},\n \"daily_trend\": [],\n \"latest_snapshot_at\": None,\n }\n\n recent_snapshots = _recent_window_snapshots(snapshots, window_days)\n latest_run_id = str(snapshots[-1].get(\"run_id\") or \"\")\n latest_run_snapshots = _dedupe_latest(\n [record for record in snapshots if str(record.get(\"run_id\") or \"\") == latest_run_id],\n \"item_id\",\n )\n latest_records_by_item = _dedupe_latest(recent_snapshots, \"item_id\")\n\n return {\n \"market_summary\": {\n **_summarize_prices(latest_run_snapshots),\n \"snapshot_time\": snapshots[-1].get(\"snapshot_time\"),\n },\n \"history_summary\": {\n \"unique_items\": len(latest_records_by_item),\n **_summarize_prices(latest_records_by_item),\n },\n \"daily_trend\": _build_daily_trend(recent_snapshots),\n \"latest_snapshot_at\": snapshots[-1].get(\"snapshot_time\"),\n }\n" }, { "path": "src/services/process_service.py", "content": "\"\"\"\n进程管理服务\n负责管理爬虫进程的启动和停止\n\"\"\"\n\nimport asyncio\nimport contextlib\nimport os\nimport signal\nimport sys\nfrom datetime import datetime\nfrom typing import Awaitable, Callable, Dict, TextIO\n\nfrom src.ai_handler import send_ntfy_notification\nfrom src.config import STATE_FILE\nfrom src.failure_guard import FailureGuard\nfrom src.infrastructure.persistence.sqlite_task_repository import find_task_by_name_sync\nfrom src.utils import build_task_log_path\n\nSTOP_TIMEOUT_SECONDS = 20\nSPIDER_DEBUG_LIMIT_ENV = \"SPIDER_DEBUG_LIMIT\"\nLifecycleHook = Callable[[int], Awaitable[None] | None]\n\n\nclass ProcessService:\n \"\"\"进程管理服务\"\"\"\n\n def __init__(self):\n self.processes: Dict[int, asyncio.subprocess.Process] = {}\n self.log_paths: Dict[int, str] = {}\n self.log_handles: Dict[int, TextIO] = {}\n self.task_names: Dict[int, str] = {}\n self.exit_watchers: Dict[int, asyncio.Task] = {}\n self.failure_guard = FailureGuard()\n self._on_started: LifecycleHook | None = None\n self._on_stopped: LifecycleHook | None = None\n\n def set_lifecycle_hooks(\n self,\n *,\n on_started: LifecycleHook | None = None,\n on_stopped: LifecycleHook | None = None,\n ) -> None:\n self._on_started = on_started\n self._on_stopped = on_stopped\n\n async def _invoke_hook(self, hook: LifecycleHook | None, task_id: int) -> None:\n if hook is None:\n return\n result = hook(task_id)\n if asyncio.iscoroutine(result):\n await result\n\n def _resolve_cookie_path(self, task_name: str) -> str | None:\n \"\"\"Best-effort cookie/state path for a task.\"\"\"\n try:\n task = find_task_by_name_sync(task_name)\n if task and isinstance(task.account_state_file, str) and task.account_state_file.strip():\n return task.account_state_file.strip()\n except Exception:\n pass\n\n return STATE_FILE if os.path.exists(STATE_FILE) else None\n\n def is_running(self, task_id: int) -> bool:\n \"\"\"检查任务是否正在运行\"\"\"\n process = self.processes.get(task_id)\n return process is not None and process.returncode is None\n\n async def _drain_finished_process(self, task_id: int) -> None:\n process = self.processes.get(task_id)\n if process is None or process.returncode is None:\n return\n\n watcher = self.exit_watchers.get(task_id)\n if watcher is not None:\n await asyncio.shield(watcher)\n return\n\n self._cleanup_runtime(task_id, process)\n await self._invoke_hook(self._on_stopped, task_id)\n\n def _open_log_file(self, task_id: int, task_name: str) -> tuple[str, TextIO]:\n os.makedirs(\"logs\", exist_ok=True)\n log_file_path = build_task_log_path(task_id, task_name)\n log_file_handle = open(log_file_path, \"a\", encoding=\"utf-8\")\n return log_file_path, log_file_handle\n\n def _build_spawn_command(self, task_name: str) -> list[str]:\n command = [\n sys.executable,\n \"-u\",\n \"spider_v2.py\",\n \"--task-name\",\n task_name,\n ]\n debug_limit = str(os.getenv(SPIDER_DEBUG_LIMIT_ENV, \"\")).strip()\n if debug_limit.isdigit() and int(debug_limit) > 0:\n command.extend([\"--debug-limit\", debug_limit])\n return command\n\n async def _spawn_process(\n self,\n task_name: str,\n log_file_handle: TextIO,\n ) -> asyncio.subprocess.Process:\n preexec_fn = os.setsid if sys.platform != \"win32\" else None\n child_env = os.environ.copy()\n child_env[\"PYTHONIOENCODING\"] = \"utf-8\"\n child_env[\"PYTHONUTF8\"] = \"1\"\n return await asyncio.create_subprocess_exec(\n *self._build_spawn_command(task_name),\n stdout=log_file_handle,\n stderr=log_file_handle,\n preexec_fn=preexec_fn,\n env=child_env,\n )\n\n def _register_runtime(\n self,\n task_id: int,\n task_name: str,\n process: asyncio.subprocess.Process,\n log_file_path: str,\n log_file_handle: TextIO,\n ) -> None:\n self.processes[task_id] = process\n self.log_paths[task_id] = log_file_path\n self.log_handles[task_id] = log_file_handle\n self.task_names[task_id] = task_name\n self.exit_watchers[task_id] = asyncio.create_task(self._watch_process_exit(process))\n\n async def start_task(self, task_id: int, task_name: str) -> bool:\n \"\"\"启动任务进程\"\"\"\n await self._drain_finished_process(task_id)\n if self.is_running(task_id):\n print(f\"任务 '{task_name}' (ID: {task_id}) 已在运行中\")\n return False\n\n decision = self.failure_guard.should_skip_start(\n task_name,\n cookie_path=self._resolve_cookie_path(task_name),\n )\n if decision.skip:\n await self._notify_skip(task_name, decision)\n return False\n\n log_file_path = \"\"\n log_file_handle = None\n try:\n log_file_path, log_file_handle = self._open_log_file(task_id, task_name)\n process = await self._spawn_process(task_name, log_file_handle)\n except Exception as exc:\n self._close_log_handle(log_file_handle)\n print(f\"启动任务 '{task_name}' 失败: {exc}\")\n return False\n\n self._register_runtime(task_id, task_name, process, log_file_path, log_file_handle)\n print(f\"启动任务 '{task_name}' (PID: {process.pid})\")\n await self._invoke_hook(self._on_started, task_id)\n return True\n\n async def _notify_skip(self, task_name: str, decision) -> None:\n print(\n f\"[FailureGuard] 跳过启动任务 '{task_name}'，已暂停重试 \"\n f\"(连续失败 {decision.consecutive_failures}/{self.failure_guard.threshold})\"\n )\n if not decision.should_notify:\n return\n try:\n await send_ntfy_notification(\n {\n \"商品标题\": f\"[任务暂停] {task_name}\",\n \"当前售价\": \"N/A\",\n \"商品链接\": \"#\",\n },\n \"任务处于暂停状态，将跳过执行。\\n\"\n f\"原因: {decision.reason}\\n\"\n f\"连续失败: {decision.consecutive_failures}/{self.failure_guard.threshold}\\n\"\n f\"暂停到: {decision.paused_until.strftime('%Y-%m-%d %H:%M:%S') if decision.paused_until else 'N/A'}\\n\"\n \"修复方法: 更新登录态/cookies文件后会自动恢复。\",\n )\n except Exception as exc:\n print(f\"发送任务暂停通知失败: {exc}\")\n\n async def _watch_process_exit(self, process: asyncio.subprocess.Process) -> None:\n await process.wait()\n task_id = self._find_task_id_by_process(process)\n if task_id is None:\n return\n self._cleanup_runtime(task_id, process)\n await self._invoke_hook(self._on_stopped, task_id)\n\n def _find_task_id_by_process(self, process: asyncio.subprocess.Process) -> int | None:\n for task_id, current_process in self.processes.items():\n if current_process is process:\n return task_id\n return None\n\n def _cleanup_runtime(\n self,\n task_id: int,\n process: asyncio.subprocess.Process,\n ) -> None:\n if self.processes.get(task_id) is not process:\n return\n self.processes.pop(task_id, None)\n self.log_paths.pop(task_id, None)\n self.task_names.pop(task_id, None)\n self._close_log_handle(self.log_handles.pop(task_id, None))\n self.exit_watchers.pop(task_id, None)\n\n def _close_log_handle(self, log_handle: TextIO | None) -> None:\n if log_handle is None:\n return\n with contextlib.suppress(Exception):\n log_handle.close()\n\n def _append_stop_marker(self, log_path: str | None) -> None:\n if not log_path:\n return\n try:\n timestamp = datetime.now().strftime(\" %Y-%m-%d %H:%M:%S\")\n with open(log_path, \"a\", encoding=\"utf-8\") as log_file:\n log_file.write(f\"[{timestamp}] !!! 任务已被终止 !!!\\n\")\n except Exception as exc:\n print(f\"写入任务终止标记失败: {exc}\")\n\n async def stop_task(self, task_id: int) -> bool:\n \"\"\"停止任务进程\"\"\"\n await self._drain_finished_process(task_id)\n process = self.processes.get(task_id)\n if process is None:\n print(f\"任务 ID {task_id} 没有正在运行的进程\")\n return False\n if process.returncode is not None:\n await self._await_exit_watcher(task_id)\n print(f\"任务进程 {process.pid} (ID: {task_id}) 已退出，略过停止\")\n return False\n\n try:\n await self._terminate_process(process, task_id)\n self._append_stop_marker(self.log_paths.get(task_id))\n await self._await_exit_watcher(task_id)\n print(f\"任务进程 {process.pid} (ID: {task_id}) 已终止\")\n return True\n except ProcessLookupError:\n print(f\"进程 (ID: {task_id}) 已不存在\")\n return False\n except Exception as exc:\n print(f\"停止任务进程 (ID: {task_id}) 时出错: {exc}\")\n return False\n\n async def _terminate_process(\n self,\n process: asyncio.subprocess.Process,\n task_id: int,\n ) -> None:\n if sys.platform != \"win32\":\n os.killpg(os.getpgid(process.pid), signal.SIGTERM)\n else:\n process.terminate()\n\n try:\n await asyncio.wait_for(process.wait(), timeout=STOP_TIMEOUT_SECONDS)\n return\n except asyncio.TimeoutError:\n print(\n f\"任务进程 {process.pid} (ID: {task_id}) 未在 \"\n f\"{STOP_TIMEOUT_SECONDS} 秒内退出，准备强制终止...\"\n )\n\n if sys.platform != \"win32\":\n with contextlib.suppress(ProcessLookupError):\n os.killpg(os.getpgid(process.pid), signal.SIGKILL)\n else:\n process.kill()\n await process.wait()\n\n async def _await_exit_watcher(self, task_id: int) -> None:\n watcher = self.exit_watchers.get(task_id)\n if watcher is None:\n return\n await asyncio.shield(watcher)\n\n def reindex_after_delete(self, deleted_task_id: int) -> None:\n \"\"\"删除任务后同步重排运行时索引，避免任务下标漂移。\"\"\"\n self.processes = self._reindex_mapping(self.processes, deleted_task_id)\n self.log_paths = self._reindex_mapping(self.log_paths, deleted_task_id)\n self.log_handles = self._reindex_mapping(self.log_handles, deleted_task_id)\n self.task_names = self._reindex_mapping(self.task_names, deleted_task_id)\n self.exit_watchers = self._reindex_mapping(self.exit_watchers, deleted_task_id)\n\n def _reindex_mapping(self, mapping: Dict[int, object], deleted_task_id: int) -> Dict[int, object]:\n reindexed: Dict[int, object] = {}\n for task_id, value in mapping.items():\n if task_id == deleted_task_id:\n continue\n next_task_id = task_id - 1 if task_id > deleted_task_id else task_id\n reindexed[next_task_id] = value\n return reindexed\n\n async def stop_all(self) -> None:\n \"\"\"停止所有任务进程\"\"\"\n task_ids = list(self.processes.keys())\n for task_id in task_ids:\n await self.stop_task(task_id)\n" }, { "path": "src/services/result_export_service.py", "content": "\"\"\"\n结果导出服务\n\"\"\"\nimport csv\nfrom io import StringIO\n\n\nEXPORT_HEADERS = [\n \"任务名称\",\n \"搜索关键字\",\n \"商品ID\",\n \"商品标题\",\n \"当前售价\",\n \"发布时间\",\n \"卖家昵称\",\n \"AI是否推荐\",\n \"分析来源\",\n \"原因\",\n \"价格观察次数\",\n \"价格最低值\",\n \"价格最高值\",\n \"市场均价\",\n \"性价比分数\",\n \"性价比标签\",\n \"商品链接\",\n]\n\n\ndef build_results_csv(records: list[dict]) -> str:\n buffer = StringIO()\n writer = csv.DictWriter(buffer, fieldnames=EXPORT_HEADERS)\n writer.writeheader()\n\n for record in records:\n item = record.get(\"商品信息\", {}) or {}\n seller = record.get(\"卖家信息\", {}) or {}\n ai_analysis = record.get(\"ai_analysis\", {}) or {}\n price_insight = record.get(\"price_insight\", {}) or {}\n writer.writerow(\n {\n \"任务名称\": record.get(\"任务名称\", \"\"),\n \"搜索关键字\": record.get(\"搜索关键字\", \"\"),\n \"商品ID\": item.get(\"商品ID\", \"\"),\n \"商品标题\": item.get(\"商品标题\", \"\"),\n \"当前售价\": item.get(\"当前售价\", \"\"),\n \"发布时间\": item.get(\"发布时间\", \"\"),\n \"卖家昵称\": seller.get(\"卖家昵称\") or item.get(\"卖家昵称\", \"\"),\n \"AI是否推荐\": \"是\" if ai_analysis.get(\"is_recommended\") else \"否\",\n \"分析来源\": ai_analysis.get(\"analysis_source\", \"\"),\n \"原因\": ai_analysis.get(\"reason\", \"\"),\n \"价格观察次数\": price_insight.get(\"observation_count\", \"\"),\n \"价格最低值\": price_insight.get(\"min_price\", \"\"),\n \"价格最高值\": price_insight.get(\"max_price\", \"\"),\n \"市场均价\": price_insight.get(\"market_avg_price\", \"\"),\n \"性价比分数\": ai_analysis.get(\"value_score\", price_insight.get(\"deal_score\", \"\")),\n \"性价比标签\": ai_analysis.get(\"value_summary\", price_insight.get(\"deal_label\", \"\")),\n \"商品链接\": item.get(\"商品链接\", \"\"),\n }\n )\n\n return buffer.getvalue()\n" }, { "path": "src/services/result_file_service.py", "content": "\"\"\"\n结果记录富化与文件名校验服务\n\"\"\"\n\nfrom src.infrastructure.persistence.storage_names import normalize_keyword_from_filename\nfrom src.services.price_history_service import (\n build_item_price_context,\n load_price_snapshots,\n parse_price_value,\n)\n\n\ndef validate_result_filename(filename: str) -> None:\n if not filename.endswith(\".jsonl\") or \"/\" in filename or \"..\" in filename:\n raise ValueError(\"无效的文件名\")\n\n\ndef enrich_records_with_price_insight(records: list[dict], filename: str) -> list[dict]:\n snapshots = load_price_snapshots(normalize_keyword_from_filename(filename))\n if not snapshots:\n return records\n\n enriched = []\n for record in records:\n info = record.get(\"商品信息\", {}) or {}\n clone = dict(record)\n clone[\"price_insight\"] = build_item_price_context(\n snapshots,\n item_id=str(info.get(\"商品ID\") or \"\"),\n current_price=parse_price_value(info.get(\"当前售价\")),\n )\n enriched.append(clone)\n return enriched\n" }, { "path": "src/services/result_storage_service.py", "content": "\"\"\"\n结果数据的 SQLite 读写服务。\n\"\"\"\nfrom __future__ import annotations\n\nimport asyncio\nimport hashlib\nimport json\n\nfrom src.infrastructure.persistence.sqlite_bootstrap import bootstrap_sqlite_storage\nfrom src.infrastructure.persistence.sqlite_connection import sqlite_connection\nfrom src.infrastructure.persistence.storage_names import build_result_filename\nfrom src.services.price_history_service import parse_price_value\n\n\nSORT_COLUMN_MAP = {\n \"crawl_time\": \"crawl_time\",\n \"publish_time\": \"COALESCE(publish_time, '')\",\n \"price\": \"COALESCE(price, 0)\",\n \"keyword_hit_count\": \"keyword_hit_count\",\n}\n\n\ndef _get_link_unique_key(link: str) -> str:\n return link.split(\"&\", 1)[0]\n\n\ndef _fallback_unique_key(record: dict, item: dict) -> str:\n item_id = str(item.get(\"商品ID\") or \"\").strip()\n if item_id:\n return f\"item:{item_id}\"\n digest = hashlib.sha1(\n json.dumps(record, ensure_ascii=False, sort_keys=True).encode(\"utf-8\")\n ).hexdigest()\n return f\"hash:{digest}\"\n\n\ndef _parse_raw_record(raw_json: str) -> dict:\n return json.loads(raw_json)\n\n\ndef _build_query_conditions(\n *,\n filename: str,\n ai_recommended_only: bool,\n keyword_recommended_only: bool,\n) -> tuple[str, list]:\n conditions = [\"result_filename = ?\"]\n params: list = [filename]\n if ai_recommended_only:\n conditions.append(\"is_recommended = 1\")\n conditions.append(\"analysis_source = ?\")\n params.append(\"ai\")\n if keyword_recommended_only:\n conditions.append(\"is_recommended = 1\")\n conditions.append(\"analysis_source = ?\")\n params.append(\"keyword\")\n return \" AND \".join(conditions), params\n\n\ndef _sort_expression(sort_by: str, sort_order: str) -> str:\n column = SORT_COLUMN_MAP.get(sort_by, SORT_COLUMN_MAP[\"crawl_time\"])\n direction = \"ASC\" if sort_order == \"asc\" else \"DESC\"\n return f\"{column} {direction}, id {direction}\"\n\n\nasync def save_result_record(record: dict, keyword: str) -> bool:\n return await asyncio.to_thread(_save_result_record_sync, record, keyword)\n\n\ndef _save_result_record_sync(record: dict, keyword: str) -> bool:\n bootstrap_sqlite_storage()\n item = record.get(\"商品信息\", {}) or {}\n analysis = record.get(\"ai_analysis\", {}) or {}\n link = str(item.get(\"商品链接\") or \"\")\n link_unique_key = _get_link_unique_key(link) if link else _fallback_unique_key(record, item)\n keyword_hit_count = analysis.get(\"keyword_hit_count\", 0)\n try:\n keyword_hit_count = int(keyword_hit_count)\n except (TypeError, ValueError):\n keyword_hit_count = 0\n\n with sqlite_connection() as conn:\n conn.execute(\n \"\"\"\n INSERT OR IGNORE INTO result_items (\n result_filename, keyword, task_name, crawl_time, publish_time, price,\n price_display, item_id, title, link, link_unique_key, seller_nickname,\n is_recommended, analysis_source, keyword_hit_count, raw_json\n ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)\n \"\"\",\n (\n build_result_filename(keyword),\n record.get(\"搜索关键字\", keyword),\n record.get(\"任务名称\", \"\"),\n record.get(\"爬取时间\", \"\"),\n item.get(\"发布时间\"),\n parse_price_value(item.get(\"当前售价\")),\n item.get(\"当前售价\"),\n item.get(\"商品ID\"),\n item.get(\"商品标题\"),\n link,\n link_unique_key,\n (record.get(\"卖家信息\", {}) or {}).get(\"卖家昵称\") or item.get(\"卖家昵称\"),\n 1 if analysis.get(\"is_recommended\") else 0,\n analysis.get(\"analysis_source\"),\n keyword_hit_count,\n json.dumps(record, ensure_ascii=False),\n ),\n )\n conn.commit()\n return True\n\n\ndef load_processed_link_keys(keyword: str) -> set[str]:\n bootstrap_sqlite_storage()\n filename = build_result_filename(keyword)\n with sqlite_connection() as conn:\n rows = conn.execute(\n \"SELECT link_unique_key FROM result_items WHERE result_filename = ?\",\n (filename,),\n ).fetchall()\n return {str(row[\"link_unique_key\"]) for row in rows if row[\"link_unique_key\"]}\n\n\nasync def list_result_filenames() -> list[str]:\n return await asyncio.to_thread(_list_result_filenames_sync)\n\n\ndef _list_result_filenames_sync() -> list[str]:\n bootstrap_sqlite_storage()\n with sqlite_connection() as conn:\n rows = conn.execute(\n \"\"\"\n SELECT result_filename, MAX(crawl_time) AS latest_crawl_time\n FROM result_items\n GROUP BY result_filename\n ORDER BY latest_crawl_time DESC, result_filename DESC\n \"\"\"\n ).fetchall()\n return [str(row[\"result_filename\"]) for row in rows]\n\n\nasync def result_file_exists(filename: str) -> bool:\n return await asyncio.to_thread(_result_file_exists_sync, filename)\n\n\ndef _result_file_exists_sync(filename: str) -> bool:\n bootstrap_sqlite_storage()\n with sqlite_connection() as conn:\n row = conn.execute(\n \"SELECT 1 FROM result_items WHERE result_filename = ? LIMIT 1\",\n (filename,),\n ).fetchone()\n return row is not None\n\n\nasync def delete_result_file_records(filename: str) -> int:\n return await asyncio.to_thread(_delete_result_file_records_sync, filename)\n\n\ndef _delete_result_file_records_sync(filename: str) -> int:\n bootstrap_sqlite_storage()\n with sqlite_connection() as conn:\n cursor = conn.execute(\n \"DELETE FROM result_items WHERE result_filename = ?\",\n (filename,),\n )\n conn.commit()\n return int(cursor.rowcount or 0)\n\n\nasync def query_result_records(\n filename: str,\n *,\n ai_recommended_only: bool,\n keyword_recommended_only: bool,\n sort_by: str,\n sort_order: str,\n page: int,\n limit: int,\n) -> tuple[int, list[dict]]:\n return await asyncio.to_thread(\n _query_result_records_sync,\n filename,\n ai_recommended_only,\n keyword_recommended_only,\n sort_by,\n sort_order,\n page,\n limit,\n )\n\n\ndef _query_result_records_sync(\n filename: str,\n ai_recommended_only: bool,\n keyword_recommended_only: bool,\n sort_by: str,\n sort_order: str,\n page: int,\n limit: int,\n) -> tuple[int, list[dict]]:\n bootstrap_sqlite_storage()\n where_clause, params = _build_query_conditions(\n filename=filename,\n ai_recommended_only=ai_recommended_only,\n keyword_recommended_only=keyword_recommended_only,\n )\n offset = max(page - 1, 0) * limit\n order_clause = _sort_expression(sort_by, sort_order)\n with sqlite_connection() as conn:\n total_row = conn.execute(\n f\"SELECT COUNT(1) AS total FROM result_items WHERE {where_clause}\",\n tuple(params),\n ).fetchone()\n rows = conn.execute(\n f\"\"\"\n SELECT raw_json\n FROM result_items\n WHERE {where_clause}\n ORDER BY {order_clause}\n LIMIT ? OFFSET ?\n \"\"\",\n tuple(params + [limit, offset]),\n ).fetchall()\n total = int(total_row[\"total\"]) if total_row else 0\n return total, [_parse_raw_record(str(row[\"raw_json\"])) for row in rows]\n\n\nasync def load_all_result_records(\n filename: str,\n *,\n ai_recommended_only: bool,\n keyword_recommended_only: bool,\n sort_by: str,\n sort_order: str,\n) -> list[dict]:\n return await asyncio.to_thread(\n _load_all_result_records_sync,\n filename,\n ai_recommended_only,\n keyword_recommended_only,\n sort_by,\n sort_order,\n )\n\n\ndef _load_all_result_records_sync(\n filename: str,\n ai_recommended_only: bool,\n keyword_recommended_only: bool,\n sort_by: str,\n sort_order: str,\n) -> list[dict]:\n bootstrap_sqlite_storage()\n where_clause, params = _build_query_conditions(\n filename=filename,\n ai_recommended_only=ai_recommended_only,\n keyword_recommended_only=keyword_recommended_only,\n )\n order_clause = _sort_expression(sort_by, sort_order)\n with sqlite_connection() as conn:\n rows = conn.execute(\n f\"\"\"\n SELECT raw_json\n FROM result_items\n WHERE {where_clause}\n ORDER BY {order_clause}\n \"\"\",\n tuple(params),\n ).fetchall()\n return [_parse_raw_record(str(row[\"raw_json\"])) for row in rows]\n\n\nasync def build_result_ndjson(filename: str) -> str:\n records = await load_all_result_records(\n filename,\n ai_recommended_only=False,\n keyword_recommended_only=False,\n sort_by=\"crawl_time\",\n sort_order=\"asc\",\n )\n return \"\\n\".join(json.dumps(record, ensure_ascii=False) for record in records)\n\n\nasync def load_result_summary(filename: str) -> dict | None:\n return await asyncio.to_thread(_load_result_summary_sync, filename)\n\n\ndef _load_result_summary_sync(filename: str) -> dict | None:\n bootstrap_sqlite_storage()\n with sqlite_connection() as conn:\n aggregate_row = conn.execute(\n \"\"\"\n SELECT\n COUNT(1) AS total_items,\n SUM(CASE WHEN is_recommended = 1 THEN 1 ELSE 0 END) AS recommended_items,\n SUM(CASE WHEN is_recommended = 1 AND analysis_source = 'ai' THEN 1 ELSE 0 END) AS ai_recommended_items,\n SUM(CASE WHEN is_recommended = 1 AND analysis_source = 'keyword' THEN 1 ELSE 0 END) AS keyword_recommended_items,\n MAX(crawl_time) AS latest_crawl_time\n FROM result_items\n WHERE result_filename = ?\n \"\"\",\n (filename,),\n ).fetchone()\n if aggregate_row is None or int(aggregate_row[\"total_items\"] or 0) == 0:\n return None\n\n latest_record = conn.execute(\n \"\"\"\n SELECT raw_json FROM result_items\n WHERE result_filename = ?\n ORDER BY crawl_time DESC, id DESC\n LIMIT 1\n \"\"\",\n (filename,),\n ).fetchone()\n latest_recommendation = conn.execute(\n \"\"\"\n SELECT raw_json FROM result_items\n WHERE result_filename = ? AND is_recommended = 1\n ORDER BY crawl_time DESC, id DESC\n LIMIT 1\n \"\"\",\n (filename,),\n ).fetchone()\n return {\n \"total_items\": int(aggregate_row[\"total_items\"] or 0),\n \"recommended_items\": int(aggregate_row[\"recommended_items\"] or 0),\n \"ai_recommended_items\": int(aggregate_row[\"ai_recommended_items\"] or 0),\n \"keyword_recommended_items\": int(aggregate_row[\"keyword_recommended_items\"] or 0),\n \"latest_crawl_time\": aggregate_row[\"latest_crawl_time\"],\n \"latest_record\": (\n _parse_raw_record(str(latest_record[\"raw_json\"])) if latest_record else None\n ),\n \"latest_recommendation\": (\n _parse_raw_record(str(latest_recommendation[\"raw_json\"]))\n if latest_recommendation\n else None\n ),\n }\n" }, { "path": "src/services/scheduler_service.py", "content": "\"\"\"\n调度服务\n负责管理定时任务的调度\n\"\"\"\nfrom datetime import datetime\nfrom apscheduler.schedulers.asyncio import AsyncIOScheduler\nfrom typing import List\n\nfrom src.core.cron_utils import build_cron_trigger\nfrom src.domain.models.task import Task\nfrom src.services.process_service import ProcessService\n\n\nclass SchedulerService:\n \"\"\"调度服务\"\"\"\n\n def __init__(self, process_service: ProcessService):\n self.scheduler = AsyncIOScheduler(timezone=\"Asia/Shanghai\")\n self.process_service = process_service\n\n def start(self):\n \"\"\"启动调度器\"\"\"\n if not self.scheduler.running:\n self.scheduler.start()\n print(\"调度器已启动\")\n\n def stop(self):\n \"\"\"停止调度器\"\"\"\n if self.scheduler.running:\n self.scheduler.shutdown()\n print(\"调度器已停止\")\n\n def get_next_run_time(self, task_id: int):\n job = self.scheduler.get_job(f\"task_{task_id}\")\n if job is None:\n return None\n\n next_run_time = getattr(job, \"next_run_time\", None)\n if next_run_time is not None:\n return next_run_time\n\n trigger = getattr(job, \"trigger\", None)\n if trigger is None or not hasattr(trigger, \"get_next_fire_time\"):\n return None\n\n try:\n now = datetime.now(self.scheduler.timezone)\n return trigger.get_next_fire_time(None, now)\n except Exception:\n return None\n\n async def reload_jobs(self, tasks: List[Task]):\n \"\"\"重新加载所有定时任务\"\"\"\n print(\"正在重新加载定时任务...\")\n self.scheduler.remove_all_jobs()\n\n for task in tasks:\n if task.enabled and task.cron:\n try:\n trigger = build_cron_trigger(\n task.cron,\n timezone=self.scheduler.timezone,\n )\n self.scheduler.add_job(\n self._run_task,\n trigger=trigger,\n args=[task.id, task.task_name],\n id=f\"task_{task.id}\",\n name=f\"Scheduled: {task.task_name}\",\n replace_existing=True\n )\n print(f\" -> 已为任务 '{task.task_name}' 添加定时规则: '{task.cron}'\")\n except ValueError as e:\n print(f\" -> [警告] 任务 '{task.task_name}' 的 Cron 表达式无效: {e}\")\n\n print(\"定时任务加载完成\")\n\n async def _run_task(self, task_id: int, task_name: str):\n \"\"\"执行定时任务\"\"\"\n print(f\"定时任务触发: 正在为任务 '{task_name}' 启动爬虫...\")\n await self.process_service.start_task(task_id, task_name)\n" }, { "path": "src/services/search_pagination.py", "content": "import asyncio\nfrom dataclasses import dataclass\nfrom typing import Any, Awaitable, Callable, Optional\n\nfrom playwright.async_api import TimeoutError as PlaywrightTimeoutError\n\nfrom src.utils import log_time, random_sleep\n\nNEXT_PAGE_SELECTOR = (\n \"button[class*='search-pagination-arrow-container']\"\n \":has([class*='search-pagination-arrow-right'])\"\n \":not([disabled])\"\n)\nSEARCH_RESULTS_API_FRAGMENT = \"/h5/mtop.taobao.idlemtopsearch.pc.search/1.0/\"\nPAGE_REQUEST_TIMEOUT_MS = 20_000\nPAGE_CLICK_TIMEOUT_MS = 10_000\nPAGE_RETRY_DELAY_SECONDS = 5\nPAGE_RETRY_COUNT = 2\nPAGE_CLICK_SLEEP_MIN_SECONDS = 2\nPAGE_CLICK_SLEEP_MAX_SECONDS = 5\n\n\n@dataclass(frozen=True)\nclass PageAdvanceResult:\n advanced: bool\n response: Optional[Any] = None\n stop_reason: Optional[str] = None\n\n\ndef is_search_results_response(\n response: Any,\n api_url_fragment: str = SEARCH_RESULTS_API_FRAGMENT,\n) -> bool:\n request = getattr(response, \"request\", None)\n request_method = getattr(request, \"method\", None)\n response_url = getattr(response, \"url\", \"\")\n return api_url_fragment in response_url and request_method == \"POST\"\n\n\nasync def advance_search_page(\n *,\n page: Any,\n page_num: int,\n logger: Callable[[str], None] = log_time,\n wait_after_click: Callable[[float, float], Awaitable[None]] = random_sleep,\n retry_sleep: Callable[[float], Awaitable[None]] = asyncio.sleep,\n max_retries: int = PAGE_RETRY_COUNT,\n) -> PageAdvanceResult:\n next_button = page.locator(NEXT_PAGE_SELECTOR).first\n if not await next_button.count():\n logger(\"已到达最后一页，未找到可用的'下一页'按钮，停止翻页。\")\n return PageAdvanceResult(advanced=False, stop_reason=\"no_next_button\")\n\n for retry_index in range(max_retries):\n try:\n await next_button.scroll_into_view_if_needed()\n async with page.expect_response(\n is_search_results_response,\n timeout=PAGE_REQUEST_TIMEOUT_MS,\n ) as response_info:\n try:\n await next_button.click(timeout=PAGE_CLICK_TIMEOUT_MS)\n except PlaywrightTimeoutError:\n logger(f\"第 {page_num} 页下一页按钮点击超时，停止翻页。\")\n return PageAdvanceResult(\n advanced=False,\n stop_reason=\"click_timeout\",\n )\n await wait_after_click(\n PAGE_CLICK_SLEEP_MIN_SECONDS,\n PAGE_CLICK_SLEEP_MAX_SECONDS,\n )\n return PageAdvanceResult(\n advanced=True,\n response=await response_info.value,\n )\n except PlaywrightTimeoutError:\n if retry_index < max_retries - 1:\n logger(\n f\"等待第 {page_num} 页搜索响应超时，\"\n f\"{PAGE_RETRY_DELAY_SECONDS}秒后重试...\"\n )\n await retry_sleep(PAGE_RETRY_DELAY_SECONDS)\n continue\n\n logger(f\"等待第 {page_num} 页搜索响应超时 {max_retries} 次，停止翻页。\")\n return PageAdvanceResult(advanced=False, stop_reason=\"response_timeout\")\n\n return PageAdvanceResult(advanced=False, stop_reason=\"unknown\")\n" }, { "path": "src/services/seller_profile_cache.py", "content": "\"\"\"\n卖家资料缓存服务\n\"\"\"\nimport asyncio\nimport copy\nimport time\nfrom dataclasses import dataclass\nfrom typing import Awaitable, Callable, Optional\n\n\nSellerProfileLoader = Callable[[str], Awaitable[dict]]\n\n\n@dataclass(frozen=True)\nclass _CacheEntry:\n value: dict\n expires_at: float\n\n\nclass SellerProfileCache:\n \"\"\"带 TTL 和并发合并的卖家资料缓存。\"\"\"\n\n def __init__(\n self,\n ttl_seconds: int = 1800,\n time_source: Optional[Callable[[], float]] = None,\n ) -> None:\n self._ttl_seconds = max(0, ttl_seconds)\n self._time_source = time_source or time.monotonic\n self._entries: dict[str, _CacheEntry] = {}\n self._inflight: dict[str, asyncio.Task] = {}\n self._lock = asyncio.Lock()\n\n def _now(self) -> float:\n return float(self._time_source())\n\n def _clone(self, value: dict) -> dict:\n return copy.deepcopy(value)\n\n def _get_entry_value(self, user_id: str) -> Optional[dict]:\n entry = self._entries.get(user_id)\n if entry is None:\n return None\n if entry.expires_at < self._now():\n self._entries.pop(user_id, None)\n return None\n return self._clone(entry.value)\n\n async def get_or_load(self, user_id: str, loader: SellerProfileLoader) -> dict:\n async with self._lock:\n cached_value = self._get_entry_value(user_id)\n if cached_value is not None:\n return cached_value\n task = self._inflight.get(user_id)\n if task is None:\n task = asyncio.create_task(self._load_and_store(user_id, loader))\n self._inflight[user_id] = task\n return self._clone(await task)\n\n async def _load_and_store(self, user_id: str, loader: SellerProfileLoader) -> dict:\n try:\n value = self._clone(await loader(user_id))\n expires_at = self._now() + self._ttl_seconds\n async with self._lock:\n self._entries[user_id] = _CacheEntry(value=value, expires_at=expires_at)\n return value\n finally:\n async with self._lock:\n self._inflight.pop(user_id, None)\n" }, { "path": "src/services/task_generation_runner.py", "content": "\"\"\"\n任务生成作业执行器\n\"\"\"\nimport os\n\nimport aiofiles\n\nfrom src.domain.models.task import TaskCreate, TaskGenerateRequest\nfrom src.prompt_utils import generate_criteria\nfrom src.services.scheduler_service import SchedulerService\nfrom src.services.task_generation_service import TaskGenerationService\nfrom src.services.task_service import TaskService\n\ndef build_criteria_filename(keyword: str) -> str:\n safe_keyword = \"\".join(\n char for char in keyword.lower().replace(\" \", \"_\")\n if char.isalnum() or char in \"_-\"\n ).rstrip()\n return f\"prompts/{safe_keyword}_criteria.txt\"\n\n\ndef build_task_create(req: TaskGenerateRequest, criteria_file: str) -> TaskCreate:\n return TaskCreate(\n task_name=req.task_name,\n enabled=True,\n keyword=req.keyword,\n description=req.description or \"\",\n analyze_images=req.analyze_images,\n max_pages=req.max_pages,\n personal_only=req.personal_only,\n min_price=req.min_price,\n max_price=req.max_price,\n cron=req.cron,\n ai_prompt_base_file=\"prompts/base_prompt.txt\",\n ai_prompt_criteria_file=criteria_file,\n account_state_file=req.account_state_file,\n account_strategy=req.account_strategy,\n free_shipping=req.free_shipping,\n new_publish_option=req.new_publish_option,\n region=req.region,\n decision_mode=req.decision_mode or \"ai\",\n keyword_rules=req.keyword_rules,\n )\n\n\nasync def save_generated_criteria(output_filename: str, generated_criteria: str) -> None:\n if not generated_criteria or not generated_criteria.strip():\n raise RuntimeError(\"AI 未能生成分析标准，返回内容为空。\")\n\n os.makedirs(\"prompts\", exist_ok=True)\n async with aiofiles.open(output_filename, \"w\", encoding=\"utf-8\") as file:\n await file.write(generated_criteria)\n\n\nasync def reload_scheduler(\n task_service: TaskService,\n scheduler_service: SchedulerService,\n) -> None:\n tasks = await task_service.get_all_tasks()\n await scheduler_service.reload_jobs(tasks)\n\n\nasync def advance_job(\n generation_service: TaskGenerationService,\n job_id: str,\n step_key: str,\n message: str,\n) -> None:\n await generation_service.advance(job_id, step_key, message)\n\n\nasync def run_ai_generation_job(\n *,\n job_id: str,\n req: TaskGenerateRequest,\n task_service: TaskService,\n scheduler_service: SchedulerService,\n generation_service: TaskGenerationService,\n) -> None:\n output_filename = build_criteria_filename(req.keyword)\n try:\n await advance_job(\n generation_service,\n job_id,\n \"prepare\",\n \"已接收请求，开始准备分析标准。\",\n )\n\n async def report_progress(step_key: str, message: str) -> None:\n await advance_job(generation_service, job_id, step_key, message)\n\n generated_criteria = await generate_criteria(\n user_description=req.description or \"\",\n reference_file_path=\"prompts/macbook_criteria.txt\",\n progress_callback=report_progress,\n )\n\n await advance_job(\n generation_service,\n job_id,\n \"persist\",\n f\"正在保存分析标准到 {output_filename}。\",\n )\n await save_generated_criteria(output_filename, generated_criteria)\n\n await advance_job(\n generation_service,\n job_id,\n \"task\",\n \"分析标准已生成，正在创建任务记录。\",\n )\n task = await task_service.create_task(build_task_create(req, output_filename))\n await reload_scheduler(task_service, scheduler_service)\n await generation_service.complete(job_id, task, f\"任务“{req.task_name}”创建完成。\")\n except Exception as exc:\n if os.path.exists(output_filename):\n os.remove(output_filename)\n await generation_service.fail(job_id, f\"AI 任务生成失败: {exc}\")\n" }, { "path": "src/services/task_generation_service.py", "content": "\"\"\"\n任务生成作业服务\n\"\"\"\nimport asyncio\nfrom copy import deepcopy\nimport threading\nfrom typing import Awaitable, Dict, Iterable, Optional\nfrom uuid import uuid4\n\nfrom src.domain.models.task import Task\nfrom src.domain.models.task_generation import TaskGenerationJob, TaskGenerationStep\n\nDEFAULT_GENERATION_STEPS: tuple[tuple[str, str], ...] = (\n (\"prepare\", \"接收创建请求\"),\n (\"reference\", \"读取参考文件\"),\n (\"prompt\", \"构建提示词\"),\n (\"llm\", \"调用 AI 生成标准\"),\n (\"persist\", \"保存分析标准\"),\n (\"task\", \"创建任务记录\"),\n)\n\n\nclass TaskGenerationService:\n \"\"\"管理 AI 任务生成的后台作业状态\"\"\"\n\n def __init__(self, step_specs: Iterable[tuple[str, str]] = DEFAULT_GENERATION_STEPS):\n self._step_specs = tuple(step_specs)\n self._jobs: Dict[str, TaskGenerationJob] = {}\n self._lock = threading.Lock()\n self._workers: set[threading.Thread] = set()\n\n async def create_job(self, task_name: str) -> TaskGenerationJob:\n job = TaskGenerationJob(\n job_id=uuid4().hex,\n task_name=task_name,\n steps=[\n TaskGenerationStep(key=key, label=label)\n for key, label in self._step_specs\n ],\n )\n with self._lock:\n self._jobs[job.job_id] = job\n return deepcopy(job)\n\n async def get_job(self, job_id: str) -> Optional[TaskGenerationJob]:\n with self._lock:\n job = self._jobs.get(job_id)\n if not job:\n return None\n return deepcopy(job)\n\n def track(self, coroutine: Awaitable[None]) -> None:\n thread: Optional[threading.Thread] = None\n\n def runner() -> None:\n try:\n asyncio.run(coroutine)\n finally:\n if thread is None:\n return\n with self._lock:\n self._workers.discard(thread)\n\n thread = threading.Thread(target=runner, daemon=True)\n with self._lock:\n self._workers.add(thread)\n thread.start()\n\n async def advance(self, job_id: str, step_key: str, message: str) -> TaskGenerationJob:\n with self._lock:\n job = self._require_job(job_id)\n target_index = self._find_step_index(job, step_key)\n job.status = \"running\"\n job.current_step = step_key\n job.message = message\n for index, step in enumerate(job.steps):\n if step.status == \"failed\":\n continue\n if index < target_index:\n step.status = \"completed\"\n elif index == target_index:\n step.status = \"running\"\n step.message = message\n elif step.status != \"pending\":\n step.status = \"pending\"\n step.message = \"\"\n return deepcopy(job)\n\n async def complete(self, job_id: str, task: Task, message: str) -> TaskGenerationJob:\n with self._lock:\n job = self._require_job(job_id)\n job.status = \"completed\"\n job.current_step = None\n job.message = message\n job.error = None\n job.task = task\n for step in job.steps:\n if step.status != \"failed\":\n step.status = \"completed\"\n return deepcopy(job)\n\n async def fail(\n self,\n job_id: str,\n error: str,\n step_key: Optional[str] = None,\n ) -> TaskGenerationJob:\n with self._lock:\n job = self._require_job(job_id)\n failed_step = step_key or job.current_step\n job.status = \"failed\"\n job.error = error\n job.message = error\n job.current_step = failed_step\n if failed_step:\n step = self._find_step(job, failed_step)\n if step:\n step.status = \"failed\"\n step.message = error\n return deepcopy(job)\n\n def _require_job(self, job_id: str) -> TaskGenerationJob:\n job = self._jobs.get(job_id)\n if not job:\n raise KeyError(f\"任务生成作业不存在: {job_id}\")\n return job\n\n def _find_step(self, job: TaskGenerationJob, step_key: str) -> Optional[TaskGenerationStep]:\n for step in job.steps:\n if step.key == step_key:\n return step\n return None\n\n def _find_step_index(self, job: TaskGenerationJob, step_key: str) -> int:\n for index, step in enumerate(job.steps):\n if step.key == step_key:\n return index\n raise KeyError(f\"未知的任务生成步骤: {step_key}\")\n" }, { "path": "src/services/task_log_cleanup_service.py", "content": "\"\"\"\n任务运行日志清理服务。\n\"\"\"\nfrom __future__ import annotations\n\nfrom datetime import datetime, timedelta\nfrom pathlib import Path\n\n\ndef cleanup_task_logs(\n logs_dir: str = \"logs\",\n *,\n keep_days: int = 7,\n now: datetime | None = None,\n) -> list[str]:\n if keep_days < 1:\n print(f\"任务日志清理已跳过：保留天数配置无效 ({keep_days})\")\n return []\n\n root = Path(logs_dir)\n if not root.exists():\n return []\n\n current_time = now or datetime.now()\n cutoff = current_time - timedelta(days=keep_days)\n removed_files: list[str] = []\n\n for path in root.glob(\"*.log\"):\n if not path.is_file():\n continue\n try:\n modified_at = datetime.fromtimestamp(path.stat().st_mtime)\n except OSError as exc:\n print(f\"读取任务日志时间失败，已跳过: {path} ({exc})\")\n continue\n\n if modified_at >= cutoff:\n continue\n\n try:\n path.unlink()\n removed_files.append(str(path))\n except OSError as exc:\n print(f\"删除历史任务日志失败，已跳过: {path} ({exc})\")\n\n if removed_files:\n print(\n f\"任务日志清理完成：已删除 {len(removed_files)} 个超过 {keep_days} 天的历史日志文件。\"\n )\n\n return removed_files\n" }, { "path": "src/services/task_payloads.py", "content": "\"\"\"\n任务接口响应序列化辅助。\n\"\"\"\nfrom __future__ import annotations\n\nfrom datetime import datetime\nfrom typing import Any\n\nfrom src.domain.models.task import Task\n\n\ndef serialize_timestamp(value: datetime | None) -> str | None:\n return value.isoformat() if value else None\n\n\ndef serialize_task(task: Task, scheduler_service) -> dict[str, Any]:\n payload = task.model_dump()\n next_run_time = None\n if task.id is not None and scheduler_service is not None:\n next_run_time = scheduler_service.get_next_run_time(task.id)\n payload[\"next_run_at\"] = serialize_timestamp(next_run_time)\n return payload\n\n\ndef serialize_tasks(tasks: list[Task], scheduler_service) -> list[dict[str, Any]]:\n return [serialize_task(task, scheduler_service) for task in tasks]\n" }, { "path": "src/services/task_service.py", "content": "\"\"\"\n任务管理服务\n封装任务相关的业务逻辑\n\"\"\"\nfrom typing import List, Optional\nfrom src.domain.models.task import Task, TaskCreate, TaskUpdate\nfrom src.domain.repositories.task_repository import TaskRepository\n\n\nclass TaskService:\n \"\"\"任务管理服务\"\"\"\n\n def __init__(self, repository: TaskRepository):\n self.repository = repository\n\n async def get_all_tasks(self) -> List[Task]:\n \"\"\"获取所有任务\"\"\"\n return await self.repository.find_all()\n\n async def get_task(self, task_id: int) -> Optional[Task]:\n \"\"\"获取单个任务\"\"\"\n return await self.repository.find_by_id(task_id)\n\n async def create_task(self, task_create: TaskCreate) -> Task:\n \"\"\"创建新任务\"\"\"\n task = Task(**task_create.model_dump(), is_running=False)\n return await self.repository.save(task)\n\n async def update_task(self, task_id: int, task_update: TaskUpdate) -> Task:\n \"\"\"更新任务\"\"\"\n task = await self.repository.find_by_id(task_id)\n if not task:\n raise ValueError(f\"任务 {task_id} 不存在\")\n\n updated_task = task.apply_update(task_update)\n return await self.repository.save(updated_task)\n\n async def delete_task(self, task_id: int) -> bool:\n \"\"\"删除任务\"\"\"\n return await self.repository.delete(task_id)\n\n async def update_task_status(self, task_id: int, is_running: bool) -> Task:\n \"\"\"更新任务运行状态\"\"\"\n task_update = TaskUpdate(is_running=is_running)\n return await self.update_task(task_id, task_update)\n" }, { "path": "src/utils.py", "content": "import asyncio\nimport json\nimport math\nimport os\nimport random\nimport re\nimport glob\nfrom datetime import datetime\nfrom functools import wraps\nfrom urllib.parse import quote\n\nfrom openai import APIStatusError\nfrom requests.exceptions import HTTPError\n\nfrom src.services.result_storage_service import save_result_record\n\n\ndef retry_on_failure(retries=3, delay=5):\n \"\"\"\n 一个通用的异步重试装饰器，增加了对HTTP错误的详细日志记录。\n \"\"\"\n def decorator(func):\n @wraps(func)\n async def wrapper(*args, **kwargs):\n for i in range(retries):\n try:\n return await func(*args, **kwargs)\n except (APIStatusError, HTTPError) as e:\n print(f\"函数 {func.__name__} 第 {i + 1}/{retries} 次尝试失败，发生HTTP错误。\")\n if hasattr(e, 'status_code'):\n print(f\" - 状态码 (Status Code): {e.status_code}\")\n if hasattr(e, 'response') and hasattr(e.response, 'text'):\n response_text = e.response.text\n print(\n f\" - 返回值 (Response): {response_text[:300]}{'...' if len(response_text) > 300 else ''}\")\n except json.JSONDecodeError as e:\n print(f\"函数 {func.__name__} 第 {i + 1}/{retries} 次尝试失败: JSON解析错误 - {e}\")\n except Exception as e:\n print(f\"函数 {func.__name__} 第 {i + 1}/{retries} 次尝试失败: {type(e).__name__} - {e}\")\n\n if i < retries - 1:\n print(f\"将在 {delay} 秒后重试...\")\n await asyncio.sleep(delay)\n\n print(f\"函数 {func.__name__} 在 {retries} 次尝试后彻底失败。\")\n return None\n return wrapper\n return decorator\n\n\nasync def safe_get(data, *keys, default=\"暂无\"):\n \"\"\"安全获取嵌套字典值\"\"\"\n for key in keys:\n try:\n data = data[key]\n except (KeyError, TypeError, IndexError):\n return default\n return data\n\n\nasync def random_sleep(min_seconds: float, max_seconds: float):\n \"\"\"异步等待一个在指定范围内的随机时间。\"\"\"\n delay = random.uniform(min_seconds, max_seconds)\n print(f\" [延迟] 等待 {delay:.2f} 秒... (范围: {min_seconds}-{max_seconds}s)\")\n await asyncio.sleep(delay)\n\n\ndef log_time(message: str, prefix: str = \"\") -> None:\n \"\"\"在日志前加上 YY-MM-DD HH:MM:SS 时间戳的简单打印。\"\"\"\n try:\n ts = datetime.now().strftime(' %Y-%m-%d %H:%M:%S')\n except Exception:\n ts = \"--:--:--\"\n print(f\"[{ts}] {prefix}{message}\")\n\n\ndef sanitize_filename(value: str) -> str:\n \"\"\"生成安全的文件名片段。\"\"\"\n if not value:\n return \"task\"\n cleaned = re.sub(r\"[^a-zA-Z0-9_-]+\", \"_\", value.strip())\n cleaned = re.sub(r\"_+\", \"_\", cleaned).strip(\"_\")\n return cleaned or \"task\"\n\n\ndef build_task_log_path(task_id: int, task_name: str) -> str:\n \"\"\"生成任务日志路径（包含任务名）。\"\"\"\n safe_name = sanitize_filename(task_name)\n filename = f\"{safe_name}_{task_id}.log\"\n return os.path.join(\"logs\", filename)\n\n\ndef resolve_task_log_path(task_id: int, task_name: str) -> str:\n \"\"\"优先使用任务名生成日志路径，不存在时回退为按 ID 匹配。\"\"\"\n primary_path = build_task_log_path(task_id, task_name)\n if os.path.exists(primary_path):\n return primary_path\n pattern = os.path.join(\"logs\", f\"*_{task_id}.log\")\n matches = glob.glob(pattern)\n if matches:\n return matches[0]\n return primary_path\n\n\ndef convert_goofish_link(url: str) -> str:\n \"\"\"\n 将Goofish商品链接转换为只包含商品ID的手机端格式。\n \"\"\"\n match_first_link = re.search(r'item\\?id=(\\d+)', url)\n if match_first_link:\n item_id = match_first_link.group(1)\n bfp_json = f'{{\"id\":{item_id}}}'\n return f\"https://pages.goofish.com/sharexy?loadingVisible=false&bft=item&bfs=idlepc.item&spm=a21ybx.item.0.0&bfp={quote(bfp_json)}\"\n return url\n\n\ndef get_link_unique_key(link: str) -> str:\n \"\"\"截取链接中第一个\"&\"之前的内容作为唯一标识依据。\"\"\"\n return link.split('&', 1)[0]\n\n\nasync def save_to_jsonl(data_record: dict, keyword: str):\n \"\"\"兼容旧调用名，实际将结果写入 SQLite。\"\"\"\n try:\n return await save_result_record(data_record, keyword)\n except Exception as e:\n print(f\"写入 SQLite 结果记录出错: {e}\")\n return False\n\n\ndef format_registration_days(total_days: int) -> str:\n \"\"\"\n 将总天数格式化为“X年Y个月”的字符串。\n \"\"\"\n if not isinstance(total_days, int) or total_days <= 0:\n return '未知'\n\n DAYS_IN_YEAR = 365.25\n DAYS_IN_MONTH = DAYS_IN_YEAR / 12\n\n years = math.floor(total_days / DAYS_IN_YEAR)\n remaining_days = total_days - (years * DAYS_IN_YEAR)\n months = round(remaining_days / DAYS_IN_MONTH)\n\n if months == 12:\n years += 1\n months = 0\n\n if years > 0 and months > 0:\n return f\"来闲鱼{years}年{months}个月\"\n elif years > 0 and months == 0:\n return f\"来闲鱼{years}年整\"\n elif years == 0 and months > 0:\n return f\"来闲鱼{months}个月\"\n else:\n return \"来闲鱼不足一个月\"\n" }, { "path": "start.sh", "content": "#!/bin/bash\n\n# 闲鱼监控系统本地启动脚本\n# 功能：清理旧构建、安装依赖、构建前端、启动服务\n\nset -e # 遇到错误立即退出\n\n# 颜色输出\nRED='\\033[0;31m'\nGREEN='\\033[0;32m'\nYELLOW='\\033[1;33m'\nNC='\\033[0m' # No Color\n\n# 获取脚本所在目录\nSCRIPT_DIR=\"$(cd \"$(dirname \"${BASH_SOURCE[0]}\")\" && pwd)\"\ncd \"$SCRIPT_DIR\"\n\necho -e \"${GREEN}========================================${NC}\"\necho -e \"${GREEN}闲鱼监控系统 - 本地启动脚本${NC}\"\necho -e \"${GREEN}========================================${NC}\"\n\n# 0. 环境与依赖检查\necho -e \"\\n${YELLOW}[1/6] 检查环境与依赖...${NC}\"\n\nOS_FAMILY=\"unknown\"\nLINUX_ID=\"\"\nLINUX_LIKE=\"\"\nPYTHON_CMD=\"python3\"\nPIP_CMD=\"python3 -m pip\"\n\nif [ -f /etc/os-release ]; then\n . /etc/os-release\n LINUX_ID=\"$ID\"\n LINUX_LIKE=\"$ID_LIKE\"\nfi\n\ncase \"$(uname -s 2>/dev/null || echo unknown)\" in\n Darwin)\n OS_FAMILY=\"macos\"\n ;;\n Linux)\n if grep -qi microsoft /proc/version 2>/dev/null; then\n OS_FAMILY=\"wsl\"\n else\n OS_FAMILY=\"linux\"\n fi\n ;;\n MINGW*|MSYS*|CYGWIN*)\n OS_FAMILY=\"windows\"\n ;;\n *)\n OS_FAMILY=\"unknown\"\n ;;\nesac\n\nMISSING_ITEMS=()\n\nif ! command -v python3 >/dev/null 2>&1; then\n MISSING_ITEMS+=(\"python3(>=3.10)\")\nelse\n if ! python3 -c 'import sys; raise SystemExit(0 if sys.version_info >= (3, 10) else 1)' >/dev/null 2>&1; then\n MISSING_ITEMS+=(\"python3(>=3.10)\")\n fi\nfi\n\nif ! python3 -m pip --version >/dev/null 2>&1; then\n MISSING_ITEMS+=(\"pip\")\nfi\n\nif ! command -v node >/dev/null 2>&1; then\n MISSING_ITEMS+=(\"node\")\nfi\n\nif ! command -v npm >/dev/null 2>&1; then\n MISSING_ITEMS+=(\"npm\")\nfi\n\nif ! python3 -m playwright --version >/dev/null 2>&1; then\n MISSING_ITEMS+=(\"playwright\")\nfi\n\nhas_browser=false\ncase \"$OS_FAMILY\" in\n macos)\n if [ -d \"/Applications/Google Chrome.app\" ] || [ -d \"/Applications/Microsoft Edge.app\" ]; then\n has_browser=true\n fi\n ;;\n linux|wsl)\n if command -v google-chrome >/dev/null 2>&1 \\\n || command -v google-chrome-stable >/dev/null 2>&1 \\\n || command -v chromium >/dev/null 2>&1 \\\n || command -v chromium-browser >/dev/null 2>&1 \\\n || command -v microsoft-edge >/dev/null 2>&1 \\\n || command -v microsoft-edge-stable >/dev/null 2>&1; then\n has_browser=true\n fi\n ;;\n windows)\n if [ -d \"/c/Program Files/Google/Chrome/Application\" ] \\\n || [ -d \"/c/Program Files (x86)/Google/Chrome/Application\" ] \\\n || [ -d \"/c/Program Files (x86)/Microsoft/Edge/Application\" ] \\\n || [ -d \"/c/Program Files/Microsoft/Edge/Application\" ]; then\n has_browser=true\n fi\n ;;\nesac\n\nif [ \"$has_browser\" = false ]; then\n MISSING_ITEMS+=(\"浏览器(Chrome 或 Edge)\")\nfi\n\n\nprint_solution_macos() {\n cat <<'EOF'\nmacOS 解决办法:\n1) 安装 Homebrew:\n /bin/bash -c \"$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)\"\n2) 安装 Python 与 Node:\n brew install python@3.11 node\n3) 安装 Playwright:\n python3 -m pip install playwright\n python3 -m playwright install chromium\n4) 安装浏览器:\n brew install --cask google-chrome\n # 或\n brew install --cask microsoft-edge\n5) 配置文件（可选）:\n cp .env.example .env\n cp config.json.example config.json\nEOF\n}\n\nprint_solution_linux_deb() {\n cat <<'EOF'\nLinux (Debian/Ubuntu) 解决办法:\n1) 安装 Python 与 pip:\n sudo apt-get update\n sudo apt-get install -y python3 python3-pip python3-venv\n2) 安装 Node.js 与 npm (LTS):\n curl -fsSL https://deb.nodesource.com/setup_lts.x | sudo -E bash -\n sudo apt-get install -y nodejs\n3) 安装 Playwright:\n python3 -m pip install playwright\n python3 -m playwright install chromium\n python3 -m playwright install-deps chromium\n4) 安装浏览器:\n sudo apt-get install -y chromium-browser || sudo apt-get install -y chromium\n # 或安装 Edge:\n sudo apt-get install -y microsoft-edge-stable\n5) 配置文件（可选）:\n cp .env.example .env\n cp config.json.example config.json\nEOF\n}\n\nprint_solution_linux_rpm() {\n cat <<'EOF'\nLinux (RHEL/CentOS/Fedora) 解决办法:\n1) 安装 Python 与 pip:\n sudo dnf install -y python3 python3-pip\n2) 安装 Node.js 与 npm (LTS):\n sudo dnf install -y nodejs\n3) 安装 Playwright:\n python3 -m pip install playwright\n python3 -m playwright install chromium\n python3 -m playwright install-deps chromium\n4) 安装浏览器:\n sudo dnf install -y chromium\n # 或安装 Edge:\n sudo dnf install -y microsoft-edge-stable\n5) 配置文件（可选）:\n cp .env.example .env\n cp config.json.example config.json\nEOF\n}\n\nprint_solution_linux_arch() {\n cat <<'EOF'\nLinux (Arch) 解决办法:\n1) 安装 Python 与 pip:\n sudo pacman -S --noconfirm python python-pip\n2) 安装 Node.js 与 npm:\n sudo pacman -S --noconfirm nodejs npm\n3) 安装 Playwright:\n python3 -m pip install playwright\n python3 -m playwright install chromium\n python3 -m playwright install-deps chromium\n4) 安装浏览器:\n sudo pacman -S --noconfirm chromium\n # 或安装 Edge:\n yay -S microsoft-edge-stable\n5) 配置文件:\n cp .env.example .env\n cp config.json.example config.json\nEOF\n}\n\nprint_solution_wsl() {\n cat <<'EOF'\nWSL 解决办法:\n1) 安装 Python 与 pip:\n sudo apt-get update\n sudo apt-get install -y python3 python3-pip python3-venv\n2) 安装 Node.js 与 npm (LTS):\n curl -fsSL https://deb.nodesource.com/setup_lts.x | sudo -E bash -\n sudo apt-get install -y nodejs\n3) 安装 Playwright:\n python3 -m pip install playwright\n python3 -m playwright install chromium\n python3 -m playwright install-deps chromium\n4) 安装浏览器:\n sudo apt-get install -y chromium-browser || sudo apt-get install -y chromium\n # 或在 Windows 安装 Chrome/Edge 并在 WSL 使用 Linux 版本浏览器\n5) 配置文件:\n cp .env.example .env\n cp config.json.example config.json\nEOF\n}\n\nprint_solution_windows() {\n cat <<'EOF'\nWindows (PowerShell) 解决办法:\n1) 安装 Python 与 Node:\n winget install Python.Python.3.11\n winget install OpenJS.NodeJS.LTS\n2) 安装 Playwright:\n py -m pip install playwright\n py -m playwright install chromium\n3) 安装浏览器:\n winget install Google.Chrome\n # 或\n winget install Microsoft.Edge\n4) 配置文件（可选）:\n Copy-Item .env.example .env\n Copy-Item config.json.example config.json\nEOF\n}\n\nprint_solution_generic() {\n cat <<'EOF'\n通用解决办法:\n1) 安装 Python 3.10+ 与 pip\n2) 安装 Node.js 与 npm\n3) 安装 Playwright:\n python3 -m pip install playwright\n python3 -m playwright install chromium\n4) 安装浏览器 Chrome 或 Edge\n5) 配置文件（可选）:\n cp .env.example .env\n cp config.json.example config.json\nEOF\n}\n\nif [ \"${#MISSING_ITEMS[@]}\" -ne 0 ]; then\n echo -e \"${RED}✗ 检测到缺失的环境/依赖:${NC}\"\n for item in \"${MISSING_ITEMS[@]}\"; do\n echo \" - $item\"\n done\n echo \"\"\n case \"$OS_FAMILY\" in\n macos)\n print_solution_macos\n ;;\n linux)\n if [ \"$LINUX_ID\" = \"arch\" ] || echo \"$LINUX_LIKE\" | grep -qi \"arch\"; then\n print_solution_linux_arch\n elif [ \"$LINUX_ID\" = \"fedora\" ] || [ \"$LINUX_ID\" = \"rhel\" ] || [ \"$LINUX_ID\" = \"centos\" ] || echo \"$LINUX_LIKE\" | grep -qi \"rhel\\|fedora\"; then\n print_solution_linux_rpm\n else\n print_solution_linux_deb\n fi\n ;;\n wsl)\n print_solution_wsl\n ;;\n windows)\n print_solution_windows\n ;;\n *)\n print_solution_generic\n ;;\n esac\n exit 1\nfi\n\necho -e \"${GREEN}✓ 环境与依赖检查通过${NC}\"\n\n# 1. 清理旧的 dist 目录\necho -e \"\\n${YELLOW}[2/6] 清理旧的构建产物...${NC}\"\nif [ -d \"dist\" ]; then\n rm -rf dist\n echo -e \"${GREEN}✓ 已删除旧的 dist 目录${NC}\"\nelse\n echo -e \"${GREEN}✓ dist 目录不存在，跳过清理${NC}\"\nfi\n\n# 2. 检查并安装 Python 依赖\necho -e \"\\n${YELLOW}[3/6] 检查 Python 依赖...${NC}\"\nif [ ! -f \"requirements.txt\" ]; then\n echo -e \"${RED}✗ 错误: requirements.txt 文件不存在${NC}\"\n exit 1\nfi\n\necho \"正在安装 Python 依赖...\"\npython3 -m pip install -r requirements.txt --quiet\necho -e \"${GREEN}✓ Python 依赖安装完成${NC}\"\n\n# 3. 构建前端\necho -e \"\\n${YELLOW}[4/6] 构建前端项目...${NC}\"\nif [ ! -d \"web-ui\" ]; then\n echo -e \"${RED}✗ 错误: web-ui 目录不存在${NC}\"\n exit 1\nfi\n\ncd web-ui\n\n# 检查 node_modules 是否存在\nif [ ! -d \"node_modules\" ]; then\n echo \"首次运行，正在安装前端依赖...\"\n npm install\nfi\n\necho \"正在构建前端...\"\nnpm run build\n\ncd \"$SCRIPT_DIR\"\n\nif [ ! -d \"dist\" ]; then\n echo -e \"${RED}✗ 错误: 前端构建失败，dist 目录未生成${NC}\"\n exit 1\nfi\n\necho -e \"${GREEN}✓ 前端构建完成，产物已输出到项目根目录 dist/${NC}\"\n\n# 4. 校验构建产物\necho -e \"\\n${YELLOW}[5/6] 校验构建产物...${NC}\"\necho -e \"${GREEN}✓ 已确认构建产物位于项目根目录 dist/${NC}\"\n\n# 5. 启动后端服务\necho -e \"\\n${YELLOW}[6/6] 启动后端服务...${NC}\"\necho -e \"${GREEN}========================================${NC}\"\necho -e \"${GREEN}服务启动中...${NC}\"\necho -e \"${GREEN}访问地址: http://localhost:8000${NC}\"\necho -e \"${GREEN}API 文档: http://localhost:8000/docs${NC}\"\necho -e \"${GREEN}========================================${NC}\\n\"\n\npython3 -m src.app\n" }, { "path": "tests/README.md", "content": "# 测试指南\n\n本项目使用 pytest 作为测试框架。以下是运行测试的指南。\n\n## 安装依赖\n\n在运行测试之前，请确保已安装所有开发依赖项：\n\n```bash\npip install -r requirements.txt\n```\n\n## 运行测试\n\n### 运行所有测试\n\n```bash\npytest\n```\n\n### 运行特定测试文件\n\n```bash\npytest tests/integration/test_api_tasks.py\n```\n\n### 运行特定测试函数\n\n```bash\npytest tests/unit/test_utils.py::test_safe_get_nested_and_default\n```\n\n### 生成覆盖率报告\n\n```bash\ncoverage run -m pytest\ncoverage report\ncoverage html # 生成 HTML 报告\n```\n\n## 测试文件结构\n\n```\ntests/\n├── __init__.py\n├── conftest.py # 共享 fixtures（API/CLI/样例数据）\n├── fixtures/ # 贴近真实的样例数据（搜索/用户/评价/任务配置）\n│ ├── config.sample.json\n│ ├── ratings.json\n│ ├── search_results.json\n│ ├── state.sample.json\n│ ├── user_head.json\n│ └── user_items.json\n├── integration/ # 关键链路集成测试（API/CLI/解析器）\n│ ├── test_api_tasks.py\n│ ├── test_cli_spider.py\n│ └── test_pipeline_parse.py\n└── unit/ # 核心纯函数单元测试\n ├── test_domain_task.py\n └── test_utils.py\n```\n\n## 编写新测试\n\n1. 新增测试放在 `tests/integration/` 或 `tests/unit/`\n2. 文件名以 `test_` 开头，函数名以 `test_` 开头\n3. 测试采用同步执行（不依赖 pytest-asyncio）\n4. 外部依赖（Playwright/AI/通知/网络）统一 mock\n5. 使用 `tests/fixtures/` 的样例数据，避免依赖真实网络\n\n## 注意事项\n\n1. 目标是离线可跑且稳定复现\n2. 集成测试优先覆盖真实运行链路（API、CLI、解析器）\n3. 如需新增真实场景样例，统一补充到 `tests/fixtures/`\n\n## Live smoke（真实冒烟测试）\n\n- 目录：`tests/live/`\n- 默认关闭；仅当显式设置 `RUN_LIVE_TESTS=1` 时才会执行\n- 推荐命令：\n\n```bash\nRUN_LIVE_TESTS=1 \\\nLIVE_TEST_ACCOUNT_STATE_FILE=/absolute/path/to/account.json \\\nLIVE_TEST_KEYWORD=\"MacBook Pro M2\" \\\npytest tests/live -m live -v\n```\n\n- 一键脚本：\n\n```bash\n./run_live_smoke.sh\n./run_live_smoke.sh --without-generation\n```\n\n- 可选环境变量：\n - `LIVE_TEST_TASK_NAME`\n - `LIVE_EXPECT_MIN_ITEMS`（默认 `1`）\n - `LIVE_TEST_DEBUG_LIMIT`（默认 `1`，只抓取/分析前 N 个新商品）\n - `LIVE_TIMEOUT_SECONDS`（默认 `180`）\n - `LIVE_ENABLE_TASK_GENERATION`（脚本默认 `1`；设为 `0` 或使用 `--without-generation` 可关闭真实 AI 任务生成慢用例）\n- live 套件会在临时工作目录中启动真实 `uvicorn`，并清空通知相关 env，避免污染仓库根目录或向真实通知通道发消息。\n" }, { "path": "tests/__init__.py", "content": "" }, { "path": "tests/conftest.py", "content": "import json\nimport os\nimport sys\nfrom datetime import datetime, timedelta\nfrom pathlib import Path\nfrom zoneinfo import ZoneInfo\n\nimport pytest\nfrom fastapi import FastAPI\nfrom fastapi.testclient import TestClient\n\n\n# Add repository root to the path so package imports work consistently\nrepo_root = Path(__file__).resolve().parents[1]\nsys.path.insert(0, str(repo_root))\n\nfrom src.api import dependencies as deps\nfrom src.api.routes import tasks\nfrom src.infrastructure.persistence.sqlite_task_repository import SqliteTaskRepository\nfrom src.services.task_service import TaskService\nfrom src.services.task_generation_service import TaskGenerationService\n\n\n@pytest.fixture()\ndef fixtures_dir() -> Path:\n return Path(__file__).parent / \"fixtures\"\n\n\n@pytest.fixture()\ndef load_json_fixture(fixtures_dir):\n def _load(name: str):\n return json.loads((fixtures_dir / name).read_text(encoding=\"utf-8\"))\n\n return _load\n\n\n@pytest.fixture()\ndef sample_task_payload():\n return {\n \"task_name\": \"Sony A7M4\",\n \"enabled\": True,\n \"keyword\": \"sony a7m4\",\n \"description\": \"Good condition body with accessories\",\n \"analyze_images\": True,\n \"max_pages\": 2,\n \"personal_only\": True,\n \"min_price\": \"8000\",\n \"max_price\": \"16000\",\n \"cron\": \"*/15 * * * *\",\n \"ai_prompt_base_file\": \"prompts/base_prompt.txt\",\n \"ai_prompt_criteria_file\": \"prompts/sony_a7m4_criteria.txt\",\n \"decision_mode\": \"ai\",\n \"keyword_rules\": [],\n }\n\n\nclass FakeProcessService:\n def __init__(self):\n self.started = []\n self.stopped = []\n self.reindexed = []\n self._on_started = None\n self._on_stopped = None\n\n def set_lifecycle_hooks(self, *, on_started=None, on_stopped=None):\n self._on_started = on_started\n self._on_stopped = on_stopped\n\n async def start_task(self, task_id: int, task_name: str) -> bool:\n self.started.append((task_id, task_name))\n if self._on_started:\n await self._on_started(task_id)\n return True\n\n async def stop_task(self, task_id: int):\n self.stopped.append(task_id)\n if self._on_stopped:\n await self._on_stopped(task_id)\n\n def reindex_after_delete(self, deleted_task_id: int):\n self.reindexed.append(deleted_task_id)\n\n\nclass FakeSchedulerService:\n def __init__(self):\n self.reload_calls = 0\n self.next_run_times = {}\n\n async def reload_jobs(self, tasks):\n self.reload_calls += 1\n base = datetime(2026, 3, 19, 8, 0, tzinfo=ZoneInfo(\"Asia/Shanghai\"))\n self.next_run_times = {\n task.id: base + timedelta(minutes=(index + 1) * 15)\n for index, task in enumerate(tasks)\n if task.id is not None and task.enabled and task.cron\n }\n\n def get_next_run_time(self, task_id: int):\n return self.next_run_times.get(task_id)\n\n\n@pytest.fixture()\ndef api_context(tmp_path):\n config_file = tmp_path / \"config.json\"\n config_file.write_text(\"[]\", encoding=\"utf-8\")\n db_path = tmp_path / \"app.sqlite3\"\n\n repository = SqliteTaskRepository(\n db_path=str(db_path),\n legacy_config_file=None,\n )\n task_service = TaskService(repository)\n process_service = FakeProcessService()\n scheduler_service = FakeSchedulerService()\n task_generation_service = TaskGenerationService()\n\n app = FastAPI()\n app.include_router(tasks.router)\n\n def override_get_task_service():\n return task_service\n\n def override_get_process_service():\n return process_service\n\n def override_get_scheduler_service():\n return scheduler_service\n\n def override_get_task_generation_service():\n return task_generation_service\n\n async def mark_started(task_id: int):\n await task_service.update_task_status(task_id, True)\n\n async def mark_stopped(task_id: int):\n task = await task_service.get_task(task_id)\n if task:\n await task_service.update_task_status(task_id, False)\n\n process_service.set_lifecycle_hooks(on_started=mark_started, on_stopped=mark_stopped)\n\n app.dependency_overrides[deps.get_task_service] = override_get_task_service\n app.dependency_overrides[deps.get_process_service] = override_get_process_service\n app.dependency_overrides[deps.get_scheduler_service] = override_get_scheduler_service\n app.dependency_overrides[deps.get_task_generation_service] = override_get_task_generation_service\n\n return {\n \"app\": app,\n \"config_file\": config_file,\n \"db_path\": db_path,\n \"process_service\": process_service,\n \"scheduler_service\": scheduler_service,\n \"task_generation_service\": task_generation_service,\n }\n\n\n@pytest.fixture()\ndef api_client(api_context):\n return TestClient(api_context[\"app\"])\n" }, { "path": "tests/fixtures/config.sample.json", "content": "[\n {\n \"task_name\": \"Sony A7M4\",\n \"enabled\": true,\n \"keyword\": \"sony a7m4\",\n \"description\": \"body only\",\n \"max_pages\": 2,\n \"personal_only\": true,\n \"min_price\": \"8000\",\n \"max_price\": \"16000\",\n \"cron\": \"*/15 * * * *\",\n \"ai_prompt_base_file\": \"prompts/base_prompt.txt\",\n \"ai_prompt_criteria_file\": \"prompts/sony_a7m4_criteria.txt\",\n \"decision_mode\": \"ai\",\n \"keyword_rules\": []\n },\n {\n \"task_name\": \"Canon R6\",\n \"enabled\": false,\n \"keyword\": \"canon r6\",\n \"description\": \"body only\",\n \"max_pages\": 1,\n \"personal_only\": true,\n \"min_price\": \"5000\",\n \"max_price\": \"12000\",\n \"cron\": \"*/30 * * * *\",\n \"ai_prompt_base_file\": \"prompts/base_prompt.txt\",\n \"ai_prompt_criteria_file\": \"prompts/canon_r6_criteria.txt\",\n \"decision_mode\": \"ai\",\n \"keyword_rules\": []\n }\n]\n" }, { "path": "tests/fixtures/ratings.json", "content": "[\n {\n \"cardData\": {\n \"rateId\": \"r1\",\n \"feedback\": \"Great seller\",\n \"rate\": 1,\n \"rateTagList\": [{\"text\": \"\\u5356\\u5bb6\"}],\n \"raterUserNick\": \"buyer_01\",\n \"gmtCreate\": \"2024-01-01\",\n \"pictCdnUrlList\": []\n }\n },\n {\n \"cardData\": {\n \"rateId\": \"r2\",\n \"feedback\": \"Nice buyer\",\n \"rate\": 1,\n \"rateTagList\": [{\"text\": \"\\u4e70\\u5bb6\"}],\n \"raterUserNick\": \"seller_02\",\n \"gmtCreate\": \"2024-02-01\",\n \"pictCdnUrlList\": [\"https://img.example.com/pic.jpg\"]\n }\n },\n {\n \"cardData\": {\n \"rateId\": \"r3\",\n \"feedback\": \"Average\",\n \"rate\": 0,\n \"rateTagList\": [{\"text\": \"\\u5356\\u5bb6\"}],\n \"raterUserNick\": \"buyer_02\",\n \"gmtCreate\": \"2024-03-01\",\n \"pictCdnUrlList\": []\n }\n }\n]\n" }, { "path": "tests/fixtures/search_results.json", "content": "{\n \"data\": {\n \"resultList\": [\n {\n \"data\": {\n \"item\": {\n \"main\": {\n \"exContent\": {\n \"title\": \"Sony A7M4 Body\",\n \"price\": [\n {\"text\": \"\\u00a5\"},\n {\"text\": \"13999\"}\n ],\n \"area\": \"Shanghai\",\n \"userNickName\": \"seller_01\",\n \"picUrl\": \"https://img.example.com/a7m4.jpg\",\n \"itemId\": \"123456\",\n \"oriPrice\": \"\\u00a516999\",\n \"fishTags\": {\n \"r1\": {\n \"tagList\": [\n {\"data\": {\"content\": \"\\u9a8c\\u8d27\\u5b9d\"}}\n ]\n }\n }\n },\n \"clickParam\": {\n \"args\": {\n \"publishTime\": \"1710000000000\",\n \"wantNum\": 12,\n \"tag\": \"freeship\"\n }\n },\n \"targetUrl\": \"fleamarket://item?id=123456\"\n }\n }\n }\n }\n ]\n }\n}\n" }, { "path": "tests/fixtures/state.sample.json", "content": "{\n \"session\": \"dummy\"\n}\n" }, { "path": "tests/fixtures/user_head.json", "content": "{\n \"data\": {\n \"module\": {\n \"base\": {\n \"displayName\": \"seller_01\",\n \"avatar\": {\"avatar\": \"https://img.example.com/avatar.jpg\"},\n \"introduction\": \"Trusted seller\",\n \"ylzTags\": [\n {\"attributes\": {\"role\": \"seller\", \"level\": \"3\"}, \"text\": \"S3\"},\n {\"attributes\": {\"role\": \"buyer\", \"level\": \"2\"}, \"text\": \"B2\"}\n ]\n },\n \"tabs\": {\n \"item\": {\"number\": 12},\n \"rate\": {\"number\": 88}\n }\n }\n }\n}\n" }, { "path": "tests/fixtures/user_items.json", "content": "[\n {\n \"cardData\": {\n \"itemStatus\": 0,\n \"id\": \"10001\",\n \"title\": \"Lens 24-70\",\n \"priceInfo\": {\"price\": \"3500\"},\n \"picInfo\": {\"picUrl\": \"https://img.example.com/lens.jpg\"}\n }\n },\n {\n \"cardData\": {\n \"itemStatus\": 1,\n \"id\": \"10002\",\n \"title\": \"Battery Grip\",\n \"priceInfo\": {\"price\": \"600\"},\n \"picInfo\": {\"picUrl\": \"https://img.example.com/grip.jpg\"}\n }\n }\n]\n" }, { "path": "tests/integration/test_api_dashboard.py", "content": "import json\n\nfrom fastapi import FastAPI\nfrom fastapi.testclient import TestClient\n\nfrom src.api import dependencies as deps\nfrom src.api.routes import dashboard\nfrom src.domain.models.task import TaskCreate\nfrom src.infrastructure.persistence.sqlite_task_repository import SqliteTaskRepository\nfrom src.services.task_service import TaskService\n\n\ndef _write_jsonl(path, records):\n with open(path, \"w\", encoding=\"utf-8\") as file:\n for record in records:\n file.write(json.dumps(record, ensure_ascii=False) + \"\\n\")\n\n\ndef test_dashboard_summary_aggregates_tasks_and_results(tmp_path, monkeypatch):\n monkeypatch.chdir(tmp_path)\n\n jsonl_dir = tmp_path / \"jsonl\"\n jsonl_dir.mkdir(parents=True, exist_ok=True)\n\n repository = SqliteTaskRepository(\n db_path=str(tmp_path / \"app.sqlite3\"),\n legacy_config_file=None,\n )\n task_service = TaskService(repository)\n app = FastAPI()\n app.include_router(dashboard.router)\n app.dependency_overrides[deps.get_task_service] = lambda: task_service\n\n client = TestClient(app)\n\n first = TaskCreate(\n task_name=\"Apple Watch 任务\",\n keyword=\"apple watch\",\n description=\"只关注价格合适且成色好的 Apple Watch。\",\n max_pages=3,\n personal_only=True,\n )\n second = TaskCreate(\n task_name=\"iPad 任务\",\n keyword=\"ipad pro\",\n description=\"关注 2024 款 iPad Pro。\",\n max_pages=2,\n personal_only=True,\n )\n\n created_first = task_service.create_task(first)\n created_second = task_service.create_task(second)\n import asyncio\n created_first = asyncio.run(created_first)\n created_second = asyncio.run(created_second)\n asyncio.run(task_service.update_task_status(created_second.id, True))\n\n records = [\n {\n \"爬取时间\": \"2026-03-10T10:00:00\",\n \"搜索关键字\": \"apple watch\",\n \"任务名称\": \"Apple Watch 任务\",\n \"商品信息\": {\n \"商品ID\": \"watch-1\",\n \"商品标题\": \"Apple Watch S10\",\n \"商品链接\": \"https://www.goofish.com/item?id=watch-1\",\n \"当前售价\": \"¥1800\",\n },\n \"ai_analysis\": {\n \"analysis_source\": \"ai\",\n \"is_recommended\": True,\n \"reason\": \"价格低于均价\",\n },\n },\n {\n \"爬取时间\": \"2026-03-10T11:00:00\",\n \"搜索关键字\": \"apple watch\",\n \"任务名称\": \"Apple Watch 任务\",\n \"商品信息\": {\n \"商品ID\": \"watch-2\",\n \"商品标题\": \"Apple Watch S10 蜂窝版\",\n \"商品链接\": \"https://www.goofish.com/item?id=watch-2\",\n \"当前售价\": \"¥2100\",\n },\n \"ai_analysis\": {\n \"analysis_source\": \"keyword\",\n \"is_recommended\": False,\n \"reason\": \"未命中规则\",\n },\n },\n ]\n _write_jsonl(jsonl_dir / \"apple_watch_full_data.jsonl\", records)\n\n response = client.get(\"/api/dashboard/summary\")\n assert response.status_code == 200\n payload = response.json()\n\n assert payload[\"summary\"][\"enabled_tasks\"] == 2\n assert payload[\"summary\"][\"running_tasks\"] == 1\n assert payload[\"summary\"][\"result_files\"] == 1\n assert payload[\"summary\"][\"scanned_items\"] == 2\n assert payload[\"summary\"][\"recommended_items\"] == 1\n assert payload[\"summary\"][\"ai_recommended_items\"] == 1\n assert payload[\"summary\"][\"keyword_recommended_items\"] == 0\n assert payload[\"focus_file\"] == \"apple_watch_full_data.jsonl\"\n\n watch_summary = next(\n item for item in payload[\"task_summaries\"] if item[\"task_name\"] == \"Apple Watch 任务\"\n )\n assert watch_summary[\"filename\"] == \"apple_watch_full_data.jsonl\"\n assert watch_summary[\"total_items\"] == 2\n assert watch_summary[\"latest_recommended_title\"] == \"Apple Watch S10\"\n\n ipad_summary = next(\n item for item in payload[\"task_summaries\"] if item[\"task_name\"] == \"iPad 任务\"\n )\n assert ipad_summary[\"filename\"] is None\n assert ipad_summary[\"is_running\"] is True\n\n statuses = {item[\"status\"] for item in payload[\"recent_activities\"]}\n assert \"AI 推荐\" in statuses\n assert \"结果已更新\" in statuses\n assert \"运行中\" in statuses\n" }, { "path": "tests/integration/test_api_results.py", "content": "import json\n\nfrom fastapi import FastAPI\nfrom fastapi.testclient import TestClient\n\nfrom src.api.routes import results\nfrom src.services.price_history_service import record_market_snapshots\n\n\ndef _write_jsonl(path, records):\n with open(path, \"w\", encoding=\"utf-8\") as f:\n for record in records:\n f.write(json.dumps(record, ensure_ascii=False) + \"\\n\")\n\n\ndef test_results_filter_and_sort_for_keyword_recommendations(tmp_path, monkeypatch):\n monkeypatch.chdir(tmp_path)\n jsonl_dir = tmp_path / \"jsonl\"\n jsonl_dir.mkdir(parents=True, exist_ok=True)\n target_file = jsonl_dir / \"demo_full_data.jsonl\"\n\n records = [\n {\n \"爬取时间\": \"2026-01-01T01:00:00\",\n \"商品信息\": {\"当前售价\": \"¥1000\", \"发布时间\": \"2026-01-01 10:00\"},\n \"ai_analysis\": {\n \"analysis_source\": \"keyword\",\n \"is_recommended\": True,\n \"keyword_hit_count\": 3,\n \"reason\": \"命中 3 个关键词\",\n },\n },\n {\n \"爬取时间\": \"2026-01-01T02:00:00\",\n \"商品信息\": {\"当前售价\": \"¥2000\", \"发布时间\": \"2026-01-01 11:00\"},\n \"ai_analysis\": {\n \"analysis_source\": \"keyword\",\n \"is_recommended\": True,\n \"keyword_hit_count\": 1,\n \"reason\": \"命中 1 个关键词\",\n },\n },\n {\n \"爬取时间\": \"2026-01-01T03:00:00\",\n \"商品信息\": {\"当前售价\": \"¥3000\", \"发布时间\": \"2026-01-01 12:00\"},\n \"ai_analysis\": {\n \"analysis_source\": \"ai\",\n \"is_recommended\": True,\n \"reason\": \"AI推荐\",\n },\n },\n ]\n _write_jsonl(target_file, records)\n\n app = FastAPI()\n app.include_router(results.router)\n client = TestClient(app)\n\n resp = client.get(\n \"/api/results/demo_full_data.jsonl\",\n params={\"keyword_recommended_only\": True, \"sort_by\": \"keyword_hit_count\", \"sort_order\": \"desc\"},\n )\n assert resp.status_code == 200\n data = resp.json()\n assert data[\"total_items\"] == 2\n assert data[\"items\"][0][\"ai_analysis\"][\"keyword_hit_count\"] == 3\n assert data[\"items\"][1][\"ai_analysis\"][\"keyword_hit_count\"] == 1\n\n resp = client.get(\n \"/api/results/demo_full_data.jsonl\",\n params={\"ai_recommended_only\": True},\n )\n assert resp.status_code == 200\n data = resp.json()\n assert data[\"total_items\"] == 1\n assert data[\"items\"][0][\"ai_analysis\"][\"analysis_source\"] == \"ai\"\n\n resp = client.get(\n \"/api/results/demo_full_data.jsonl\",\n params={\"ai_recommended_only\": True, \"keyword_recommended_only\": True},\n )\n assert resp.status_code == 400\n\n\ndef test_results_insights_and_export_csv(tmp_path, monkeypatch):\n monkeypatch.chdir(tmp_path)\n jsonl_dir = tmp_path / \"jsonl\"\n jsonl_dir.mkdir(parents=True, exist_ok=True)\n target_file = jsonl_dir / \"demo_full_data.jsonl\"\n\n records = [\n {\n \"爬取时间\": \"2026-01-02T09:00:00\",\n \"搜索关键字\": \"demo\",\n \"任务名称\": \"Demo 任务\",\n \"商品信息\": {\n \"商品ID\": \"1001\",\n \"商品标题\": \"Demo One\",\n \"商品链接\": \"https://www.goofish.com/item?id=1001\",\n \"当前售价\": \"¥950\",\n \"发布时间\": \"2026-01-02 08:30\",\n },\n \"卖家信息\": {\"卖家昵称\": \"卖家A\"},\n \"ai_analysis\": {\n \"analysis_source\": \"ai\",\n \"is_recommended\": True,\n \"reason\": \"价格低于近期均价\",\n },\n },\n {\n \"爬取时间\": \"2026-01-02T09:05:00\",\n \"搜索关键字\": \"demo\",\n \"任务名称\": \"Demo 任务\",\n \"商品信息\": {\n \"商品ID\": \"1002\",\n \"商品标题\": \"Demo Two\",\n \"商品链接\": \"https://www.goofish.com/item?id=1002\",\n \"当前售价\": \"¥1200\",\n \"发布时间\": \"2026-01-02 08:45\",\n },\n \"卖家信息\": {\"卖家昵称\": \"卖家B\"},\n \"ai_analysis\": {\n \"analysis_source\": \"keyword\",\n \"is_recommended\": False,\n \"reason\": \"未命中\",\n \"keyword_hit_count\": 0,\n },\n },\n ]\n _write_jsonl(target_file, records)\n\n record_market_snapshots(\n keyword=\"demo\",\n task_name=\"Demo 任务\",\n items=[\n {\n \"商品ID\": \"1001\",\n \"商品标题\": \"Demo One\",\n \"当前售价\": \"¥1000\",\n \"商品链接\": \"https://www.goofish.com/item?id=1001\",\n },\n {\n \"商品ID\": \"1002\",\n \"商品标题\": \"Demo Two\",\n \"当前售价\": \"¥1200\",\n \"商品链接\": \"https://www.goofish.com/item?id=1002\",\n },\n ],\n run_id=\"run-1\",\n snapshot_time=\"2026-01-01T10:00:00\",\n seen_item_ids=set(),\n )\n record_market_snapshots(\n keyword=\"demo\",\n task_name=\"Demo 任务\",\n items=[\n {\n \"商品ID\": \"1001\",\n \"商品标题\": \"Demo One\",\n \"当前售价\": \"¥950\",\n \"商品链接\": \"https://www.goofish.com/item?id=1001\",\n },\n {\n \"商品ID\": \"1002\",\n \"商品标题\": \"Demo Two\",\n \"当前售价\": \"¥1180\",\n \"商品链接\": \"https://www.goofish.com/item?id=1002\",\n },\n ],\n run_id=\"run-2\",\n snapshot_time=\"2026-01-02T10:00:00\",\n seen_item_ids=set(),\n )\n\n app = FastAPI()\n app.include_router(results.router)\n client = TestClient(app)\n\n insights_resp = client.get(\"/api/results/demo_full_data.jsonl/insights\")\n assert insights_resp.status_code == 200\n insights = insights_resp.json()\n assert insights[\"market_summary\"][\"sample_count\"] == 2\n assert len(insights[\"daily_trend\"]) == 2\n\n list_resp = client.get(\"/api/results/demo_full_data.jsonl\")\n assert list_resp.status_code == 200\n items = list_resp.json()[\"items\"]\n assert items[0][\"price_insight\"][\"observation_count\"] >= 1\n\n export_resp = client.get(\n \"/api/results/demo_full_data.jsonl/export\",\n params={\"sort_by\": \"price\", \"sort_order\": \"asc\"},\n )\n assert export_resp.status_code == 200\n assert \"text/csv\" in export_resp.headers[\"content-type\"]\n text = export_resp.text\n assert \"任务名称,搜索关键字,商品ID,商品标题\" in text\n assert \"Demo One\" in text\n\n\ndef test_results_export_csv_supports_unicode_filename(tmp_path, monkeypatch):\n monkeypatch.chdir(tmp_path)\n jsonl_dir = tmp_path / \"jsonl\"\n jsonl_dir.mkdir(parents=True, exist_ok=True)\n target_file = jsonl_dir / \"演示_full_data.jsonl\"\n\n records = [\n {\n \"爬取时间\": \"2026-01-02T09:00:00\",\n \"搜索关键字\": \"演示\",\n \"任务名称\": \"演示任务\",\n \"商品信息\": {\n \"商品ID\": \"1001\",\n \"商品标题\": \"演示商品\",\n \"商品链接\": \"https://www.goofish.com/item?id=1001\",\n \"当前售价\": \"¥950\",\n \"发布时间\": \"2026-01-02 08:30\",\n },\n \"卖家信息\": {\"卖家昵称\": \"卖家A\"},\n \"ai_analysis\": {\n \"analysis_source\": \"ai\",\n \"is_recommended\": True,\n \"reason\": \"价格合理\",\n },\n }\n ]\n _write_jsonl(target_file, records)\n\n app = FastAPI()\n app.include_router(results.router)\n client = TestClient(app)\n\n export_resp = client.get(\"/api/results/演示_full_data.jsonl/export\")\n assert export_resp.status_code == 200\n assert \"text/csv\" in export_resp.headers[\"content-type\"]\n disposition = export_resp.headers[\"content-disposition\"]\n assert 'filename=\"export.csv\"' in disposition\n assert \"filename*=UTF-8''%E6%BC%94%E7%A4%BA_full_data.csv\" in disposition\n" }, { "path": "tests/integration/test_api_settings.py", "content": "from fastapi import FastAPI\nfrom fastapi.testclient import TestClient\n\nfrom src.api import dependencies as deps\nfrom src.api.routes import settings\nfrom src.infrastructure.config.env_manager import env_manager\n\n\n_SETTINGS_ENV_KEYS = [\n \"ACCOUNT_ROTATION_ENABLED\",\n \"ACCOUNT_ROTATION_MODE\",\n \"ACCOUNT_ROTATION_RETRY_LIMIT\",\n \"ACCOUNT_BLACKLIST_TTL\",\n \"ACCOUNT_STATE_DIR\",\n \"PROXY_ROTATION_ENABLED\",\n \"PROXY_ROTATION_MODE\",\n \"PROXY_POOL\",\n \"PROXY_ROTATION_RETRY_LIMIT\",\n \"PROXY_BLACKLIST_TTL\",\n \"OPENAI_API_KEY\",\n \"OPENAI_BASE_URL\",\n \"OPENAI_MODEL_NAME\",\n \"SKIP_AI_ANALYSIS\",\n \"PROXY_URL\",\n \"NTFY_TOPIC_URL\",\n \"GOTIFY_URL\",\n \"GOTIFY_TOKEN\",\n \"BARK_URL\",\n \"WX_BOT_URL\",\n \"TELEGRAM_BOT_TOKEN\",\n \"TELEGRAM_CHAT_ID\",\n \"TELEGRAM_API_BASE_URL\",\n \"WEBHOOK_URL\",\n \"WEBHOOK_METHOD\",\n \"WEBHOOK_HEADERS\",\n \"WEBHOOK_CONTENT_TYPE\",\n \"WEBHOOK_QUERY_PARAMETERS\",\n \"WEBHOOK_BODY\",\n \"PCURL_TO_MOBILE\",\n]\n\n\nclass _IdleProcessService:\n def __init__(self) -> None:\n self.processes = {}\n\n\ndef _build_settings_client() -> TestClient:\n app = FastAPI()\n app.include_router(settings.router)\n app.dependency_overrides[deps.get_process_service] = _IdleProcessService\n return TestClient(app)\n\n\ndef _clear_settings_env(monkeypatch) -> None:\n for key in _SETTINGS_ENV_KEYS:\n monkeypatch.delenv(key, raising=False)\n\n\ndef test_rotation_settings_include_account_rotation_fields(tmp_path, monkeypatch):\n _clear_settings_env(monkeypatch)\n env_file = tmp_path / \".env\"\n env_file.write_text(\n \"\\n\".join(\n [\n \"ACCOUNT_ROTATION_ENABLED=false\",\n \"ACCOUNT_ROTATION_MODE=per_task\",\n \"ACCOUNT_ROTATION_RETRY_LIMIT=2\",\n \"ACCOUNT_BLACKLIST_TTL=300\",\n \"ACCOUNT_STATE_DIR=state\",\n \"PROXY_ROTATION_ENABLED=false\",\n \"PROXY_ROTATION_MODE=per_task\",\n \"PROXY_ROTATION_RETRY_LIMIT=2\",\n \"PROXY_BLACKLIST_TTL=300\",\n ]\n ),\n encoding=\"utf-8\",\n )\n monkeypatch.setattr(env_manager, \"env_file\", env_file)\n\n client = _build_settings_client()\n\n response = client.get(\"/api/settings/rotation\")\n assert response.status_code == 200\n payload = response.json()\n assert payload[\"ACCOUNT_ROTATION_ENABLED\"] is False\n assert payload[\"ACCOUNT_ROTATION_MODE\"] == \"per_task\"\n assert payload[\"ACCOUNT_STATE_DIR\"] == \"state\"\n\n update_response = client.put(\n \"/api/settings/rotation\",\n json={\n \"ACCOUNT_ROTATION_ENABLED\": True,\n \"ACCOUNT_ROTATION_MODE\": \"on_failure\",\n \"ACCOUNT_ROTATION_RETRY_LIMIT\": 4,\n \"ACCOUNT_BLACKLIST_TTL\": 900,\n \"ACCOUNT_STATE_DIR\": \"accounts\",\n },\n )\n assert update_response.status_code == 200\n\n latest = env_file.read_text(encoding=\"utf-8\")\n assert \"ACCOUNT_ROTATION_ENABLED=true\" in latest\n assert \"ACCOUNT_ROTATION_MODE=on_failure\" in latest\n assert \"ACCOUNT_ROTATION_RETRY_LIMIT=4\" in latest\n assert \"ACCOUNT_BLACKLIST_TTL=900\" in latest\n assert \"ACCOUNT_STATE_DIR=accounts\" in latest\n\n\ndef test_notification_settings_redact_sensitive_values_and_expose_flags(tmp_path, monkeypatch):\n _clear_settings_env(monkeypatch)\n env_file = tmp_path / \".env\"\n env_file.write_text(\n \"\\n\".join(\n [\n \"NTFY_TOPIC_URL=https://ntfy.sh/demo-topic\",\n \"GOTIFY_URL=https://gotify.example.com\",\n \"GOTIFY_TOKEN=secret-token\",\n \"BARK_URL=https://api.day.app/private-key/\",\n \"WX_BOT_URL=https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=secret\",\n \"TELEGRAM_BOT_TOKEN=telegram-secret\",\n \"TELEGRAM_CHAT_ID=123456\",\n \"TELEGRAM_API_BASE_URL=https://tg.example.com/proxy\",\n \"WEBHOOK_URL=https://hooks.example.com/notify?token=secret\",\n 'WEBHOOK_HEADERS={\"Authorization\":\"Bearer secret\"}',\n 'WEBHOOK_BODY={\"message\":\"{{content}}\"}',\n ]\n ),\n encoding=\"utf-8\",\n )\n monkeypatch.setattr(env_manager, \"env_file\", env_file)\n client = _build_settings_client()\n\n response = client.get(\"/api/settings/notifications\")\n\n assert response.status_code == 200\n payload = response.json()\n assert payload[\"NTFY_TOPIC_URL\"] == \"https://ntfy.sh/demo-topic\"\n assert payload[\"GOTIFY_URL\"] == \"https://gotify.example.com\"\n assert payload[\"TELEGRAM_CHAT_ID\"] == \"123456\"\n assert payload[\"TELEGRAM_API_BASE_URL\"] == \"https://tg.example.com/proxy\"\n assert payload[\"BARK_URL\"] == \"\"\n assert payload[\"WX_BOT_URL\"] == \"\"\n assert payload[\"GOTIFY_TOKEN\"] == \"\"\n assert payload[\"TELEGRAM_BOT_TOKEN\"] == \"\"\n assert payload[\"WEBHOOK_URL\"] == \"\"\n assert payload[\"WEBHOOK_HEADERS\"] == \"\"\n assert payload[\"BARK_URL_SET\"] is True\n assert payload[\"WX_BOT_URL_SET\"] is True\n assert payload[\"GOTIFY_TOKEN_SET\"] is True\n assert payload[\"TELEGRAM_BOT_TOKEN_SET\"] is True\n assert payload[\"WEBHOOK_URL_SET\"] is True\n assert payload[\"WEBHOOK_HEADERS_SET\"] is True\n assert payload[\"WEBHOOK_BODY\"] == '{\"message\":\"{{content}}\"}'\n\n\ndef test_update_notification_settings_rejects_invalid_channel_config(tmp_path, monkeypatch):\n _clear_settings_env(monkeypatch)\n env_file = tmp_path / \".env\"\n env_file.write_text(\"\", encoding=\"utf-8\")\n monkeypatch.setattr(env_manager, \"env_file\", env_file)\n client = _build_settings_client()\n\n gotify_response = client.put(\n \"/api/settings/notifications\",\n json={\"GOTIFY_URL\": \"https://gotify.example.com\"},\n )\n assert gotify_response.status_code == 422\n assert \"GOTIFY_TOKEN\" in gotify_response.text\n\n telegram_proxy_response = client.put(\n \"/api/settings/notifications\",\n json={\"TELEGRAM_API_BASE_URL\": \"not-a-url\"},\n )\n assert telegram_proxy_response.status_code == 422\n assert \"TELEGRAM_API_BASE_URL\" in telegram_proxy_response.text\n\n webhook_response = client.put(\n \"/api/settings/notifications\",\n json={\n \"WEBHOOK_URL\": \"https://hooks.example.com/notify\",\n \"WEBHOOK_METHOD\": \"POST\",\n \"WEBHOOK_CONTENT_TYPE\": \"JSON\",\n \"WEBHOOK_HEADERS\": '{\"Authorization\": \"Bearer secret\"',\n },\n )\n assert webhook_response.status_code == 422\n assert \"WEBHOOK_HEADERS\" in webhook_response.text\n\n\ndef test_system_status_includes_notification_channel_flags(tmp_path, monkeypatch):\n _clear_settings_env(monkeypatch)\n env_file = tmp_path / \".env\"\n env_file.write_text(\n \"\\n\".join(\n [\n \"NTFY_TOPIC_URL=https://ntfy.sh/demo-topic\",\n \"GOTIFY_URL=https://gotify.example.com\",\n \"GOTIFY_TOKEN=secret-token\",\n \"BARK_URL=https://api.day.app/private-key/\",\n \"WX_BOT_URL=https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=secret\",\n \"TELEGRAM_BOT_TOKEN=telegram-secret\",\n \"TELEGRAM_CHAT_ID=123456\",\n \"WEBHOOK_URL=https://hooks.example.com/notify\",\n ]\n ),\n encoding=\"utf-8\",\n )\n monkeypatch.setattr(env_manager, \"env_file\", env_file)\n client = _build_settings_client()\n\n response = client.get(\"/api/settings/status\")\n\n assert response.status_code == 200\n env_payload = response.json()[\"env_file\"]\n assert env_payload[\"ntfy_topic_url_set\"] is True\n assert env_payload[\"gotify_url_set\"] is True\n assert env_payload[\"gotify_token_set\"] is True\n assert env_payload[\"bark_url_set\"] is True\n assert env_payload[\"wx_bot_url_set\"] is True\n assert env_payload[\"telegram_bot_token_set\"] is True\n assert env_payload[\"telegram_chat_id_set\"] is True\n assert env_payload[\"webhook_url_set\"] is True\n\n\ndef test_notification_test_endpoint_merges_stored_secret_values(tmp_path, monkeypatch):\n _clear_settings_env(monkeypatch)\n env_file = tmp_path / \".env\"\n env_file.write_text(\n \"\\n\".join(\n [\n \"TELEGRAM_BOT_TOKEN=stored-token\",\n \"TELEGRAM_CHAT_ID=10001\",\n \"TELEGRAM_API_BASE_URL=https://tg-proxy.example.com/base\",\n ]\n ),\n encoding=\"utf-8\",\n )\n monkeypatch.setattr(env_manager, \"env_file\", env_file)\n client = _build_settings_client()\n\n captured = {}\n\n class _FakeResponse:\n status_code = 200\n\n def raise_for_status(self):\n return None\n\n def json(self):\n return {\"ok\": True}\n\n def _fake_post(url, json=None, headers=None, timeout=None):\n captured[\"url\"] = url\n captured[\"json\"] = json\n return _FakeResponse()\n\n monkeypatch.setattr(\"requests.post\", _fake_post)\n\n response = client.post(\n \"/api/settings/notifications/test\",\n json={\n \"channel\": \"telegram\",\n \"settings\": {\n \"TELEGRAM_CHAT_ID\": \"20002\",\n },\n },\n )\n\n assert response.status_code == 200\n payload = response.json()\n assert payload[\"results\"][\"telegram\"][\"success\"] is True\n assert captured[\"url\"] == \"https://tg-proxy.example.com/base/botstored-token/sendMessage\"\n assert captured[\"json\"][\"chat_id\"] == \"20002\"\n\n\ndef test_ai_settings_fall_back_to_runtime_environment_when_env_file_missing(tmp_path, monkeypatch):\n _clear_settings_env(monkeypatch)\n env_file = tmp_path / \".env\"\n monkeypatch.setattr(env_manager, \"env_file\", env_file)\n monkeypatch.setenv(\"OPENAI_API_KEY\", \"runtime-key\")\n monkeypatch.setenv(\"OPENAI_BASE_URL\", \"https://runtime.example.com/v1\")\n monkeypatch.setenv(\"OPENAI_MODEL_NAME\", \"runtime-model\")\n monkeypatch.setenv(\"PROXY_URL\", \"http://127.0.0.1:7890\")\n client = _build_settings_client()\n\n ai_response = client.get(\"/api/settings/ai\")\n assert ai_response.status_code == 200\n assert ai_response.json() == {\n \"OPENAI_BASE_URL\": \"https://runtime.example.com/v1\",\n \"OPENAI_MODEL_NAME\": \"runtime-model\",\n \"SKIP_AI_ANALYSIS\": False,\n \"PROXY_URL\": \"http://127.0.0.1:7890\",\n }\n\n status_response = client.get(\"/api/settings/status\")\n assert status_response.status_code == 200\n env_payload = status_response.json()[\"env_file\"]\n assert env_payload[\"exists\"] is False\n assert env_payload[\"openai_api_key_set\"] is True\n assert env_payload[\"openai_base_url_set\"] is True\n assert env_payload[\"openai_model_name_set\"] is True\n\n\ndef test_notification_settings_fall_back_to_runtime_environment_when_env_file_missing(\n tmp_path, monkeypatch\n):\n _clear_settings_env(monkeypatch)\n env_file = tmp_path / \".env\"\n monkeypatch.setattr(env_manager, \"env_file\", env_file)\n monkeypatch.setenv(\"NTFY_TOPIC_URL\", \"https://ntfy.sh/runtime-topic\")\n monkeypatch.setenv(\"TELEGRAM_BOT_TOKEN\", \"runtime-telegram-token\")\n monkeypatch.setenv(\"TELEGRAM_CHAT_ID\", \"20001\")\n monkeypatch.setenv(\"TELEGRAM_API_BASE_URL\", \"https://runtime-tg-proxy.example.com\")\n monkeypatch.setenv(\"BARK_URL\", \"https://api.day.app/runtime-secret/\")\n client = _build_settings_client()\n\n response = client.get(\"/api/settings/notifications\")\n\n assert response.status_code == 200\n payload = response.json()\n assert payload[\"NTFY_TOPIC_URL\"] == \"https://ntfy.sh/runtime-topic\"\n assert payload[\"TELEGRAM_CHAT_ID\"] == \"20001\"\n assert payload[\"TELEGRAM_API_BASE_URL\"] == \"https://runtime-tg-proxy.example.com\"\n assert payload[\"BARK_URL\"] == \"\"\n assert payload[\"BARK_URL_SET\"] is True\n assert payload[\"TELEGRAM_BOT_TOKEN_SET\"] is True\n assert sorted(payload[\"CONFIGURED_CHANNELS\"]) == [\"bark\", \"ntfy\", \"telegram\"]\n\n\ndef test_ai_test_endpoint_falls_back_to_responses_when_chat_completions_api_404(\n tmp_path, monkeypatch\n):\n _clear_settings_env(monkeypatch)\n env_file = tmp_path / \".env\"\n env_file.write_text(\"\", encoding=\"utf-8\")\n monkeypatch.setattr(env_manager, \"env_file\", env_file)\n client = _build_settings_client()\n request_history = []\n\n class _FakeOpenAI:\n def __init__(self, **_kwargs):\n self.responses = type(\n \"_Responses\",\n (),\n {\"create\": self._responses_create},\n )()\n self.chat = type(\n \"_Chat\",\n (),\n {\n \"completions\": type(\n \"_Completions\",\n (),\n {\"create\": self._chat_create},\n )()\n },\n )()\n\n def _responses_create(self, **kwargs):\n request_history.append((\"responses\", kwargs))\n return type(\n \"_Response\",\n (),\n {\"output_text\": \"OK\"},\n )()\n\n def _chat_create(self, **kwargs):\n request_history.append((\"chat\", kwargs))\n raise Exception(\"Error code: 404 - page not found\")\n\n import openai\n\n monkeypatch.setattr(openai, \"OpenAI\", _FakeOpenAI)\n\n response = client.post(\n \"/api/settings/ai/test\",\n json={\n \"OPENAI_API_KEY\": \"demo\",\n \"OPENAI_BASE_URL\": \"https://example.com/v1/\",\n \"OPENAI_MODEL_NAME\": \"demo-model\",\n },\n )\n\n assert response.status_code == 200\n payload = response.json()\n assert payload[\"success\"] is True\n assert payload[\"response\"] == \"OK\"\n assert request_history[0][0] == \"chat\"\n assert request_history[0][1][\"messages\"][0][\"content\"] == settings.AI_TEST_PROMPT\n assert request_history[1][0] == \"responses\"\n assert request_history[1][1][\"input\"][0][\"content\"][0][\"text\"] == settings.AI_TEST_PROMPT\n" }, { "path": "tests/integration/test_api_tasks.py", "content": "import asyncio\nimport time\n\n\ndef test_create_list_update_delete_task(api_client, api_context, sample_task_payload):\n response = api_client.post(\"/api/tasks/\", json=sample_task_payload)\n assert response.status_code == 200\n created = response.json()[\"task\"]\n assert created[\"task_name\"] == sample_task_payload[\"task_name\"]\n assert created[\"analyze_images\"] is True\n assert created[\"next_run_at\"] == \"2026-03-19T08:15:00+08:00\"\n\n response = api_client.get(\"/api/tasks\")\n assert response.status_code == 200\n tasks = response.json()\n assert len(tasks) == 1\n assert tasks[0][\"keyword\"] == sample_task_payload[\"keyword\"]\n assert tasks[0][\"analyze_images\"] is True\n assert tasks[0][\"next_run_at\"] == \"2026-03-19T08:15:00+08:00\"\n\n response = api_client.patch(\"/api/tasks/0\", json={\"enabled\": False, \"analyze_images\": False})\n assert response.status_code == 200\n updated = response.json()[\"task\"]\n assert updated[\"enabled\"] is False\n assert updated[\"analyze_images\"] is False\n assert updated[\"next_run_at\"] is None\n\n response = api_client.delete(\"/api/tasks/0\")\n assert response.status_code == 200\n\n response = api_client.get(\"/api/tasks\")\n assert response.status_code == 200\n assert response.json() == []\n\n\ndef test_start_stop_task_updates_status(api_client, api_context, sample_task_payload):\n response = api_client.post(\"/api/tasks/\", json=sample_task_payload)\n assert response.status_code == 200\n\n response = api_client.post(\"/api/tasks/start/0\")\n assert response.status_code == 200\n\n response = api_client.get(\"/api/tasks/0\")\n assert response.status_code == 200\n assert response.json()[\"is_running\"] is True\n\n response = api_client.post(\"/api/tasks/stop/0\")\n assert response.status_code == 200\n\n response = api_client.get(\"/api/tasks/0\")\n assert response.status_code == 200\n assert response.json()[\"is_running\"] is False\n\n process_service = api_context[\"process_service\"]\n assert process_service.started == [(0, sample_task_payload[\"task_name\"])]\n assert process_service.stopped == [0]\n\n\ndef test_generate_keyword_mode_task_without_ai_criteria(api_client):\n payload = {\n \"task_name\": \"A7M4 关键词筛选\",\n \"keyword\": \"sony a7m4\",\n \"description\": \"\",\n \"decision_mode\": \"keyword\",\n \"keyword_rules\": [\"a7m4\", \"验货宝\"],\n \"max_pages\": 2,\n \"personal_only\": True,\n }\n\n response = api_client.post(\"/api/tasks/generate\", json=payload)\n assert response.status_code == 200\n created = response.json()[\"task\"]\n assert created[\"decision_mode\"] == \"keyword\"\n assert created[\"ai_prompt_criteria_file\"] == \"\"\n assert created[\"keyword_rules\"] == [\"a7m4\", \"验货宝\"]\n\n\ndef test_generate_ai_task_returns_job_and_completes_async(api_client, api_context, monkeypatch):\n payload = {\n \"task_name\": \"Apple Watch S10\",\n \"keyword\": \"apple watch s10\",\n \"description\": \"只看国行蜂窝版，电池健康高于 95%，拒绝维修机。\",\n \"analyze_images\": False,\n \"decision_mode\": \"ai\",\n \"max_pages\": 2,\n \"personal_only\": True,\n }\n\n async def fake_generate_criteria(*_args, **_kwargs):\n await asyncio.sleep(0.05)\n return \"[V6.3 核心升级]\\\\nApple Watch criteria\"\n\n monkeypatch.setattr(\n \"src.services.task_generation_runner.generate_criteria\",\n fake_generate_criteria,\n )\n\n response = api_client.post(\"/api/tasks/generate\", json=payload)\n\n assert response.status_code == 202\n job = response.json()[\"job\"]\n assert isinstance(job[\"job_id\"], str)\n assert job[\"status\"] in {\"queued\", \"running\"}\n assert job[\"task\"] is None\n\n status_response = api_client.get(f\"/api/tasks/generate-jobs/{job['job_id']}\")\n assert status_response.status_code == 200\n\n for _ in range(50):\n status_response = api_client.get(f\"/api/tasks/generate-jobs/{job['job_id']}\")\n latest_job = status_response.json()[\"job\"]\n if latest_job[\"status\"] == \"completed\":\n break\n time.sleep(0.02)\n else:\n raise AssertionError(\"任务生成作业未在预期时间内完成\")\n\n assert latest_job[\"task\"][\"task_name\"] == payload[\"task_name\"]\n assert latest_job[\"task\"][\"ai_prompt_criteria_file\"].endswith(\"_criteria.txt\")\n assert latest_job[\"task\"][\"analyze_images\"] is False\n assert api_context[\"scheduler_service\"].reload_calls == 1\n\n\ndef test_create_task_accepts_cron_alias(api_client, sample_task_payload):\n payload = dict(sample_task_payload)\n payload[\"cron\"] = \"@daily\"\n\n response = api_client.post(\"/api/tasks/\", json=payload)\n\n assert response.status_code == 200\n assert response.json()[\"task\"][\"cron\"] == \"0 0 * * *\"\n\n\ndef test_create_task_rejects_fixed_account_strategy_without_state_file(api_client, sample_task_payload):\n payload = dict(sample_task_payload)\n payload[\"account_strategy\"] = \"fixed\"\n\n response = api_client.post(\"/api/tasks/\", json=payload)\n\n assert response.status_code == 422\n\n\ndef test_create_task_accepts_rotate_account_strategy(api_client, sample_task_payload):\n payload = dict(sample_task_payload)\n payload[\"account_strategy\"] = \"rotate\"\n\n response = api_client.post(\"/api/tasks/\", json=payload)\n\n assert response.status_code == 200\n task = response.json()[\"task\"]\n assert task[\"account_strategy\"] == \"rotate\"\n\n\ndef test_update_task_accepts_six_field_cron_expression(api_client, sample_task_payload):\n create_response = api_client.post(\"/api/tasks/\", json=sample_task_payload)\n assert create_response.status_code == 200\n\n response = api_client.patch(\"/api/tasks/0\", json={\"cron\": \"0 0 8 * * *\"})\n\n assert response.status_code == 200\n\n task_response = api_client.get(\"/api/tasks/0\")\n assert task_response.status_code == 200\n assert task_response.json()[\"cron\"] == \"0 0 8 * * *\"\n\n\ndef test_create_task_rejects_invalid_cron_expression(api_client, sample_task_payload):\n payload = dict(sample_task_payload)\n payload[\"cron\"] = \"every day at 8\"\n\n response = api_client.post(\"/api/tasks/\", json=payload)\n\n assert response.status_code == 422\n\n\ndef test_delete_task_stops_runtime_and_reindexes_process_state(\n api_client,\n api_context,\n sample_task_payload,\n):\n second_payload = dict(sample_task_payload)\n second_payload[\"task_name\"] = \"Sony A7CR\"\n second_payload[\"keyword\"] = \"sony a7cr\"\n second_payload[\"ai_prompt_criteria_file\"] = \"prompts/sony_a7cr_criteria.txt\"\n\n assert api_client.post(\"/api/tasks/\", json=sample_task_payload).status_code == 200\n assert api_client.post(\"/api/tasks/\", json=second_payload).status_code == 200\n assert api_client.post(\"/api/tasks/start/0\").status_code == 200\n\n response = api_client.delete(\"/api/tasks/0\")\n\n assert response.status_code == 200\n process_service = api_context[\"process_service\"]\n assert process_service.stopped == [0]\n assert process_service.reindexed == []\n" }, { "path": "tests/integration/test_cli_spider.py", "content": "import asyncio\nimport importlib\nimport json\nimport sys\nimport types\n\n\ndef test_cli_runs_single_task_with_prompt(tmp_path, load_json_fixture, monkeypatch):\n fake_scraper = types.ModuleType(\"src.scraper\")\n\n async def placeholder_scrape(task_config, debug_limit):\n return 0\n\n fake_scraper.scrape_xianyu = placeholder_scrape\n monkeypatch.setitem(sys.modules, \"src.scraper\", fake_scraper)\n sys.modules.pop(\"spider_v2\", None)\n\n spider_v2 = importlib.import_module(\"spider_v2\")\n config_data = load_json_fixture(\"config.sample.json\")\n\n base_prompt = \"Base prompt. \" + (\"x\" * 120) + \" {{CRITERIA_SECTION}}\"\n criteria_prompt = \"Criteria text for A7M4.\"\n\n base_path = tmp_path / \"base_prompt.txt\"\n criteria_path = tmp_path / \"criteria_prompt.txt\"\n base_path.write_text(base_prompt, encoding=\"utf-8\")\n criteria_path.write_text(criteria_prompt, encoding=\"utf-8\")\n\n config_data[0][\"ai_prompt_base_file\"] = str(base_path)\n config_data[0][\"ai_prompt_criteria_file\"] = str(criteria_path)\n\n config_data[1][\"ai_prompt_base_file\"] = str(base_path)\n config_data[1][\"ai_prompt_criteria_file\"] = str(criteria_path)\n\n config_path = tmp_path / \"config.json\"\n config_path.write_text(json.dumps(config_data, ensure_ascii=False), encoding=\"utf-8\")\n\n state_path = tmp_path / \"state.json\"\n state_path.write_text(\"{}\", encoding=\"utf-8\")\n\n monkeypatch.setattr(spider_v2, \"STATE_FILE\", str(state_path))\n\n called = []\n\n async def fake_scrape_xianyu(task_config, debug_limit):\n called.append(task_config[\"task_name\"])\n assert \"{{CRITERIA_SECTION}}\" not in task_config[\"ai_prompt_text\"]\n assert \"Criteria text for A7M4.\" in task_config[\"ai_prompt_text\"]\n return 1\n\n monkeypatch.setattr(spider_v2, \"scrape_xianyu\", fake_scrape_xianyu)\n monkeypatch.setattr(sys, \"argv\", [\"spider_v2.py\", \"--config\", str(config_path), \"--task-name\", \"Sony A7M4\"])\n\n asyncio.run(spider_v2.main())\n\n assert called == [\"Sony A7M4\"]\n\n\ndef test_cli_runs_keyword_mode_without_prompt_files(tmp_path, load_json_fixture, monkeypatch):\n fake_scraper = types.ModuleType(\"src.scraper\")\n\n async def placeholder_scrape(task_config, debug_limit):\n return 0\n\n fake_scraper.scrape_xianyu = placeholder_scrape\n monkeypatch.setitem(sys.modules, \"src.scraper\", fake_scraper)\n sys.modules.pop(\"spider_v2\", None)\n\n spider_v2 = importlib.import_module(\"spider_v2\")\n config_data = load_json_fixture(\"config.sample.json\")\n config_data[0][\"enabled\"] = True\n config_data[0][\"decision_mode\"] = \"keyword\"\n config_data[0][\"keyword_rules\"] = [\"a7m4\", \"验货宝\"]\n config_data[0][\"ai_prompt_base_file\"] = \"missing_base.txt\"\n config_data[0][\"ai_prompt_criteria_file\"] = \"missing_criteria.txt\"\n\n config_path = tmp_path / \"config.json\"\n config_path.write_text(json.dumps(config_data, ensure_ascii=False), encoding=\"utf-8\")\n\n state_path = tmp_path / \"state.json\"\n state_path.write_text(\"{}\", encoding=\"utf-8\")\n monkeypatch.setattr(spider_v2, \"STATE_FILE\", str(state_path))\n\n captured = []\n\n async def fake_scrape_xianyu(task_config, debug_limit):\n captured.append(task_config)\n return 1\n\n monkeypatch.setattr(spider_v2, \"scrape_xianyu\", fake_scrape_xianyu)\n monkeypatch.setattr(sys, \"argv\", [\"spider_v2.py\", \"--config\", str(config_path), \"--task-name\", \"Sony A7M4\"])\n\n asyncio.run(spider_v2.main())\n\n assert len(captured) == 1\n assert captured[0][\"decision_mode\"] == \"keyword\"\n assert captured[0][\"ai_prompt_text\"] == \"\"\n" }, { "path": "tests/integration/test_pipeline_parse.py", "content": "import asyncio\n\nfrom src.parsers import (\n _parse_search_results_json,\n _parse_user_items_data,\n calculate_reputation_from_ratings,\n parse_ratings_data,\n parse_user_head_data,\n)\n\n\ndef test_parse_search_results(load_json_fixture):\n raw = load_json_fixture(\"search_results.json\")\n items = asyncio.run(_parse_search_results_json(raw, source=\"search\"))\n assert len(items) == 1\n item = items[0]\n assert item[\"商品标题\"] == \"Sony A7M4 Body\"\n assert item[\"当前售价\"].startswith(\"¥\")\n assert \"包邮\" in item[\"商品标签\"]\n assert \"验货宝\" in item[\"商品标签\"]\n assert item[\"商品链接\"].startswith(\"https://www.goofish.com/\")\n\n\ndef test_parse_user_head_and_items(load_json_fixture):\n head_json = load_json_fixture(\"user_head.json\")\n items_json = load_json_fixture(\"user_items.json\")\n\n head = asyncio.run(parse_user_head_data(head_json))\n assert head[\"卖家昵称\"] == \"seller_01\"\n assert head[\"卖家收到的评价总数\"] == 88\n\n items = asyncio.run(_parse_user_items_data(items_json))\n assert items[0][\"商品状态\"] == \"在售\"\n assert items[1][\"商品状态\"] == \"已售\"\n\n\ndef test_parse_ratings_and_reputation(load_json_fixture):\n ratings_json = load_json_fixture(\"ratings.json\")\n ratings = asyncio.run(parse_ratings_data(ratings_json))\n assert ratings[0][\"评价类型\"] == \"好评\"\n\n reputation = asyncio.run(calculate_reputation_from_ratings(ratings_json))\n assert reputation[\"作为卖家的好评数\"].startswith(\"1/\")\n assert reputation[\"作为买家的好评数\"].startswith(\"1/\")\n" }, { "path": "tests/live/_support.py", "content": "from __future__ import annotations\n\nimport os\nimport shutil\nimport socket\nimport subprocess\nimport time\nfrom dataclasses import dataclass\nfrom pathlib import Path\n\nimport requests\nfrom dotenv import dotenv_values\n\n\nDEFAULT_EXPECT_MIN_ITEMS = 1\nDEFAULT_LIVE_DEBUG_LIMIT = 1\nDEFAULT_LIVE_KEYWORD = \"MacBook Pro M2\"\nDEFAULT_LIVE_TIMEOUT_SECONDS = 180\nHEALTH_TIMEOUT_SECONDS = 60\nNOTIFICATION_ENV_KEYS = (\n \"NTFY_TOPIC_URL\",\n \"GOTIFY_URL\",\n \"GOTIFY_TOKEN\",\n \"BARK_URL\",\n \"WX_BOT_URL\",\n \"TELEGRAM_BOT_TOKEN\",\n \"TELEGRAM_CHAT_ID\",\n \"TELEGRAM_API_BASE_URL\",\n \"WEBHOOK_URL\",\n \"WEBHOOK_METHOD\",\n \"WEBHOOK_HEADERS\",\n \"WEBHOOK_CONTENT_TYPE\",\n \"WEBHOOK_QUERY_PARAMETERS\",\n \"WEBHOOK_BODY\",\n)\n\n\n@dataclass(frozen=True)\nclass LiveTestSettings:\n repo_root: Path\n keyword: str\n task_name: str\n expect_min_items: int\n debug_limit: int\n timeout_seconds: int\n enable_task_generation: bool\n account_source_path: Path\n ai_test_payload: dict[str, str]\n\n\n@dataclass(frozen=True)\nclass LiveServer:\n base_url: str\n workspace: Path\n server_log_path: Path\n account_state_file: Path\n settings: LiveTestSettings\n\n\ndef env_flag(name: str, default: bool = False) -> bool:\n value = os.getenv(name)\n if value is None:\n return default\n return str(value).strip().lower() in {\"1\", \"true\", \"yes\", \"on\"}\n\n\ndef env_int(name: str, default: int) -> int:\n value = os.getenv(name)\n if value is None:\n return default\n return int(value)\n\n\ndef load_runtime_env(repo_root: Path) -> dict[str, str]:\n runtime_env = os.environ.copy()\n env_file = repo_root / \".env\"\n if not env_file.exists():\n return runtime_env\n file_values = {\n key: value\n for key, value in dotenv_values(env_file, encoding=\"utf-8\").items()\n if key and value is not None\n }\n file_values.update(runtime_env)\n return file_values\n\n\ndef build_ai_test_payload(runtime_env: dict[str, str]) -> dict[str, str]:\n payload = {\n \"OPENAI_BASE_URL\": runtime_env.get(\"OPENAI_BASE_URL\", \"\"),\n \"OPENAI_MODEL_NAME\": runtime_env.get(\"OPENAI_MODEL_NAME\", \"\"),\n }\n api_key = runtime_env.get(\"OPENAI_API_KEY\")\n if api_key:\n payload[\"OPENAI_API_KEY\"] = api_key\n proxy_url = runtime_env.get(\"PROXY_URL\")\n if proxy_url:\n payload[\"PROXY_URL\"] = proxy_url\n return payload\n\n\ndef resolve_account_source(repo_root: Path) -> Path:\n configured = os.getenv(\"LIVE_TEST_ACCOUNT_STATE_FILE\")\n if configured:\n return Path(configured).expanduser().resolve()\n state_dir = repo_root / \"state\"\n candidates = sorted(state_dir.glob(\"*.json\"))\n if not candidates:\n raise FileNotFoundError(\n \"LIVE_TEST_ACCOUNT_STATE_FILE 未设置，且 state/ 下没有可用 JSON 登录态文件。\"\n )\n return candidates[0]\n\n\ndef load_live_settings(repo_root: Path) -> LiveTestSettings:\n runtime_env = load_runtime_env(repo_root)\n return LiveTestSettings(\n repo_root=repo_root,\n keyword=os.getenv(\"LIVE_TEST_KEYWORD\", DEFAULT_LIVE_KEYWORD).strip(),\n task_name=(os.getenv(\"LIVE_TEST_TASK_NAME\", \"Live Smoke Task\").strip() or \"Live Smoke Task\"),\n expect_min_items=env_int(\"LIVE_EXPECT_MIN_ITEMS\", DEFAULT_EXPECT_MIN_ITEMS),\n debug_limit=env_int(\"LIVE_TEST_DEBUG_LIMIT\", DEFAULT_LIVE_DEBUG_LIMIT),\n timeout_seconds=env_int(\"LIVE_TIMEOUT_SECONDS\", DEFAULT_LIVE_TIMEOUT_SECONDS),\n enable_task_generation=env_flag(\"LIVE_ENABLE_TASK_GENERATION\"),\n account_source_path=resolve_account_source(repo_root),\n ai_test_payload=build_ai_test_payload(runtime_env),\n )\n\n\ndef mirror_path(source: Path, destination: Path) -> None:\n if destination.exists() or destination.is_symlink():\n return\n try:\n destination.symlink_to(source, target_is_directory=source.is_dir())\n except OSError:\n if source.is_dir():\n shutil.copytree(source, destination)\n return\n shutil.copy2(source, destination)\n\n\ndef prepare_workspace(workspace: Path, settings: LiveTestSettings) -> Path:\n for name in (\"src\", \"spider_v2.py\", \"static\", \"dist\"):\n mirror_path(settings.repo_root / name, workspace / name)\n shutil.copytree(settings.repo_root / \"prompts\", workspace / \"prompts\", dirs_exist_ok=True)\n state_dir = workspace / \"state\"\n state_dir.mkdir(parents=True, exist_ok=True)\n account_target = state_dir / settings.account_source_path.name\n shutil.copy2(settings.account_source_path, account_target)\n for name in (\"logs\", \"images\", \"data\"):\n (workspace / name).mkdir(parents=True, exist_ok=True)\n return account_target\n\n\ndef build_server_env(workspace: Path, repo_root: Path, port: int) -> dict[str, str]:\n env = load_runtime_env(repo_root)\n python_path_parts = [str(repo_root)]\n if env.get(\"PYTHONPATH\"):\n python_path_parts.append(env[\"PYTHONPATH\"])\n debug_limit = str(os.getenv(\"LIVE_TEST_DEBUG_LIMIT\", DEFAULT_LIVE_DEBUG_LIMIT)).strip()\n env.update(\n {\n \"APP_DATABASE_FILE\": str(workspace / \"data\" / \"live.sqlite3\"),\n \"ACCOUNT_STATE_DIR\": str(workspace / \"state\"),\n \"RUN_HEADLESS\": \"true\",\n \"SKIP_AI_ANALYSIS\": \"false\",\n \"AI_DEBUG_MODE\": \"true\",\n \"PYTHONUNBUFFERED\": \"1\",\n \"SERVER_PORT\": str(port),\n \"SPIDER_DEBUG_LIMIT\": debug_limit,\n \"PYTHONPATH\": os.pathsep.join(python_path_parts),\n }\n )\n for key in NOTIFICATION_ENV_KEYS:\n env[key] = \"\"\n return env\n\n\ndef find_free_port() -> int:\n with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:\n sock.bind((\"127.0.0.1\", 0))\n sock.listen(1)\n return int(sock.getsockname()[1])\n\n\ndef wait_for_server_ready(base_url: str, process: subprocess.Popen, log_path: Path) -> None:\n deadline = time.monotonic() + HEALTH_TIMEOUT_SECONDS\n last_error = \"unknown\"\n while time.monotonic() < deadline:\n if process.poll() is not None:\n break\n try:\n response = requests.get(f\"{base_url}/health\", timeout=2)\n if response.status_code == 200:\n return\n last_error = f\"health status={response.status_code} body={response.text[:200]}\"\n except requests.RequestException as exc:\n last_error = str(exc)\n time.sleep(1)\n\n log_excerpt = \"\"\n if log_path.exists():\n log_excerpt = log_path.read_text(encoding=\"utf-8\", errors=\"ignore\")[-4000:]\n raise RuntimeError(\n \"Live app 未在预期时间内启动。\"\n f\" last_error={last_error}\\nserver_log={log_path}\\n{log_excerpt}\"\n )\n\n\ndef terminate_process(process: subprocess.Popen, timeout_seconds: int = 20) -> None:\n if process.poll() is not None:\n return\n process.terminate()\n try:\n process.wait(timeout=timeout_seconds)\n except subprocess.TimeoutExpired:\n process.kill()\n process.wait(timeout=5)\n" }, { "path": "tests/live/conftest.py", "content": "from __future__ import annotations\n\nimport os\nimport shutil\nimport subprocess\nimport sys\nfrom pathlib import Path\n\nimport pytest\n\nfrom tests.live._support import (\n LiveServer,\n build_server_env,\n find_free_port,\n load_live_settings,\n prepare_workspace,\n terminate_process,\n wait_for_server_ready,\n)\n\n\nLIVE_SKIP_REASON = \"真实流量 live smoke 默认关闭；请显式设置 RUN_LIVE_TESTS=1。\"\n\n\ndef pytest_collection_modifyitems(config, items):\n if os.environ.get(\"RUN_LIVE_TESTS\") == \"1\":\n return\n skip_marker = pytest.mark.skip(reason=LIVE_SKIP_REASON)\n for item in items:\n if \"live\" not in item.keywords and \"live_slow\" not in item.keywords:\n continue\n item.add_marker(skip_marker)\n\n\n@pytest.fixture(scope=\"session\")\ndef live_settings():\n if os.environ.get(\"RUN_LIVE_TESTS\") != \"1\":\n pytest.skip(LIVE_SKIP_REASON)\n repo_root = Path(__file__).resolve().parents[2]\n settings = load_live_settings(repo_root)\n if not settings.account_source_path.exists():\n pytest.fail(f\"live 登录态文件不存在: {settings.account_source_path}\")\n if not settings.ai_test_payload.get(\"OPENAI_BASE_URL\") or not settings.ai_test_payload.get(\n \"OPENAI_MODEL_NAME\"\n ):\n pytest.fail(\"live 测试需要 OPENAI_BASE_URL 与 OPENAI_MODEL_NAME。\")\n return settings\n\n\n@pytest.fixture(scope=\"session\")\ndef live_server(live_settings, request, tmp_path_factory):\n workspace = tmp_path_factory.mktemp(\"live-smoke\")\n account_state_file = prepare_workspace(workspace, live_settings)\n port = find_free_port()\n base_url = f\"http://127.0.0.1:{port}\"\n log_path = workspace / \"live-app.log\"\n env = build_server_env(workspace, live_settings.repo_root, port)\n log_handle = log_path.open(\"w\", encoding=\"utf-8\")\n process = subprocess.Popen(\n [\n sys.executable,\n \"-m\",\n \"uvicorn\",\n \"src.app:app\",\n \"--host\",\n \"127.0.0.1\",\n \"--port\",\n str(port),\n ],\n cwd=workspace,\n env=env,\n stdout=log_handle,\n stderr=subprocess.STDOUT,\n text=True,\n )\n try:\n wait_for_server_ready(base_url, process, log_path)\n except Exception:\n terminate_process(process)\n log_handle.close()\n raise\n\n server = LiveServer(\n base_url=base_url,\n workspace=workspace,\n server_log_path=log_path,\n account_state_file=account_state_file,\n settings=live_settings,\n )\n try:\n yield server\n finally:\n terminate_process(process)\n log_handle.close()\n if request.session.testsfailed:\n print(f\"live smoke 失败，保留工作目录供排查: {workspace}\")\n return\n shutil.rmtree(workspace, ignore_errors=True)\n" }, { "path": "tests/live/test_live_smoke.py", "content": "from __future__ import annotations\n\nimport time\nfrom pathlib import Path\n\nimport pytest\nimport requests\n\nfrom src.infrastructure.persistence.storage_names import build_result_filename\n\n\npytestmark = pytest.mark.live\n\nREQUEST_TIMEOUT_SECONDS = 60\nTASK_POLL_INTERVAL_SECONDS = 2\nFORBIDDEN_LOG_MARKERS = (\n \"Login required\",\n \"passport.goofish.com\",\n \"FAIL_SYS_USER_VALIDATE\",\n \"AI客户端未初始化\",\n \"未找到可用的登录状态文件\",\n \"检测到登录失效/重定向\",\n)\n\ndef api_request(session: requests.Session, method: str, url: str, **kwargs) -> requests.Response:\n kwargs.setdefault(\"timeout\", REQUEST_TIMEOUT_SECONDS)\n return session.request(method=method, url=url, **kwargs)\n\ndef fetch_task(session: requests.Session, base_url: str, task_id: int) -> dict:\n response = api_request(session, \"get\", f\"{base_url}/api/tasks/{task_id}\")\n assert response.status_code == 200, response.text\n return response.json()\n\ndef fetch_results_or_none(\n session: requests.Session,\n base_url: str,\n filename: str,\n *,\n limit: int = 5,\n) -> dict | None:\n response = api_request(\n session,\n \"get\",\n f\"{base_url}/api/results/{filename}\",\n params={\"page\": 1, \"limit\": limit},\n )\n if response.status_code == 404:\n return None\n assert response.status_code == 200, response.text\n return response.json()\n\ndef find_task_log(workspace: Path, task_id: int) -> Path | None:\n log_dir = workspace / \"logs\"\n matches = sorted(log_dir.glob(f\"*_{task_id}.log\"))\n return matches[0] if matches else None\n\ndef read_task_log(workspace: Path, task_id: int) -> tuple[Path | None, str]:\n log_path = find_task_log(workspace, task_id)\n if log_path is None:\n return None, \"\"\n return log_path, log_path.read_text(encoding=\"utf-8\", errors=\"ignore\")\n\ndef assert_log_is_clean(log_text: str, log_path: Path | None) -> None:\n assert log_path is not None, \"live 任务日志不存在。\"\n for marker in FORBIDDEN_LOG_MARKERS:\n assert marker not in log_text, f\"日志包含失败标记 {marker}，请检查 {log_path}\"\n\ndef wait_for_task_running(\n session: requests.Session,\n base_url: str,\n task_id: int,\n timeout_seconds: int,\n) -> dict:\n deadline = time.monotonic() + timeout_seconds\n last_task = {}\n while time.monotonic() < deadline:\n last_task = fetch_task(session, base_url, task_id)\n if last_task.get(\"is_running\"):\n return last_task\n time.sleep(TASK_POLL_INTERVAL_SECONDS)\n pytest.fail(f\"任务 {task_id} 未在预期时间内进入运行态: {last_task}\")\n\ndef wait_for_task_completion(\n session: requests.Session,\n base_url: str,\n task_id: int,\n filename: str,\n expect_min_items: int,\n timeout_seconds: int,\n workspace: Path,\n) -> tuple[dict, dict | None]:\n deadline = time.monotonic() + timeout_seconds\n last_task = {}\n last_results = None\n stop_sent = False\n while time.monotonic() < deadline:\n last_task = fetch_task(session, base_url, task_id)\n last_results = fetch_results_or_none(session, base_url, filename)\n if (\n last_results\n and last_results.get(\"total_items\", 0) >= expect_min_items\n and last_task.get(\"is_running\")\n and not stop_sent\n ):\n stop_response = api_request(session, \"post\", f\"{base_url}/api/tasks/stop/{task_id}\")\n assert stop_response.status_code == 200, stop_response.text\n stop_sent = True\n if not last_task.get(\"is_running\"):\n return last_task, last_results\n time.sleep(TASK_POLL_INTERVAL_SECONDS)\n\n log_path, log_text = read_task_log(workspace, task_id)\n pytest.fail(\n f\"任务 {task_id} 在 {timeout_seconds}s 内未结束。log={log_path}\\n{log_text[-4000:]}\"\n )\n\ndef delete_task_safely(session: requests.Session, base_url: str, task_id: int) -> None:\n response = api_request(session, \"delete\", f\"{base_url}/api/tasks/{task_id}\")\n assert response.status_code in {200, 404}, response.text\n\ndef build_live_task_payload(account_state_file: Path, task_name: str, keyword: str) -> dict:\n return {\n \"task_name\": task_name,\n \"enabled\": True,\n \"keyword\": keyword,\n \"description\": \"Live smoke task for real Goofish traffic and real AI response validation.\",\n \"analyze_images\": False,\n \"max_pages\": 1,\n \"personal_only\": True,\n \"ai_prompt_base_file\": \"prompts/base_prompt.txt\",\n \"ai_prompt_criteria_file\": \"prompts/macbook_criteria.txt\",\n \"account_state_file\": str(account_state_file),\n \"account_strategy\": \"fixed\",\n \"decision_mode\": \"ai\",\n }\n\ndef test_live_preflight_smoke(live_server):\n with requests.Session() as session:\n health_response = api_request(session, \"get\", f\"{live_server.base_url}/health\")\n assert health_response.status_code == 200, health_response.text\n assert health_response.json()[\"status\"] == \"healthy\"\n\n ai_response = api_request(\n session,\n \"post\",\n f\"{live_server.base_url}/api/settings/ai/test\",\n json=live_server.settings.ai_test_payload,\n )\n assert ai_response.status_code == 200, ai_response.text\n ai_result = ai_response.json()\n assert ai_result[\"success\"] is True, ai_result\n assert live_server.account_state_file.exists()\n\ndef test_live_real_traffic_task_smoke(live_server):\n task_name = live_server.settings.task_name\n keyword = live_server.settings.keyword\n filename = build_result_filename(keyword)\n payload = build_live_task_payload(live_server.account_state_file, task_name, keyword)\n\n with requests.Session() as session:\n create_response = api_request(\n session,\n \"post\",\n f\"{live_server.base_url}/api/tasks/\",\n json=payload,\n )\n assert create_response.status_code == 200, create_response.text\n created_task = create_response.json()[\"task\"]\n task_id = created_task[\"id\"]\n\n try:\n start_response = api_request(\n session,\n \"post\",\n f\"{live_server.base_url}/api/tasks/start/{task_id}\",\n )\n assert start_response.status_code == 200, start_response.text\n\n final_task, result_data = wait_for_task_completion(\n session,\n live_server.base_url,\n task_id,\n filename,\n live_server.settings.expect_min_items,\n live_server.settings.timeout_seconds,\n live_server.workspace,\n )\n assert final_task[\"is_running\"] is False\n\n files_response = api_request(session, \"get\", f\"{live_server.base_url}/api/results/files\")\n assert files_response.status_code == 200, files_response.text\n assert filename in files_response.json()[\"files\"]\n\n if result_data is None:\n result_data = fetch_results_or_none(session, live_server.base_url, filename)\n assert result_data is not None, f\"结果文件 {filename} 未生成。\"\n assert result_data[\"total_items\"] >= live_server.settings.expect_min_items\n\n item = result_data[\"items\"][0]\n product = item.get(\"商品信息\", {})\n analysis = item.get(\"ai_analysis\", {})\n assert product.get(\"商品标题\"), item\n assert product.get(\"商品链接\"), item\n assert product.get(\"当前售价\"), item\n assert analysis, item\n assert analysis.get(\"analysis_source\") == \"ai\", item\n\n log_path, log_text = read_task_log(live_server.workspace, task_id)\n assert_log_is_clean(log_text, log_path)\n finally:\n delete_task_safely(session, live_server.base_url, task_id)\n\n\n@pytest.mark.live_slow\ndef test_live_ai_task_generation_job(live_server):\n if not live_server.settings.enable_task_generation:\n pytest.skip(\"未设置 LIVE_ENABLE_TASK_GENERATION=1，跳过真实 AI 任务生成测试。\")\n\n payload = {\n \"task_name\": f\"{live_server.settings.task_name} Generated\",\n \"keyword\": live_server.settings.keyword,\n \"description\": \"Generate a practical second-hand inspection criteria for live smoke validation.\",\n \"analyze_images\": False,\n \"max_pages\": 1,\n \"personal_only\": True,\n \"account_state_file\": str(live_server.account_state_file),\n \"account_strategy\": \"fixed\",\n \"decision_mode\": \"ai\",\n }\n\n with requests.Session() as session:\n response = api_request(\n session,\n \"post\",\n f\"{live_server.base_url}/api/tasks/generate\",\n json=payload,\n )\n assert response.status_code == 202, response.text\n job = response.json()[\"job\"]\n job_id = job[\"job_id\"]\n\n deadline = time.monotonic() + live_server.settings.timeout_seconds\n latest_job = job\n while time.monotonic() < deadline:\n status_response = api_request(\n session,\n \"get\",\n f\"{live_server.base_url}/api/tasks/generate-jobs/{job_id}\",\n )\n assert status_response.status_code == 200, status_response.text\n latest_job = status_response.json()[\"job\"]\n if latest_job[\"status\"] == \"completed\":\n break\n if latest_job[\"status\"] == \"failed\":\n pytest.fail(f\"真实 AI 任务生成失败: {latest_job}\")\n time.sleep(TASK_POLL_INTERVAL_SECONDS)\n else:\n pytest.fail(f\"真实 AI 任务生成超时: {latest_job}\")\n\n task = latest_job[\"task\"]\n assert task[\"ai_prompt_criteria_file\"]\n task_id = task[\"id\"]\n\n try:\n start_response = api_request(\n session,\n \"post\",\n f\"{live_server.base_url}/api/tasks/start/{task_id}\",\n )\n assert start_response.status_code == 200, start_response.text\n wait_for_task_running(\n session,\n live_server.base_url,\n task_id,\n timeout_seconds=min(live_server.settings.timeout_seconds, 30),\n )\n stop_response = api_request(\n session,\n \"post\",\n f\"{live_server.base_url}/api/tasks/stop/{task_id}\",\n )\n assert stop_response.status_code == 200, stop_response.text\n final_task, _ = wait_for_task_completion(\n session,\n live_server.base_url,\n task_id,\n build_result_filename(live_server.settings.keyword),\n expect_min_items=0,\n timeout_seconds=min(live_server.settings.timeout_seconds, 60),\n workspace=live_server.workspace,\n )\n assert final_task[\"is_running\"] is False\n finally:\n delete_task_safely(session, live_server.base_url, task_id)\n" }, { "path": "tests/test_failure_guard.py", "content": "from __future__ import annotations\n\nfrom datetime import datetime, timedelta\n\nfrom src.failure_guard import FailureGuard\n\n\ndef test_failure_guard_opens_circuit_after_threshold_and_rate_limits(tmp_path):\n guard_path = tmp_path / \"guard.json\"\n cookie_path = tmp_path / \"xianyu_state.json\"\n cookie_path.write_text(\"{}\", encoding=\"utf-8\")\n\n guard = FailureGuard(\n path=str(guard_path),\n threshold=3,\n pause_seconds=3 * 24 * 60 * 60,\n tz_name=\"Asia/Shanghai\",\n )\n\n base = datetime(2026, 3, 4, 12, 0, 0)\n\n r1 = guard.record_failure(\"task-a\", \"err-1\", cookie_path=str(cookie_path), now=base)\n assert r1[\"should_notify\"] is False\n assert r1[\"opened_circuit\"] is False\n\n r2 = guard.record_failure(\"task-a\", \"err-2\", cookie_path=str(cookie_path), now=base)\n assert r2[\"should_notify\"] is False\n assert r2[\"opened_circuit\"] is False\n\n r3 = guard.record_failure(\"task-a\", \"err-3\", cookie_path=str(cookie_path), now=base)\n assert r3[\"should_notify\"] is True\n assert r3[\"opened_circuit\"] is True\n assert r3[\"paused_until\"] is not None\n\n d0 = guard.should_skip_start(\"task-a\", cookie_path=str(cookie_path), now=base)\n assert d0.skip is True\n assert d0.should_notify is False\n\n next_day = base + timedelta(days=1, minutes=1)\n d1 = guard.should_skip_start(\"task-a\", cookie_path=str(cookie_path), now=next_day)\n assert d1.skip is True\n assert d1.should_notify is True\n\n d1b = guard.should_skip_start(\"task-a\", cookie_path=str(cookie_path), now=next_day)\n assert d1b.skip is True\n assert d1b.should_notify is False\n\n\ndef test_failure_guard_auto_recovers_on_cookie_change(tmp_path):\n guard_path = tmp_path / \"guard.json\"\n cookie_path = tmp_path / \"xianyu_state.json\"\n cookie_path.write_text(\"{}\", encoding=\"utf-8\")\n\n guard = FailureGuard(\n path=str(guard_path),\n threshold=2,\n pause_seconds=3 * 24 * 60 * 60,\n tz_name=\"Asia/Shanghai\",\n )\n\n base = datetime(2026, 3, 4, 12, 0, 0)\n\n guard.record_failure(\"task-a\", \"err-1\", cookie_path=str(cookie_path), now=base)\n guard.record_failure(\"task-a\", \"err-2\", cookie_path=str(cookie_path), now=base)\n\n paused = guard.should_skip_start(\"task-a\", cookie_path=str(cookie_path), now=base)\n assert paused.skip is True\n\n cookie_path.write_text('{\"updated\": true}', encoding=\"utf-8\")\n\n recovered = guard.should_skip_start(\n \"task-a\",\n cookie_path=str(cookie_path),\n now=base + timedelta(minutes=1),\n )\n assert recovered.skip is False\n" }, { "path": "tests/test_frontend_build_paths.py", "content": "from __future__ import annotations\n\nfrom pathlib import Path\n\n\nREPO_ROOT = Path(__file__).resolve().parents[1]\nROOT_DIST = \"/dist\"\n\n\ndef read_repo_file(relative_path: str) -> str:\n return (REPO_ROOT / relative_path).read_text(encoding=\"utf-8\")\n\n\ndef test_frontend_build_output_path_is_consistent_across_configs():\n vite_config = read_repo_file(\"web-ui/vite.config.ts\")\n dockerfile = read_repo_file(\"Dockerfile\")\n frontend_dockerfile = read_repo_file(\"web-ui/Dockerfile\")\n dockerignore = read_repo_file(\".dockerignore\")\n start_script = read_repo_file(\"start.sh\")\n dockerignore_lines = dockerignore.splitlines()\n\n assert \"path.resolve(__dirname, '../dist')\" in vite_config\n assert (\n f\"COPY --from=frontend-builder {ROOT_DIST} /app/dist\" in dockerfile\n ), \"Docker multi-stage copy must use the Vite build output path.\"\n assert (\n f\"COPY --from=builder {ROOT_DIST} /usr/share/nginx/html\"\n in frontend_dockerfile\n ), \"Frontend-only Docker build must use the Vite build output path.\"\n assert \"dist/\" in dockerignore_lines\n assert \"web-ui/dist\" not in dockerignore_lines\n assert '[ ! -d \"dist\" ]' in start_script\n assert \"cp -r web-ui/dist ./\" not in start_script\n" }, { "path": "tests/unit/test_ai_client.py", "content": "import asyncio\nfrom types import SimpleNamespace\n\nimport pytest\n\nfrom src.infrastructure.external.ai_client import AIClient\nfrom src.services.ai_request_compat import build_responses_input\n\n\ndef _build_fake_client(responses_create_impl, chat_create_impl=None):\n responses = SimpleNamespace(create=responses_create_impl)\n chat = SimpleNamespace(\n completions=SimpleNamespace(create=chat_create_impl or responses_create_impl)\n )\n return SimpleNamespace(responses=responses, chat=chat)\n\n\ndef test_build_messages_without_images_uses_text_only_content():\n client = AIClient.__new__(AIClient)\n\n messages = client._build_messages(\n {\"商品信息\": {\"商品标题\": \"MacBook Pro M2\"}, \"卖家信息\": {\"卖家信用等级\": \"优秀\"}},\n [],\n \"只分析文字描述和卖家资质。\",\n )\n\n content = messages[0][\"content\"]\n assert isinstance(content, str)\n assert \"MacBook Pro M2\" in content\n assert \"未提供商品图片\" in content\n\n\ndef test_build_messages_with_images_uses_multimodal_content(monkeypatch):\n client = AIClient.__new__(AIClient)\n monkeypatch.setattr(AIClient, \"encode_image\", staticmethod(lambda _path: \"ZmFrZQ==\"))\n\n messages = client._build_messages(\n {\"商品信息\": {\"商品标题\": \"MacBook Pro M2\"}},\n [\"fake-image.jpg\"],\n \"结合图片和文字综合判断。\",\n )\n\n content = messages[0][\"content\"]\n assert isinstance(content, list)\n assert content[0][\"type\"] == \"image_url\"\n assert content[-1][\"type\"] == \"text\"\n\n\ndef test_build_responses_input_converts_multimodal_messages():\n result = build_responses_input(\n [\n {\n \"role\": \"user\",\n \"content\": [\n {\"type\": \"image_url\", \"image_url\": {\"url\": \"data:image/jpeg;base64,ZmFrZQ==\"}},\n {\"type\": \"text\", \"text\": \"hello\"},\n ],\n }\n ]\n )\n\n assert result == [\n {\n \"role\": \"user\",\n \"content\": [\n {\n \"type\": \"input_image\",\n \"image_url\": \"data:image/jpeg;base64,ZmFrZQ==\",\n \"detail\": \"auto\",\n },\n {\"type\": \"input_text\", \"text\": \"hello\"},\n ],\n }\n ]\n\n\ndef test_call_ai_retries_without_structured_output_when_model_rejects_it():\n client = AIClient.__new__(AIClient)\n client.settings = SimpleNamespace(\n model_name=\"fake-model\",\n enable_response_format=True,\n enable_thinking=False,\n )\n request_history = []\n\n async def fake_create(**kwargs):\n request_history.append(kwargs)\n if len(request_history) == 1:\n raise Exception(\n \"Error code: 400 - {'error': {'code': 'InvalidParameter', \"\n \"'message': 'The parameter `response_format.type` specified in \"\n \"the request are not valid: `json_object` is not supported by \"\n \"this model.', 'param': 'response_format.type'}}\"\n )\n return SimpleNamespace(\n choices=[\n SimpleNamespace(\n message=SimpleNamespace(content='{\"ok\":true}')\n )\n ]\n )\n\n client.client = _build_fake_client(fake_create)\n\n response = asyncio.run(client._call_ai([{\"role\": \"user\", \"content\": \"hi\"}]))\n\n assert response == '{\"ok\":true}'\n assert request_history[0][\"messages\"][0][\"content\"] == \"hi\"\n assert request_history[0][\"response_format\"][\"type\"] == \"json_object\"\n assert \"response_format\" not in request_history[1]\n\n\ndef test_call_ai_falls_back_to_responses_when_chat_completions_api_is_missing():\n client = AIClient.__new__(AIClient)\n client.settings = SimpleNamespace(\n model_name=\"fake-model\",\n enable_response_format=True,\n enable_thinking=False,\n )\n request_history = []\n\n async def fake_chat_create(**kwargs):\n request_history.append((\"chat\", kwargs))\n raise Exception(\"Error code: 404 - page not found\")\n\n async def fake_responses_create(**kwargs):\n request_history.append((\"responses\", kwargs))\n if len([item for item in request_history if item[0] == \"responses\"]) == 1:\n raise Exception(\n \"Error code: 400 - {'error': {'code': 'InvalidParameter', \"\n \"'message': 'The parameter `text.format.type` specified in \"\n \"the request are not valid: `json_object` is not supported by \"\n \"this model.', 'param': 'text.format.type'}}\"\n )\n return SimpleNamespace(output_text='{\"ok\":true}')\n\n client.client = _build_fake_client(fake_responses_create, fake_chat_create)\n\n response = asyncio.run(client._call_ai([{\"role\": \"user\", \"content\": \"hi\"}]))\n\n assert response == '{\"ok\":true}'\n assert request_history[0][0] == \"chat\"\n assert request_history[1][0] == \"responses\"\n assert request_history[1][1][\"text\"][\"format\"][\"type\"] == \"json_object\"\n assert request_history[2][0] == \"responses\"\n assert \"text\" not in request_history[2][1]\n\n\ndef test_call_ai_retries_without_temperature_when_gateway_rejects_it():\n client = AIClient.__new__(AIClient)\n client.settings = SimpleNamespace(\n model_name=\"fake-model\",\n enable_response_format=False,\n enable_thinking=False,\n )\n request_history = []\n\n async def fake_create(**kwargs):\n request_history.append(kwargs)\n if len(request_history) == 1:\n raise Exception(\"temperature is not supported by this gateway\")\n return SimpleNamespace(\n choices=[\n SimpleNamespace(\n message=SimpleNamespace(content='{\"ok\":true}')\n )\n ]\n )\n\n client.client = _build_fake_client(fake_create)\n\n response = asyncio.run(client._call_ai([{\"role\": \"user\", \"content\": \"hi\"}]))\n\n assert response == '{\"ok\":true}'\n assert request_history[0][\"temperature\"] == 0.1\n assert \"temperature\" not in request_history[1]\n\n\ndef test_call_ai_retries_when_response_content_is_empty():\n client = AIClient.__new__(AIClient)\n client.settings = SimpleNamespace(\n model_name=\"fake-model\",\n enable_response_format=False,\n enable_thinking=False,\n )\n request_history = []\n\n async def fake_create(**kwargs):\n request_history.append(kwargs)\n if len(request_history) < 4:\n return SimpleNamespace(output_text=\"\")\n return SimpleNamespace(output_text='{\"ok\":true}')\n\n client.client = _build_fake_client(fake_create)\n\n response = asyncio.run(client._call_ai([{\"role\": \"user\", \"content\": \"hi\"}]))\n\n assert response == '{\"ok\":true}'\n assert len(request_history) == 4\n\n\ndef test_call_ai_raises_after_all_empty_response_retries_are_exhausted():\n client = AIClient.__new__(AIClient)\n client.settings = SimpleNamespace(\n model_name=\"fake-model\",\n enable_response_format=False,\n enable_thinking=False,\n )\n request_history = []\n\n async def fake_create(**kwargs):\n request_history.append(kwargs)\n return SimpleNamespace(output_text=\"\")\n\n client.client = _build_fake_client(fake_create)\n\n with pytest.raises(ValueError, match=\"AI响应内容为空\"):\n asyncio.run(client._call_ai([{\"role\": \"user\", \"content\": \"hi\"}]))\n\n assert len(request_history) == 4\n\n\ndef test_close_closes_underlying_async_client_and_clears_reference():\n client = AIClient.__new__(AIClient)\n close_state = {\"closed\": False}\n\n async def fake_close():\n close_state[\"closed\"] = True\n\n client.client = SimpleNamespace(close=fake_close)\n\n asyncio.run(client.close())\n\n assert close_state[\"closed\"] is True\n assert client.client is None\n\n\ndef test_parse_response_uses_first_json_object_when_response_contains_multiple_objects():\n client = AIClient.__new__(AIClient)\n\n result = client._parse_response(\"\"\"```json\n{\"ok\": true, \"reason\": \"first\"}\n{\"ok\": false, \"reason\": \"second\"}\n```\"\"\")\n\n assert result == {\"ok\": True, \"reason\": \"first\"}\n" }, { "path": "tests/unit/test_ai_handler_analysis.py", "content": "import asyncio\nfrom types import SimpleNamespace\n\nimport pytest\n\nimport src.ai_handler as ai_handler\nimport src.config as app_config\n\n\ndef _build_fake_client(responses_create_impl, chat_create_impl=None):\n responses = SimpleNamespace(create=responses_create_impl)\n chat = SimpleNamespace(\n completions=SimpleNamespace(create=chat_create_impl or responses_create_impl)\n )\n return SimpleNamespace(responses=responses, chat=chat)\n\n\ndef test_get_ai_analysis_stops_after_internal_retries_when_content_is_none(\n monkeypatch, tmp_path\n):\n monkeypatch.chdir(tmp_path)\n call_count = {\"value\": 0}\n\n async def fake_create(**_kwargs):\n call_count[\"value\"] += 1\n return SimpleNamespace(output_text=\"\")\n\n monkeypatch.setattr(ai_handler, \"client\", _build_fake_client(fake_create))\n monkeypatch.setattr(ai_handler, \"MODEL_NAME\", \"fake-model\")\n monkeypatch.setattr(ai_handler, \"ENABLE_RESPONSE_FORMAT\", True)\n monkeypatch.setattr(app_config, \"ENABLE_RESPONSE_FORMAT\", True)\n\n with pytest.raises(ValueError, match=\"AI响应内容为空\"):\n asyncio.run(\n ai_handler.get_ai_analysis(\n {\"商品信息\": {\"商品ID\": \"1\", \"商品标题\": \"测试商品\"}},\n image_paths=[],\n prompt_text=\"请输出 JSON\",\n )\n )\n\n assert call_count[\"value\"] == 4\n\n\ndef test_get_ai_analysis_returns_parsed_json(monkeypatch, tmp_path):\n monkeypatch.chdir(tmp_path)\n call_count = {\"value\": 0}\n\n async def fake_create(**_kwargs):\n call_count[\"value\"] += 1\n return SimpleNamespace(\n output_text=(\n '{\"prompt_version\":\"v1\",\"is_recommended\":true,'\n '\"reason\":\"ok\",\"risk_tags\":[],\"criteria_analysis\":{\"seller_type\":\"个人\"}}'\n )\n )\n\n monkeypatch.setattr(ai_handler, \"client\", _build_fake_client(fake_create))\n monkeypatch.setattr(ai_handler, \"MODEL_NAME\", \"fake-model\")\n monkeypatch.setattr(ai_handler, \"ENABLE_RESPONSE_FORMAT\", True)\n monkeypatch.setattr(app_config, \"ENABLE_RESPONSE_FORMAT\", True)\n\n result = asyncio.run(\n ai_handler.get_ai_analysis(\n {\"商品信息\": {\"商品ID\": \"2\", \"商品标题\": \"测试商品2\"}},\n image_paths=[],\n prompt_text=\"请输出 JSON\",\n )\n )\n\n assert result[\"is_recommended\"] is True\n assert call_count[\"value\"] == 1\n\n\ndef test_get_ai_analysis_retries_without_structured_output_when_model_rejects_it(\n monkeypatch, tmp_path\n):\n monkeypatch.chdir(tmp_path)\n request_history = []\n\n async def fake_create(**kwargs):\n request_history.append(kwargs)\n if len(request_history) == 1:\n raise Exception(\n \"Error code: 400 - {'error': {'code': 'InvalidParameter', \"\n \"'message': 'The parameter `response_format.type` specified in \"\n \"the request are not valid: `json_object` is not supported by \"\n \"this model.', 'param': 'response_format.type'}}\"\n )\n return SimpleNamespace(\n choices=[\n SimpleNamespace(\n message=SimpleNamespace(\n content=(\n '{\"prompt_version\":\"v1\",\"is_recommended\":true,'\n '\"reason\":\"ok\",\"risk_tags\":[],\"criteria_analysis\":{\"seller_type\":\"个人\"}}'\n )\n )\n )\n ]\n )\n\n monkeypatch.setattr(ai_handler, \"client\", _build_fake_client(fake_create))\n monkeypatch.setattr(ai_handler, \"MODEL_NAME\", \"fake-model\")\n monkeypatch.setattr(ai_handler, \"ENABLE_RESPONSE_FORMAT\", True)\n monkeypatch.setattr(app_config, \"ENABLE_RESPONSE_FORMAT\", True)\n\n result = asyncio.run(\n ai_handler.get_ai_analysis(\n {\"商品信息\": {\"商品ID\": \"3\", \"商品标题\": \"测试商品3\"}},\n image_paths=[],\n prompt_text=\"请输出 JSON\",\n )\n )\n\n assert result[\"reason\"] == \"ok\"\n assert request_history[0][\"messages\"][0][\"role\"] == \"user\"\n assert request_history[0][\"response_format\"][\"type\"] == \"json_object\"\n assert \"response_format\" not in request_history[1]\n assert ai_handler.ENABLE_RESPONSE_FORMAT is True\n\n\ndef test_get_ai_analysis_falls_back_to_responses_when_chat_completions_api_is_missing(\n monkeypatch, tmp_path\n):\n monkeypatch.chdir(tmp_path)\n request_history = []\n\n async def fake_chat_create(**kwargs):\n request_history.append((\"chat\", kwargs))\n raise Exception(\"Error code: 404 - page not found\")\n\n async def fake_responses_create(**kwargs):\n request_history.append((\"responses\", kwargs))\n if len([item for item in request_history if item[0] == \"responses\"]) == 1:\n raise Exception(\n \"Error code: 400 - {'error': {'code': 'InvalidParameter', \"\n \"'message': 'The parameter `text.format.type` specified in \"\n \"the request are not valid: `json_object` is not supported by \"\n \"this model.', 'param': 'text.format.type'}}\"\n )\n return SimpleNamespace(\n output_text=(\n '{\"prompt_version\":\"v1\",\"is_recommended\":true,'\n '\"reason\":\"ok\",\"risk_tags\":[],\"criteria_analysis\":{\"seller_type\":\"个人\"}}'\n )\n )\n\n monkeypatch.setattr(\n ai_handler,\n \"client\",\n _build_fake_client(fake_responses_create, fake_chat_create),\n )\n monkeypatch.setattr(ai_handler, \"MODEL_NAME\", \"fake-model\")\n monkeypatch.setattr(ai_handler, \"ENABLE_RESPONSE_FORMAT\", True)\n monkeypatch.setattr(app_config, \"ENABLE_RESPONSE_FORMAT\", True)\n\n result = asyncio.run(\n ai_handler.get_ai_analysis(\n {\"商品信息\": {\"商品ID\": \"4\", \"商品标题\": \"测试商品4\"}},\n image_paths=[],\n prompt_text=\"请输出 JSON\",\n )\n )\n\n assert result[\"reason\"] == \"ok\"\n assert request_history[0][0] == \"chat\"\n assert request_history[0][1][\"messages\"][0][\"role\"] == \"user\"\n assert request_history[1][0] == \"responses\"\n assert request_history[1][1][\"text\"][\"format\"][\"type\"] == \"json_object\"\n assert request_history[2][0] == \"responses\"\n assert \"text\" not in request_history[2][1]\n\n\ndef test_get_ai_analysis_retries_without_temperature_when_gateway_rejects_it(\n monkeypatch, tmp_path\n):\n monkeypatch.chdir(tmp_path)\n request_history = []\n\n async def fake_create(**kwargs):\n request_history.append(kwargs)\n if len(request_history) == 1:\n raise Exception(\"temperature is unsupported for this model\")\n return SimpleNamespace(\n choices=[\n SimpleNamespace(\n message=SimpleNamespace(\n content=(\n '{\"prompt_version\":\"v1\",\"is_recommended\":true,'\n '\"reason\":\"ok\",\"risk_tags\":[],\"criteria_analysis\":{\"seller_type\":\"个人\"}}'\n )\n )\n )\n ]\n )\n\n monkeypatch.setattr(ai_handler, \"client\", _build_fake_client(fake_create))\n monkeypatch.setattr(ai_handler, \"MODEL_NAME\", \"fake-model\")\n monkeypatch.setattr(ai_handler, \"ENABLE_RESPONSE_FORMAT\", True)\n monkeypatch.setattr(app_config, \"ENABLE_RESPONSE_FORMAT\", True)\n\n result = asyncio.run(\n ai_handler.get_ai_analysis(\n {\"商品信息\": {\"商品ID\": \"4\", \"商品标题\": \"测试商品4\"}},\n image_paths=[],\n prompt_text=\"请输出 JSON\",\n )\n )\n\n assert result[\"reason\"] == \"ok\"\n assert request_history[0][\"temperature\"] == 0.1\n assert \"temperature\" not in request_history[1]\n\n\ndef test_get_ai_analysis_uses_first_json_object_when_model_returns_multiple_objects(\n monkeypatch, tmp_path\n):\n monkeypatch.chdir(tmp_path)\n\n async def fake_create(**_kwargs):\n return SimpleNamespace(\n output_text=\"\"\"```json\n{\"prompt_version\":\"v1\",\"is_recommended\":true,\"reason\":\"first\",\"risk_tags\":[],\"criteria_analysis\":{\"seller_type\":\"个人\"}}\n{\"prompt_version\":\"v1\",\"is_recommended\":false,\"reason\":\"second\",\"risk_tags\":[],\"criteria_analysis\":{\"seller_type\":\"商家\"}}\n```\"\"\"\n )\n\n monkeypatch.setattr(ai_handler, \"client\", _build_fake_client(fake_create))\n monkeypatch.setattr(ai_handler, \"MODEL_NAME\", \"fake-model\")\n monkeypatch.setattr(ai_handler, \"ENABLE_RESPONSE_FORMAT\", True)\n monkeypatch.setattr(app_config, \"ENABLE_RESPONSE_FORMAT\", True)\n\n result = asyncio.run(\n ai_handler.get_ai_analysis(\n {\"商品信息\": {\"商品ID\": \"5\", \"商品标题\": \"测试商品5\"}},\n image_paths=[],\n prompt_text=\"请输出 JSON\",\n )\n )\n\n assert result[\"is_recommended\"] is True\n assert result[\"reason\"] == \"first\"\n" }, { "path": "tests/unit/test_ai_handler_downloads.py", "content": "import asyncio\nfrom pathlib import Path\n\nimport src.ai_handler as ai_handler\n\n\ndef test_download_all_images_runs_with_concurrency(tmp_path, monkeypatch):\n monkeypatch.setattr(ai_handler, \"IMAGE_SAVE_DIR\", str(tmp_path / \"images\"))\n\n active_downloads = 0\n max_active_downloads = 0\n\n async def fake_download(url, save_path):\n nonlocal active_downloads, max_active_downloads\n active_downloads += 1\n max_active_downloads = max(max_active_downloads, active_downloads)\n await asyncio.sleep(0.02)\n Path(save_path).parent.mkdir(parents=True, exist_ok=True)\n Path(save_path).write_text(\"ok\", encoding=\"utf-8\")\n active_downloads -= 1\n return save_path\n\n monkeypatch.setattr(ai_handler, \"_download_single_image\", fake_download)\n\n async def run():\n return await ai_handler.download_all_images(\n \"product-1\",\n [\n \"https://example.com/1.jpg\",\n \"https://example.com/2.jpg\",\n \"https://example.com/3.jpg\",\n ],\n task_name=\"demo\",\n concurrency=3,\n )\n\n paths = asyncio.run(run())\n assert len(paths) == 3\n assert max_active_downloads == 3\n" }, { "path": "tests/unit/test_ai_request_compat.py", "content": "from src.services.ai_request_compat import (\n is_responses_api_unsupported_error,\n is_temperature_unsupported_error,\n remove_temperature_param,\n)\n\n\ndef test_is_temperature_unsupported_error_detects_unsupported_message():\n err = Exception(\"temperature is not supported by this gateway\")\n assert is_temperature_unsupported_error(err) is True\n\n\ndef test_remove_temperature_param_removes_only_temperature():\n params = {\"model\": \"x\", \"temperature\": 0.5, \"max_output_tokens\": 128}\n result = remove_temperature_param(params)\n\n assert \"temperature\" not in result\n assert result[\"model\"] == \"x\"\n assert result[\"max_output_tokens\"] == 128\n\n\ndef test_is_responses_api_unsupported_error_detects_gemini_plain_404():\n class _Resp:\n text = \"\"\n\n class _Err(Exception):\n status_code = 404\n body = \"\"\n response = _Resp()\n\n def __str__(self):\n return \"Error code: 404\"\n\n assert is_responses_api_unsupported_error(_Err()) is True\n" }, { "path": "tests/unit/test_ai_response_parser.py", "content": "import pytest\n\nfrom src.services.ai_response_parser import parse_ai_response_json\n\n\ndef test_parse_ai_response_json_uses_first_object_when_multiple_json_objects_are_concatenated():\n content = \"\"\"```json\n{\"is_recommended\": true, \"reason\": \"first\"}\n{\"is_recommended\": false, \"reason\": \"second\"}\n```\"\"\"\n\n result = parse_ai_response_json(content)\n\n assert result == {\"is_recommended\": True, \"reason\": \"first\"}\n\n\ndef test_parse_ai_response_json_extracts_json_from_wrapped_text():\n content = \"\"\"分析结果如下：\n\n```json\n{\"is_recommended\": true, \"reason\": \"wrapped\"}\n```\n\n请按第一份结果处理。\"\"\"\n\n result = parse_ai_response_json(content)\n\n assert result == {\"is_recommended\": True, \"reason\": \"wrapped\"}\n\n\ndef test_parse_ai_response_json_raises_when_no_json_exists():\n with pytest.raises(ValueError):\n parse_ai_response_json(\"没有任何 JSON 内容\")\n" }, { "path": "tests/unit/test_app_lifespan.py", "content": "import asyncio\n\nimport src.app as app_module\n\n\nclass _FakeTaskService:\n def __init__(self, _repo):\n self.updated = []\n\n async def get_all_tasks(self):\n return []\n\n async def update_task_status(self, task_id, is_running):\n self.updated.append((task_id, is_running))\n\n\nclass _FakeSchedulerService:\n def __init__(self):\n self.started = False\n self.stopped = False\n self.reload_payload = None\n\n async def reload_jobs(self, tasks):\n self.reload_payload = list(tasks)\n\n def start(self):\n self.started = True\n\n def stop(self):\n self.stopped = True\n\n\nclass _FakeProcessService:\n def __init__(self):\n self.stop_all_called = False\n\n async def stop_all(self):\n self.stop_all_called = True\n\n\ndef test_lifespan_cleans_task_logs_on_startup(monkeypatch):\n called = {}\n fake_scheduler = _FakeSchedulerService()\n fake_process = _FakeProcessService()\n\n monkeypatch.setattr(app_module, \"scheduler_service\", fake_scheduler)\n monkeypatch.setattr(app_module, \"process_service\", fake_process)\n monkeypatch.setattr(app_module, \"TaskService\", _FakeTaskService)\n monkeypatch.setattr(app_module, \"SqliteTaskRepository\", lambda: object())\n monkeypatch.setattr(app_module, \"bootstrap_sqlite_storage\", lambda: called.setdefault(\"bootstrapped\", True))\n monkeypatch.setattr(\n app_module,\n \"cleanup_task_logs\",\n lambda *args, **kwargs: called.setdefault(\"keep_days\", kwargs.get(\"keep_days\")),\n )\n monkeypatch.setattr(app_module.app_settings, \"task_log_retention_days\", 9)\n\n async def _run():\n async with app_module.lifespan(None):\n assert fake_scheduler.started is True\n assert fake_scheduler.reload_payload == []\n\n asyncio.run(_run())\n\n assert called[\"bootstrapped\"] is True\n assert called[\"keep_days\"] == 9\n assert fake_scheduler.stopped is True\n assert fake_process.stop_all_called is True\n" }, { "path": "tests/unit/test_cron_utils.py", "content": "from src.core.cron_utils import build_cron_trigger, validate_cron_expression\n\n\ndef test_validate_cron_expression_normalizes_alias():\n assert validate_cron_expression(\"@daily\") == \"0 0 * * *\"\n\n\ndef test_validate_cron_expression_accepts_six_fields():\n assert validate_cron_expression(\"0 0 8 * * *\") == \"0 0 8 * * *\"\n\n\ndef test_build_cron_trigger_accepts_alias_and_timezone():\n trigger = build_cron_trigger(\"@hourly\", timezone=\"Asia/Shanghai\")\n\n assert trigger is not None\n assert str(trigger.timezone) == \"Asia/Shanghai\"\n\n\ndef test_validate_cron_expression_rejects_invalid_value():\n try:\n validate_cron_expression(\"not-a-cron\")\n except ValueError as exc:\n assert \"支持 5 段\" in str(exc)\n return\n\n raise AssertionError(\"非法 cron 应该抛出 ValueError\")\n" }, { "path": "tests/unit/test_domain_task.py", "content": "from src.domain.models.task import Task, TaskGenerateRequest, TaskUpdate\n\n\ndef test_task_can_start_and_stop():\n task = Task(\n id=1,\n task_name=\"Sony A7M4\",\n enabled=True,\n keyword=\"sony a7m4\",\n description=\"body\",\n max_pages=2,\n personal_only=True,\n min_price=None,\n max_price=None,\n cron=None,\n ai_prompt_base_file=\"prompts/base_prompt.txt\",\n ai_prompt_criteria_file=\"prompts/sony_a7m4_criteria.txt\",\n is_running=False,\n )\n\n assert task.can_start() is True\n assert task.can_stop() is False\n\n running = task.model_copy(update={\"is_running\": True})\n assert running.can_start() is False\n assert running.can_stop() is True\n\n\ndef test_task_apply_update():\n task = Task(\n id=1,\n task_name=\"Sony A7M4\",\n enabled=True,\n keyword=\"sony a7m4\",\n description=\"body\",\n max_pages=2,\n personal_only=True,\n min_price=None,\n max_price=None,\n cron=None,\n ai_prompt_base_file=\"prompts/base_prompt.txt\",\n ai_prompt_criteria_file=\"prompts/sony_a7m4_criteria.txt\",\n is_running=False,\n )\n\n update = TaskUpdate(enabled=False, max_pages=5)\n updated = task.apply_update(update)\n\n assert updated.enabled is False\n assert updated.max_pages == 5\n assert updated.task_name == task.task_name\n\n\ndef test_legacy_keyword_groups_are_flattened_to_keyword_rules():\n task = Task(\n id=1,\n task_name=\"Sony A7M4\",\n enabled=True,\n keyword=\"sony a7m4\",\n description=\"body\",\n max_pages=2,\n personal_only=True,\n min_price=None,\n max_price=None,\n cron=None,\n ai_prompt_base_file=\"prompts/base_prompt.txt\",\n ai_prompt_criteria_file=\"prompts/sony_a7m4_criteria.txt\",\n decision_mode=\"keyword\",\n keyword_rule_groups=[\n {\"name\": \"组1\", \"include_keywords\": [\"a7m4\", \"验货宝\"], \"exclude_keywords\": [\"瑕疵\"]},\n {\"name\": \"组2\", \"include_keywords\": [\"全画幅\", \"a7m4\"], \"exclude_keywords\": [\"拆修\"]},\n ],\n is_running=False,\n )\n\n assert task.keyword_rules == [\"a7m4\", \"验货宝\", \"全画幅\"]\n\n\ndef test_generate_request_accepts_legacy_group_payload():\n req = TaskGenerateRequest(\n task_name=\"legacy\",\n keyword=\"sony a7m4\",\n description=\"\",\n decision_mode=\"keyword\",\n keyword_rule_groups=[{\"include_keywords\": [\"a7m4\", \"验货宝\"], \"exclude_keywords\": [\"瑕疵\"]}],\n )\n assert req.keyword_rules == [\"a7m4\", \"验货宝\"]\n\n\ndef test_generate_request_enables_image_analysis_by_default():\n req = TaskGenerateRequest(\n task_name=\"Sony A7M4\",\n keyword=\"sony a7m4\",\n description=\"只看机身成色和卖家信用。\",\n decision_mode=\"ai\",\n )\n assert req.analyze_images is True\n\n\ndef test_generate_request_infers_fixed_account_strategy_from_state_file():\n req = TaskGenerateRequest(\n task_name=\"Sony A7M4\",\n keyword=\"sony a7m4\",\n description=\"只看机身成色和卖家信用。\",\n decision_mode=\"ai\",\n account_state_file=\"state/acc_1.json\",\n )\n\n assert req.account_strategy == \"fixed\"\n\n\ndef test_generate_request_requires_state_file_for_fixed_account_strategy():\n try:\n TaskGenerateRequest(\n task_name=\"Sony A7M4\",\n keyword=\"sony a7m4\",\n description=\"只看机身成色和卖家信用。\",\n decision_mode=\"ai\",\n account_strategy=\"fixed\",\n )\n except ValueError as exc:\n assert \"固定账号模式下必须选择账号\" in str(exc)\n return\n\n raise AssertionError(\"固定账号模式应要求 account_state_file\")\n" }, { "path": "tests/unit/test_item_analysis_dispatcher.py", "content": "import asyncio\n\nfrom src.services.item_analysis_dispatcher import (\n ItemAnalysisDispatcher,\n ItemAnalysisJob,\n)\n\n\ndef test_item_analysis_dispatcher_uses_bounded_concurrency():\n active_ai_calls = 0\n max_active_ai_calls = 0\n saved_records = []\n notifications = []\n\n async def seller_loader(user_id: str):\n await asyncio.sleep(0.005)\n return {\"卖家ID\": user_id}\n\n async def image_downloader(product_id: str, image_urls: list[str], task_name: str):\n return []\n\n async def ai_analyzer(record: dict, image_paths: list[str], prompt_text: str):\n nonlocal active_ai_calls, max_active_ai_calls\n active_ai_calls += 1\n max_active_ai_calls = max(max_active_ai_calls, active_ai_calls)\n await asyncio.sleep(0.03)\n active_ai_calls -= 1\n return {\n \"analysis_source\": \"ai\",\n \"is_recommended\": True,\n \"reason\": f\"推荐 {record['商品信息']['商品ID']}\",\n \"keyword_hit_count\": 0,\n }\n\n async def notifier(item_data: dict, reason: str):\n notifications.append((item_data[\"商品ID\"], reason))\n\n async def saver(record: dict, keyword: str):\n saved_records.append((keyword, record))\n return True\n\n async def run():\n dispatcher = ItemAnalysisDispatcher(\n concurrency=2,\n skip_ai_analysis=False,\n seller_loader=seller_loader,\n image_downloader=image_downloader,\n ai_analyzer=ai_analyzer,\n notifier=notifier,\n saver=saver,\n )\n for index in range(3):\n dispatcher.submit(\n ItemAnalysisJob(\n keyword=\"demo\",\n task_name=\"Demo\",\n decision_mode=\"ai\",\n analyze_images=False,\n prompt_text=\"prompt\",\n keyword_rules=(),\n final_record={\n \"商品信息\": {\"商品ID\": str(index), \"商品图片列表\": []},\n \"卖家信息\": {},\n },\n seller_id=f\"seller-{index}\",\n zhima_credit_text=\"优秀\",\n registration_duration_text=\"来闲鱼1年\",\n )\n )\n await dispatcher.join()\n return dispatcher\n\n dispatcher = asyncio.run(run())\n assert dispatcher.completed_count == 3\n assert len(saved_records) == 3\n assert len(notifications) == 3\n assert max_active_ai_calls == 2\n assert saved_records[0][1][\"卖家信息\"][\"卖家ID\"].startswith(\"seller-\")\n\n\ndef test_item_analysis_dispatcher_supports_keyword_mode_without_ai():\n saved_records = []\n\n async def seller_loader(user_id: str):\n return {\"卖家标签\": \"个人闲置\"}\n\n async def image_downloader(product_id: str, image_urls: list[str], task_name: str):\n raise AssertionError(\"关键词模式不应下载图片\")\n\n async def ai_analyzer(record: dict, image_paths: list[str], prompt_text: str):\n raise AssertionError(\"关键词模式不应调用 AI\")\n\n async def notifier(item_data: dict, reason: str):\n return None\n\n async def saver(record: dict, keyword: str):\n saved_records.append(record)\n return True\n\n async def run():\n dispatcher = ItemAnalysisDispatcher(\n concurrency=1,\n skip_ai_analysis=False,\n seller_loader=seller_loader,\n image_downloader=image_downloader,\n ai_analyzer=ai_analyzer,\n notifier=notifier,\n saver=saver,\n )\n dispatcher.submit(\n ItemAnalysisJob(\n keyword=\"demo\",\n task_name=\"Demo\",\n decision_mode=\"keyword\",\n analyze_images=False,\n prompt_text=\"\",\n keyword_rules=(\"个人闲置\",),\n final_record={\n \"商品信息\": {\"商品ID\": \"1\", \"商品标题\": \"演示商品\"},\n \"卖家信息\": {},\n },\n seller_id=\"seller-1\",\n zhima_credit_text=\"优秀\",\n registration_duration_text=\"来闲鱼1年\",\n )\n )\n await dispatcher.join()\n\n asyncio.run(run())\n assert saved_records[0][\"ai_analysis\"][\"analysis_source\"] == \"keyword\"\n assert saved_records[0][\"ai_analysis\"][\"is_recommended\"] is True\n" }, { "path": "tests/unit/test_keyword_rule_engine.py", "content": "from src.keyword_rule_engine import build_search_text, evaluate_keyword_rules\n\n\ndef _sample_record():\n return {\n \"商品信息\": {\n \"商品标题\": \"Sony A7M4 全画幅相机\",\n \"当前售价\": \"10000\",\n \"商品标签\": [\"验货宝\", \"包邮\"],\n },\n \"卖家信息\": {\n \"卖家昵称\": \"摄影器材店\",\n \"卖家个性签名\": \"可验机，支持同城面交\",\n },\n }\n\n\ndef test_build_search_text_contains_product_and_seller_fields():\n text = build_search_text(_sample_record())\n assert \"sony a7m4\" in text\n assert \"摄影器材店\" in text\n assert \"支持同城面交\" in text\n\n\ndef test_keyword_rules_or_match_any_keyword():\n text = build_search_text(_sample_record())\n result = evaluate_keyword_rules([\"a7m4\", \"佳能\"], text)\n assert result[\"is_recommended\"] is True\n assert result[\"analysis_source\"] == \"keyword\"\n assert result[\"keyword_hit_count\"] == 1\n assert result[\"matched_keywords\"] == [\"a7m4\"]\n\n\ndef test_keyword_rules_count_multiple_hits():\n text = build_search_text(_sample_record())\n result = evaluate_keyword_rules([\"a7m4\", \"验货宝\", \"摄影器材店\"], text)\n assert result[\"is_recommended\"] is True\n assert result[\"keyword_hit_count\"] == 3\n\n\ndef test_keyword_rules_case_insensitive_contains():\n text = build_search_text(_sample_record())\n result = evaluate_keyword_rules([\"SONY\", \"A7M4\"], text)\n assert result[\"is_recommended\"] is True\n assert result[\"keyword_hit_count\"] == 2\n\n\ndef test_keyword_rules_no_match():\n text = build_search_text(_sample_record())\n result = evaluate_keyword_rules([\"佳能\", \"单反\"], text)\n assert result[\"is_recommended\"] is False\n assert result[\"keyword_hit_count\"] == 0\n\n\ndef test_keyword_rules_do_not_partially_match_alphanumeric_prefixes():\n result = evaluate_keyword_rules([\"q1\"], \"富士 q1r5 旗舰相机\")\n assert result[\"is_recommended\"] is False\n assert result[\"keyword_hit_count\"] == 0\n\n\ndef test_keyword_rules_still_match_full_alphanumeric_token():\n result = evaluate_keyword_rules([\"q1r5\"], \"富士 q1r5 旗舰相机\")\n assert result[\"is_recommended\"] is True\n assert result[\"keyword_hit_count\"] == 1\n" }, { "path": "tests/unit/test_notification_service.py", "content": "import asyncio\n\nfrom src.infrastructure.external.notification_clients.base import NotificationClient\nfrom src.infrastructure.external.notification_clients.webhook_client import WebhookClient\nfrom src.services.notification_service import NotificationService\n\n\nclass _OkClient(NotificationClient):\n channel_key = \"ok\"\n display_name = \"OK\"\n\n async def send(self, product_data, reason):\n return None\n\n\nclass _FailClient(NotificationClient):\n channel_key = \"fail\"\n display_name = \"FAIL\"\n\n async def send(self, product_data, reason):\n raise RuntimeError(\"boom\")\n\n\ndef test_notification_service_collects_success_and_failure_results():\n service = NotificationService([_OkClient(enabled=True), _FailClient(enabled=True)])\n\n results = asyncio.run(\n service.send_notification({\"商品标题\": \"Sony A7M4\"}, \"价格合适\")\n )\n\n assert results[\"ok\"][\"success\"] is True\n assert results[\"ok\"][\"message\"] == \"发送成功\"\n assert results[\"fail\"][\"success\"] is False\n assert results[\"fail\"][\"message\"] == \"boom\"\n\n\ndef test_webhook_client_renders_json_templates(monkeypatch):\n captured = {}\n\n class _FakeResponse:\n def raise_for_status(self):\n return None\n\n def _fake_post(url, headers=None, json=None, data=None, timeout=None):\n captured[\"url\"] = url\n captured[\"headers\"] = headers\n captured[\"json\"] = json\n captured[\"data\"] = data\n return _FakeResponse()\n\n monkeypatch.setattr(\"requests.post\", _fake_post)\n\n client = WebhookClient(\n webhook_url=\"https://hooks.example.com/notify\",\n webhook_method=\"POST\",\n webhook_headers='{\"Authorization\":\"Bearer token\"}',\n webhook_content_type=\"JSON\",\n webhook_query_parameters='{\"task\":\"{{title}}\"}',\n webhook_body='{\"message\":\"{{content}}\",\"link\":\"{{desktop_link}}\"}',\n pcurl_to_mobile=False,\n )\n\n asyncio.run(\n client.send(\n {\n \"商品标题\": \"Sony A7M4\",\n \"当前售价\": \"9999\",\n \"商品链接\": \"https://www.goofish.com/item/123\",\n },\n \"价格合适\",\n )\n )\n\n assert \"task=%F0%9F%9A%A8+%E6%96%B0%E6%8E%A8%E8%8D%90%21+Sony+A7M4\" in captured[\"url\"]\n assert captured[\"headers\"][\"Authorization\"] == \"Bearer token\"\n assert captured[\"json\"][\"message\"].startswith(\"价格: 9999\")\n assert captured[\"json\"][\"link\"] == \"https://www.goofish.com/item/123\"\n assert captured[\"data\"] is None\n" }, { "path": "tests/unit/test_price_history_service.py", "content": "from src.services.price_history_service import (\n build_item_price_context,\n build_price_history_insights,\n load_price_snapshots,\n record_market_snapshots,\n)\n\n\ndef test_record_market_snapshots_and_build_price_history_insights(tmp_path, monkeypatch):\n monkeypatch.chdir(tmp_path)\n seen_item_ids = set()\n\n run1_items = [\n {\n \"商品ID\": \"1001\",\n \"商品标题\": \"Sony A7M4 单机\",\n \"当前售价\": \"¥10000\",\n \"商品标签\": [\"验货宝\"],\n \"发货地区\": \"上海\",\n \"卖家昵称\": \"卖家A\",\n \"商品链接\": \"https://www.goofish.com/item?id=1001\",\n \"发布时间\": \"2026-01-01 09:00\",\n },\n {\n \"商品ID\": \"1002\",\n \"商品标题\": \"Sony A7M4 套机\",\n \"当前售价\": \"¥12000\",\n \"商品标签\": [\"包邮\"],\n \"发货地区\": \"杭州\",\n \"卖家昵称\": \"卖家B\",\n \"商品链接\": \"https://www.goofish.com/item?id=1002\",\n \"发布时间\": \"2026-01-01 10:00\",\n },\n ]\n run2_items = [\n {\n \"商品ID\": \"1001\",\n \"商品标题\": \"Sony A7M4 单机\",\n \"当前售价\": \"¥9500\",\n \"商品标签\": [\"验货宝\"],\n \"发货地区\": \"上海\",\n \"卖家昵称\": \"卖家A\",\n \"商品链接\": \"https://www.goofish.com/item?id=1001\",\n \"发布时间\": \"2026-01-02 09:00\",\n },\n {\n \"商品ID\": \"1003\",\n \"商品标题\": \"Sony A7M4 全套\",\n \"当前售价\": \"¥13000\",\n \"商品标签\": [\"同城\"],\n \"发货地区\": \"南京\",\n \"卖家昵称\": \"卖家C\",\n \"商品链接\": \"https://www.goofish.com/item?id=1003\",\n \"发布时间\": \"2026-01-02 11:00\",\n },\n ]\n\n inserted_run1 = record_market_snapshots(\n keyword=\"sony a7m4\",\n task_name=\"Sony A7M4 监控\",\n items=run1_items,\n run_id=\"run-1\",\n snapshot_time=\"2026-01-01T12:00:00\",\n seen_item_ids=seen_item_ids,\n )\n assert len(inserted_run1) == 2\n\n inserted_run2 = record_market_snapshots(\n keyword=\"sony a7m4\",\n task_name=\"Sony A7M4 监控\",\n items=run2_items,\n run_id=\"run-2\",\n snapshot_time=\"2026-01-02T12:00:00\",\n seen_item_ids=set(),\n )\n assert len(inserted_run2) == 2\n\n snapshots = load_price_snapshots(\"sony a7m4\")\n assert len(snapshots) == 4\n\n insights = build_price_history_insights(\"sony a7m4\")\n assert insights[\"market_summary\"][\"sample_count\"] == 2\n assert insights[\"market_summary\"][\"avg_price\"] == 11250.0\n assert insights[\"market_summary\"][\"min_price\"] == 9500.0\n assert insights[\"history_summary\"][\"unique_items\"] == 3\n assert len(insights[\"daily_trend\"]) == 2\n assert insights[\"daily_trend\"][0][\"day\"] == \"2026-01-01\"\n assert insights[\"daily_trend\"][1][\"day\"] == \"2026-01-02\"\n\n item_context = build_item_price_context(\n snapshots,\n item_id=\"1001\",\n current_price=9500.0,\n )\n assert item_context[\"observation_count\"] == 2\n assert item_context[\"min_price\"] == 9500.0\n assert item_context[\"max_price\"] == 10000.0\n assert item_context[\"price_change_amount\"] == -500.0\n assert item_context[\"deal_label\"] == \"高性价比\"\n" }, { "path": "tests/unit/test_process_service.py", "content": "import asyncio\nimport sys\nfrom types import SimpleNamespace\n\nfrom src.services.process_service import ProcessService\n\n\nclass FakeProcess:\n def __init__(self, pid: int):\n self.pid = pid\n self.returncode = None\n self._done = asyncio.Event()\n\n async def wait(self):\n await self._done.wait()\n return self.returncode\n\n def finish(self, returncode: int = 0):\n self.returncode = returncode\n self._done.set()\n\n def terminate(self):\n self.finish(-15)\n\n def kill(self):\n self.finish(-9)\n\n\ndef test_process_service_marks_task_stopped_when_process_exits(monkeypatch, tmp_path):\n fake_process = FakeProcess(pid=4321)\n events = []\n\n async def run_scenario():\n service = ProcessService()\n service.failure_guard.should_skip_start = lambda *args, **kwargs: SimpleNamespace(\n skip=False,\n should_notify=False,\n reason=\"\",\n consecutive_failures=0,\n paused_until=None,\n )\n\n stopped = asyncio.Event()\n\n async def on_started(task_id: int):\n events.append((\"started\", task_id))\n\n async def on_stopped(task_id: int):\n events.append((\"stopped\", task_id))\n stopped.set()\n\n service.set_lifecycle_hooks(on_started=on_started, on_stopped=on_stopped)\n\n async def fake_create_subprocess_exec(*_args, **_kwargs):\n return fake_process\n\n monkeypatch.setattr(\n \"src.services.process_service.build_task_log_path\",\n lambda task_id, _task_name: str(tmp_path / f\"task-{task_id}.log\"),\n )\n monkeypatch.setattr(asyncio, \"create_subprocess_exec\", fake_create_subprocess_exec)\n\n started = await service.start_task(0, \"task-a\")\n assert started is True\n assert events == [(\"started\", 0)]\n assert service.is_running(0) is True\n\n fake_process.finish(0)\n await asyncio.wait_for(stopped.wait(), timeout=1)\n\n assert (\"stopped\", 0) in events\n assert service.is_running(0) is False\n\n asyncio.run(run_scenario())\n\n\ndef test_process_service_reindexes_runtime_maps_after_delete():\n service = ProcessService()\n proc_a = object()\n proc_c = object()\n watcher_a = object()\n watcher_c = object()\n\n service.processes = {0: proc_a, 2: proc_c}\n service.log_paths = {0: \"a.log\", 2: \"c.log\"}\n service.task_names = {0: \"A\", 2: \"C\"}\n service.exit_watchers = {0: watcher_a, 2: watcher_c}\n\n service.reindex_after_delete(1)\n\n assert service.processes == {0: proc_a, 1: proc_c}\n assert service.log_paths == {0: \"a.log\", 1: \"c.log\"}\n assert service.task_names == {0: \"A\", 1: \"C\"}\n assert service.exit_watchers == {0: watcher_a, 1: watcher_c}\n\n\ndef test_process_service_adds_debug_limit_arg_when_env_enabled(monkeypatch):\n monkeypatch.setenv(\"SPIDER_DEBUG_LIMIT\", \"1\")\n service = ProcessService()\n\n command = service._build_spawn_command(\"task-a\")\n\n assert command == [\n sys.executable,\n \"-u\",\n \"spider_v2.py\",\n \"--task-name\",\n \"task-a\",\n \"--debug-limit\",\n \"1\",\n ]\n" }, { "path": "tests/unit/test_prompt_utils.py", "content": "import asyncio\n\nimport pytest\n\nimport src.prompt_utils as prompt_utils\nfrom src.services.ai_response_parser import EmptyAIResponseError\n\n\ndef test_generate_criteria_closes_ai_client_after_success(monkeypatch, tmp_path):\n close_state = {\"closed\": False}\n reference_file = tmp_path / \"reference.txt\"\n reference_file.write_text(\"reference\", encoding=\"utf-8\")\n\n class FakeAIClient:\n def is_available(self):\n return True\n\n def refresh(self):\n raise AssertionError(\"refresh should not be called\")\n\n async def _call_ai(self, *_args, **_kwargs):\n return \"generated criteria\"\n\n async def close(self):\n close_state[\"closed\"] = True\n\n monkeypatch.setattr(prompt_utils, \"AIClient\", FakeAIClient)\n\n result = asyncio.run(\n prompt_utils.generate_criteria(\"need a gpu\", str(reference_file))\n )\n\n assert result == \"generated criteria\"\n assert close_state[\"closed\"] is True\n\n\ndef test_generate_criteria_closes_ai_client_after_ai_failure(monkeypatch, tmp_path):\n close_state = {\"closed\": False}\n reference_file = tmp_path / \"reference.txt\"\n reference_file.write_text(\"reference\", encoding=\"utf-8\")\n\n class FakeAIClient:\n def is_available(self):\n return True\n\n def refresh(self):\n raise AssertionError(\"refresh should not be called\")\n\n async def _call_ai(self, *_args, **_kwargs):\n raise EmptyAIResponseError(\"AI响应内容为空。\")\n\n async def close(self):\n close_state[\"closed\"] = True\n\n monkeypatch.setattr(prompt_utils, \"AIClient\", FakeAIClient)\n\n with pytest.raises(EmptyAIResponseError, match=\"AI响应内容为空\"):\n asyncio.run(prompt_utils.generate_criteria(\"need a gpu\", str(reference_file)))\n\n assert close_state[\"closed\"] is True\n" }, { "path": "tests/unit/test_scraper_browser_channel.py", "content": "import importlib\n\n\ndef _load_scraper(monkeypatch, *, login_is_edge: bool, running_in_docker: bool):\n monkeypatch.setenv(\"LOGIN_IS_EDGE\", \"true\" if login_is_edge else \"false\")\n monkeypatch.setenv(\"RUNNING_IN_DOCKER\", \"true\" if running_in_docker else \"false\")\n\n import src.config as config_module\n import src.scraper as scraper_module\n\n importlib.reload(config_module)\n reloaded_scraper = importlib.reload(scraper_module)\n reloaded_scraper.EDGE_DOCKER_WARNING_PRINTED = False\n return reloaded_scraper\n\n\ndef test_resolve_browser_channel_uses_chromium_in_docker_even_when_edge_requested(monkeypatch, capsys):\n scraper = _load_scraper(monkeypatch, login_is_edge=True, running_in_docker=True)\n\n assert scraper._resolve_browser_channel() == \"chromium\"\n assert \"Docker 镜像未内置 Edge\" in capsys.readouterr().out\n\n\ndef test_resolve_browser_channel_uses_msedge_locally_when_requested(monkeypatch):\n scraper = _load_scraper(monkeypatch, login_is_edge=True, running_in_docker=False)\n\n assert scraper._resolve_browser_channel() == \"msedge\"\n" }, { "path": "tests/unit/test_search_pagination.py", "content": "import asyncio\n\nfrom playwright.async_api import TimeoutError as PlaywrightTimeoutError\n\nfrom src.services.search_pagination import advance_search_page\nfrom src.services.search_pagination import is_search_results_response\n\n\nclass FakeRequest:\n def __init__(self, method: str = \"POST\"):\n self.method = method\n\n\nclass FakeResponse:\n def __init__(self, url: str, ok: bool = True, method: str = \"POST\"):\n self.url = url\n self.ok = ok\n self.request = FakeRequest(method)\n\n\nclass FakeLocator:\n def __init__(self, count: int, click_error: Exception | None = None):\n self._count = count\n self.clicks = 0\n self.scrolls = 0\n self.click_timeout = None\n self._click_error = click_error\n\n @property\n def first(self):\n return self\n\n async def count(self) -> int:\n return self._count\n\n async def scroll_into_view_if_needed(self) -> None:\n self.scrolls += 1\n\n async def click(self, timeout: int | None = None) -> None:\n self.clicks += 1\n self.click_timeout = timeout\n if self._click_error is not None:\n raise self._click_error\n\n\nclass FakeResponseContext:\n def __init__(self, outcome):\n self._outcome = outcome\n\n async def __aenter__(self):\n return self\n\n async def __aexit__(self, exc_type, exc, tb):\n return False\n\n @property\n def value(self):\n return self._resolve()\n\n async def _resolve(self):\n if isinstance(self._outcome, Exception):\n raise self._outcome\n return self._outcome\n\n\nclass FakePage:\n def __init__(\n self,\n next_button_count: int,\n outcomes: list[object],\n click_error: Exception | None = None,\n ):\n self.locator_stub = FakeLocator(next_button_count, click_error=click_error)\n self._outcomes = list(outcomes)\n\n def locator(self, _selector: str) -> FakeLocator:\n return self.locator_stub\n\n def expect_response(self, _predicate, timeout: int):\n assert timeout == 20000\n if not self._outcomes:\n raise AssertionError(\"missing fake response outcome\")\n return FakeResponseContext(self._outcomes.pop(0))\n\n\nasync def _noop_random_sleep(_min_seconds: float, _max_seconds: float) -> None:\n return None\n\n\nasync def _noop_sleep(_seconds: float) -> None:\n return None\n\n\ndef test_advance_search_page_stops_when_no_next_button() -> None:\n page = FakePage(next_button_count=0, outcomes=[])\n logs: list[str] = []\n\n result = asyncio.run(\n advance_search_page(\n page=page,\n page_num=2,\n logger=logs.append,\n wait_after_click=_noop_random_sleep,\n retry_sleep=_noop_sleep,\n )\n )\n\n assert result.advanced is False\n assert result.response is None\n assert result.stop_reason == \"no_next_button\"\n assert page.locator_stub.clicks == 0\n assert logs == [\"已到达最后一页，未找到可用的'下一页'按钮，停止翻页。\"]\n\n\ndef test_advance_search_page_stops_after_timeout_retries() -> None:\n page = FakePage(\n next_button_count=1,\n outcomes=[\n PlaywrightTimeoutError(\"page 2 timeout\"),\n PlaywrightTimeoutError(\"page 2 timeout\"),\n ],\n )\n logs: list[str] = []\n\n result = asyncio.run(\n advance_search_page(\n page=page,\n page_num=2,\n logger=logs.append,\n wait_after_click=_noop_random_sleep,\n retry_sleep=_noop_sleep,\n )\n )\n\n assert result.advanced is False\n assert result.response is None\n assert result.stop_reason == \"response_timeout\"\n assert page.locator_stub.clicks == 2\n assert page.locator_stub.scrolls == 2\n assert logs == [\n \"等待第 2 页搜索响应超时，5秒后重试...\",\n \"等待第 2 页搜索响应超时 2 次，停止翻页。\",\n ]\n\n\ndef test_advance_search_page_returns_new_response_on_success() -> None:\n response = FakeResponse(\n url=\"https://example.com/h5/mtop.taobao.idlemtopsearch.pc.search/1.0/?page=2\"\n )\n page = FakePage(next_button_count=1, outcomes=[response])\n\n result = asyncio.run(\n advance_search_page(\n page=page,\n page_num=2,\n logger=lambda _message: None,\n wait_after_click=_noop_random_sleep,\n retry_sleep=_noop_sleep,\n )\n )\n\n assert result.advanced is True\n assert result.response is response\n assert result.stop_reason is None\n assert page.locator_stub.clicks == 1\n assert page.locator_stub.scrolls == 1\n assert page.locator_stub.click_timeout == 10000\n\n\ndef test_advance_search_page_stops_when_click_times_out() -> None:\n page = FakePage(\n next_button_count=1,\n outcomes=[FakeResponse(url=\"https://example.com/unused\")],\n click_error=PlaywrightTimeoutError(\"click timeout\"),\n )\n logs: list[str] = []\n\n result = asyncio.run(\n advance_search_page(\n page=page,\n page_num=2,\n logger=logs.append,\n wait_after_click=_noop_random_sleep,\n retry_sleep=_noop_sleep,\n )\n )\n\n assert result.advanced is False\n assert result.response is None\n assert result.stop_reason == \"click_timeout\"\n assert page.locator_stub.clicks == 1\n assert logs == [\"第 2 页下一页按钮点击超时，停止翻页。\"]\n\n\ndef test_is_search_results_response_matches_exact_search_api() -> None:\n response = FakeResponse(\n url=\"https://h5api.m.goofish.com/h5/mtop.taobao.idlemtopsearch.pc.search/1.0/?foo=bar\",\n method=\"POST\",\n )\n\n assert is_search_results_response(response) is True\n\n\ndef test_is_search_results_response_rejects_search_shade_api() -> None:\n response = FakeResponse(\n url=\"https://h5api.m.goofish.com/h5/mtop.taobao.idlemtopsearch.pc.search.shade/1.0/?foo=bar\",\n method=\"POST\",\n )\n\n assert is_search_results_response(response) is False\n\n\ndef test_is_search_results_response_rejects_non_post_request() -> None:\n response = FakeResponse(\n url=\"https://h5api.m.goofish.com/h5/mtop.taobao.idlemtopsearch.pc.search/1.0/?foo=bar\",\n method=\"GET\",\n )\n\n assert is_search_results_response(response) is False\n" }, { "path": "tests/unit/test_seller_profile_cache.py", "content": "import asyncio\n\nfrom src.services.seller_profile_cache import SellerProfileCache\n\n\ndef test_seller_profile_cache_reuses_value_and_returns_copy():\n clock = {\"value\": 100.0}\n cache = SellerProfileCache(ttl_seconds=60, time_source=lambda: clock[\"value\"])\n loader_calls = 0\n\n async def loader(user_id: str):\n nonlocal loader_calls\n loader_calls += 1\n return {\"user_id\": user_id, \"items\": []}\n\n async def run():\n first = await cache.get_or_load(\"seller-1\", loader)\n first[\"items\"].append(\"mutated\")\n second = await cache.get_or_load(\"seller-1\", loader)\n return second\n\n second = asyncio.run(run())\n assert loader_calls == 1\n assert second == {\"user_id\": \"seller-1\", \"items\": []}\n\n\ndef test_seller_profile_cache_coalesces_inflight_requests():\n cache = SellerProfileCache(ttl_seconds=60)\n loader_calls = 0\n\n async def loader(user_id: str):\n nonlocal loader_calls\n loader_calls += 1\n await asyncio.sleep(0.02)\n return {\"user_id\": user_id}\n\n async def run():\n return await asyncio.gather(\n cache.get_or_load(\"seller-2\", loader),\n cache.get_or_load(\"seller-2\", loader),\n )\n\n results = asyncio.run(run())\n assert loader_calls == 1\n assert results == [{\"user_id\": \"seller-2\"}, {\"user_id\": \"seller-2\"}]\n" }, { "path": "tests/unit/test_task_log_cleanup_service.py", "content": "from __future__ import annotations\n\nfrom datetime import datetime\nfrom pathlib import Path\n\nfrom src.services.task_log_cleanup_service import cleanup_task_logs\n\n\ndef _write_file(path: Path, content: str = \"log\") -> None:\n path.parent.mkdir(parents=True, exist_ok=True)\n path.write_text(content, encoding=\"utf-8\")\n\n\ndef _set_mtime(path: Path, when: datetime) -> None:\n timestamp = when.timestamp()\n path.touch()\n path.chmod(0o644)\n import os\n\n os.utime(path, (timestamp, timestamp))\n\n\ndef test_cleanup_task_logs_removes_only_expired_top_level_logs(tmp_path):\n logs_dir = tmp_path / \"logs\"\n old_log = logs_dir / \"old_task.log\"\n recent_log = logs_dir / \"recent_task.log\"\n nested_ai_log = logs_dir / \"ai\" / \"old_ai.log\"\n state_file = logs_dir / \"task-failure-guard.json\"\n\n _write_file(old_log)\n _write_file(recent_log)\n _write_file(nested_ai_log)\n _write_file(state_file, \"{}\")\n\n now = datetime(2026, 3, 19, 12, 0, 0)\n _set_mtime(old_log, datetime(2026, 3, 1, 0, 0, 0))\n _set_mtime(recent_log, datetime(2026, 3, 18, 23, 0, 0))\n _set_mtime(nested_ai_log, datetime(2026, 3, 1, 0, 0, 0))\n _set_mtime(state_file, datetime(2026, 3, 1, 0, 0, 0))\n\n removed = cleanup_task_logs(str(logs_dir), keep_days=7, now=now)\n\n assert removed == [str(old_log)]\n assert not old_log.exists()\n assert recent_log.exists()\n assert nested_ai_log.exists()\n assert state_file.exists()\n\n\ndef test_cleanup_task_logs_skips_when_retention_is_invalid(tmp_path):\n logs_dir = tmp_path / \"logs\"\n task_log = logs_dir / \"task.log\"\n _write_file(task_log)\n\n removed = cleanup_task_logs(str(logs_dir), keep_days=0, now=datetime(2026, 3, 19, 12, 0, 0))\n\n assert removed == []\n assert task_log.exists()\n" }, { "path": "tests/unit/test_utils.py", "content": "import asyncio\n\nfrom src.services.result_storage_service import load_all_result_records\nfrom src.utils import (\n format_registration_days,\n get_link_unique_key,\n safe_get,\n save_to_jsonl,\n)\n\n\ndef test_safe_get_nested_and_default():\n data = {\"a\": {\"b\": [{\"c\": \"value\"}]}}\n assert asyncio.run(safe_get(data, \"a\", \"b\", 0, \"c\")) == \"value\"\n assert asyncio.run(safe_get(data, \"a\", \"b\", 1, \"c\", default=\"missing\")) == \"missing\"\n\n\ndef test_format_registration_days():\n assert format_registration_days(400).startswith(\"\\u6765\\u95f2\\u9c7c\")\n assert format_registration_days(-1) == \"\\u672a\\u77e5\"\n\n\ndef test_get_link_unique_key():\n link = \"https://www.goofish.com/item?id=123&foo=bar\"\n assert get_link_unique_key(link) == \"https://www.goofish.com/item?id=123\"\n\n\ndef test_save_to_jsonl(tmp_path, monkeypatch):\n monkeypatch.chdir(tmp_path)\n record = {\n \"爬取时间\": \"2026-01-01T10:00:00\",\n \"搜索关键字\": \"sony a7m4\",\n \"任务名称\": \"Sony A7M4\",\n \"商品信息\": {\n \"商品ID\": \"1\",\n \"商品标题\": \"Sony A7M4\",\n \"商品链接\": \"https://www.goofish.com/item?id=1\",\n \"当前售价\": \"¥10000\",\n },\n }\n\n ok = asyncio.run(save_to_jsonl(record, keyword=\"sony a7m4\"))\n assert ok is True\n\n records = asyncio.run(\n load_all_result_records(\n \"sony_a7m4_full_data.jsonl\",\n ai_recommended_only=False,\n keyword_recommended_only=False,\n sort_by=\"crawl_time\",\n sort_order=\"asc\",\n )\n )\n assert records == [record]\n" }, { "path": "web-ui/.gitignore", "content": "# Logs\nlogs\n*.log\nnpm-debug.log*\nyarn-debug.log*\nyarn-error.log*\npnpm-debug.log*\nlerna-debug.log*\n\nnode_modules\ndist\ndist-ssr\n*.local\n\n# Editor directories and files\n.vscode/*\n!.vscode/extensions.json\n.idea\n.DS_Store\n*.suo\n*.ntvs*\n*.njsproj\n*.sln\n*.sw?\n" }, { "path": "web-ui/.vscode/extensions.json", "content": "{\n \"recommendations\": [\"Vue.volar\"]\n}\n" }, { "path": "web-ui/Dockerfile", "content": "# Stage 1: Build the Vue application\nFROM node:22-alpine AS builder\n\nWORKDIR /app\n\n# Copy package files and install dependencies\nCOPY package*.json ./\nRUN npm install\n\n# Copy source code and build\nCOPY . .\nRUN npm run build\n\n# Stage 2: Serve with Nginx\nFROM nginx:alpine\n\n# Remove default nginx static assets\nRUN rm -rf /usr/share/nginx/html/*\n\n# Vite 输出到容器根目录 /dist，而不是 /app/dist\nCOPY --from=builder /dist /usr/share/nginx/html\n\n# Copy custom nginx configuration\nCOPY nginx.conf /etc/nginx/conf.d/default.conf\n\nEXPOSE 80\n\nCMD [\"nginx\", \"-g\", \"daemon off;\"]\n" }, { "path": "web-ui/README.md", "content": "# Vue 3 + TypeScript + Vite\n\nThis template should help get you started developing with Vue 3 and TypeScript in Vite. The template uses Vue 3 `\n \n\n" }, { "path": "web-ui/nginx.conf", "content": "server {\n listen 80;\n server_name localhost;\n\n root /usr/share/nginx/html;\n index index.html;\n\n # Serve static frontend files\n location / {\n try_files $uri $uri/ /index.html;\n }\n\n # Proxy API requests to the backend service\n location /api/ {\n proxy_pass http://app:8000/api/;\n proxy_set_header Host $host;\n proxy_set_header X-Real-IP $remote_addr;\n proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;\n }\n\n # Proxy Auth requests\n location /auth/ {\n proxy_pass http://app:8000/auth/;\n proxy_set_header Host $host;\n proxy_set_header X-Real-IP $remote_addr;\n }\n\n # Proxy static files from backend if needed (though we serve frontend here)\n # But backend might serve user images or other static assets via /static\n location /static/ {\n proxy_pass http://app:8000/static/;\n }\n\n # Proxy WebSocket requests\n location /ws {\n proxy_pass http://app:8000/ws;\n proxy_http_version 1.1;\n proxy_set_header Upgrade $http_upgrade;\n proxy_set_header Connection \"Upgrade\";\n proxy_set_header Host $host;\n }\n}\n" }, { "path": "web-ui/package.json", "content": "{\n \"name\": \"web-ui\",\n \"private\": true,\n \"version\": \"0.0.0\",\n \"type\": \"module\",\n \"scripts\": {\n \"dev\": \"vite\",\n \"build\": \"vue-tsc -b && vite build\",\n \"preview\": \"vite preview\"\n },\n \"dependencies\": {\n \"@vueuse/core\": \"^14.1.0\",\n \"class-variance-authority\": \"^0.7.1\",\n \"clsx\": \"^2.1.1\",\n \"lucide-vue-next\": \"^0.562.0\",\n \"reka-ui\": \"^2.7.0\",\n \"tailwind-merge\": \"^3.4.0\",\n \"vue\": \"^3.5.24\",\n \"vue-i18n\": \"^10.0.7\",\n \"vue-router\": \"^4.6.4\"\n },\n \"devDependencies\": {\n \"@types/node\": \"^24.10.4\",\n \"@vitejs/plugin-vue\": \"^6.0.1\",\n \"@vue/tsconfig\": \"^0.8.1\",\n \"autoprefixer\": \"^10.4.23\",\n \"postcss\": \"^8.5.6\",\n \"tailwindcss\": \"^3.4.17\",\n \"tailwindcss-animate\": \"^1.0.7\",\n \"typescript\": \"~5.9.3\",\n \"vite\": \"^7.2.4\",\n \"vue-tsc\": \"^3.1.4\"\n }\n}\n" }, { "path": "web-ui/postcss.config.cjs", "content": "module.exports = {\n plugins: {\n tailwindcss: {},\n autoprefixer: {},\n },\n}\n" }, { "path": "web-ui/src/App.vue", "content": "\n\n" }, { "path": "web-ui/src/api/accounts.ts", "content": "import { http } from '@/lib/http'\n\nexport interface AccountItem {\n name: string\n path: string\n}\n\nexport interface AccountDetail extends AccountItem {\n content: string\n}\n\nexport async function listAccounts(): Promise {\n return await http('/api/accounts')\n}\n\nexport async function getAccount(name: string): Promise {\n return await http(`/api/accounts/${encodeURIComponent(name)}`)\n}\n\nexport async function createAccount(payload: { name: string; content: string }): Promise {\n return await http('/api/accounts', {\n method: 'POST',\n headers: { 'Content-Type': 'application/json' },\n body: JSON.stringify(payload),\n })\n}\n\nexport async function updateAccount(name: string, content: string): Promise {\n return await http(`/api/accounts/${encodeURIComponent(name)}`, {\n method: 'PUT',\n headers: { 'Content-Type': 'application/json' },\n body: JSON.stringify({ content }),\n })\n}\n\nexport async function deleteAccount(name: string): Promise<{ message: string }> {\n return await http(`/api/accounts/${encodeURIComponent(name)}`, { method: 'DELETE' })\n}\n" }, { "path": "web-ui/src/api/dashboard.ts", "content": "import { http } from '@/lib/http'\nimport type { DashboardSnapshot } from '@/types/dashboard.d.ts'\n\nexport async function getDashboardSummary(): Promise {\n return await http('/api/dashboard/summary')\n}\n" }, { "path": "web-ui/src/api/logs.ts", "content": "import { http } from '@/lib/http'\n\nexport async function getLogs(fromPos: number = 0, taskId?: number | null): Promise<{ new_content: string; new_pos: number }> {\n const params: Record = { from_pos: fromPos }\n if (taskId !== null && taskId !== undefined) {\n params.task_id = taskId\n }\n return await http('/api/logs', { params })\n}\n\nexport async function clearLogs(taskId?: number | null): Promise {\n const params: Record = {}\n if (taskId !== null && taskId !== undefined) {\n params.task_id = taskId\n }\n await http('/api/logs', { method: 'DELETE', params })\n}\n\nexport async function getLogTail(\n taskId: number,\n offsetLines: number = 0,\n limitLines: number = 50\n): Promise<{ content: string; has_more: boolean; next_offset: number; new_pos: number }> {\n return await http('/api/logs/tail', {\n params: {\n task_id: taskId,\n offset_lines: offsetLines,\n limit_lines: limitLines,\n },\n })\n}\n" }, { "path": "web-ui/src/api/prompts.ts", "content": "import { http } from '@/lib/http'\n\nexport interface PromptContent {\n filename: string\n content: string\n}\n\nexport async function listPrompts(): Promise {\n return await http('/api/prompts')\n}\n\nexport async function getPromptContent(filename: string): Promise {\n return await http(`/api/prompts/${filename}`)\n}\n\nexport async function updatePrompt(filename: string, content: string): Promise<{ message: string }> {\n return await http(`/api/prompts/${filename}`, {\n method: 'PUT',\n headers: { 'Content-Type': 'application/json' },\n body: JSON.stringify({ content }),\n })\n}\n" }, { "path": "web-ui/src/api/results.ts", "content": "import type { ResultInsights, ResultItem } from '@/types/result.d.ts'\nimport { http } from '@/lib/http'\n\nexport interface GetResultContentParams {\n recommended_only?: boolean;\n ai_recommended_only?: boolean;\n keyword_recommended_only?: boolean;\n sort_by?: 'crawl_time' | 'publish_time' | 'price' | 'keyword_hit_count';\n sort_order?: 'asc' | 'desc';\n page?: number;\n limit?: number;\n}\n\nexport async function getResultFiles(): Promise {\n const data = await http('/api/results/files')\n return data.files || []\n}\n\nexport async function deleteResultFile(filename: string): Promise<{ message: string }> {\n return await http(`/api/results/files/${filename}`, { method: 'DELETE' })\n}\n\nexport async function getResultContent(\n filename: string,\n params: GetResultContentParams = {}\n): Promise<{ total_items: number; items: ResultItem[] }> {\n return await http(`/api/results/${filename}`, { params: params as Record })\n}\n\nexport async function getResultInsights(filename: string): Promise {\n return await http(`/api/results/${filename}/insights`)\n}\n\nexport function buildResultExportUrl(filename: string, params: GetResultContentParams = {}): string {\n const searchParams = new URLSearchParams()\n Object.entries(params).forEach(([key, value]) => {\n if (value !== undefined && value !== null) {\n searchParams.set(key, String(value))\n }\n })\n const queryString = searchParams.toString()\n return `/api/results/${encodeURIComponent(filename)}/export${queryString ? `?${queryString}` : ''}`\n}\n\nexport function downloadResultExport(filename: string, params: GetResultContentParams = {}) {\n const url = buildResultExportUrl(filename, params)\n const link = document.createElement('a')\n link.href = url\n link.download = ''\n document.body.appendChild(link)\n link.click()\n document.body.removeChild(link)\n}\n" }, { "path": "web-ui/src/api/settings.ts", "content": "import { http } from '@/lib/http'\n\nexport interface NotificationSettings {\n NTFY_TOPIC_URL?: string\n GOTIFY_URL?: string\n GOTIFY_TOKEN?: string\n BARK_URL?: string\n WX_BOT_URL?: string\n TELEGRAM_BOT_TOKEN?: string\n TELEGRAM_CHAT_ID?: string\n TELEGRAM_API_BASE_URL?: string\n WEBHOOK_URL?: string\n WEBHOOK_METHOD?: string\n WEBHOOK_HEADERS?: string\n WEBHOOK_CONTENT_TYPE?: string\n WEBHOOK_QUERY_PARAMETERS?: string\n WEBHOOK_BODY?: string\n PCURL_TO_MOBILE?: boolean\n BARK_URL_SET?: boolean\n GOTIFY_TOKEN_SET?: boolean\n WX_BOT_URL_SET?: boolean\n TELEGRAM_BOT_TOKEN_SET?: boolean\n WEBHOOK_URL_SET?: boolean\n WEBHOOK_HEADERS_SET?: boolean\n CONFIGURED_CHANNELS?: string[]\n}\n\nexport interface NotificationSettingsUpdate {\n NTFY_TOPIC_URL?: string | null\n GOTIFY_URL?: string | null\n GOTIFY_TOKEN?: string | null\n BARK_URL?: string | null\n WX_BOT_URL?: string | null\n TELEGRAM_BOT_TOKEN?: string | null\n TELEGRAM_CHAT_ID?: string | null\n TELEGRAM_API_BASE_URL?: string | null\n WEBHOOK_URL?: string | null\n WEBHOOK_METHOD?: string | null\n WEBHOOK_HEADERS?: string | null\n WEBHOOK_CONTENT_TYPE?: string | null\n WEBHOOK_QUERY_PARAMETERS?: string | null\n WEBHOOK_BODY?: string | null\n PCURL_TO_MOBILE?: boolean\n}\n\nexport interface NotificationTestResponse {\n message: string\n results: Record\n}\n\nexport interface AiSettings {\n OPENAI_API_KEY?: string\n OPENAI_BASE_URL?: string\n OPENAI_MODEL_NAME?: string\n PROXY_URL?: string\n}\n\nexport interface RotationSettings {\n ACCOUNT_ROTATION_ENABLED?: boolean\n ACCOUNT_ROTATION_MODE?: string\n ACCOUNT_ROTATION_RETRY_LIMIT?: number\n ACCOUNT_BLACKLIST_TTL?: number\n ACCOUNT_STATE_DIR?: string\n PROXY_ROTATION_ENABLED?: boolean\n PROXY_ROTATION_MODE?: string\n PROXY_POOL?: string\n PROXY_ROTATION_RETRY_LIMIT?: number\n PROXY_BLACKLIST_TTL?: number\n}\n\nexport interface SystemStatus {\n scraper_running: boolean\n running_task_ids?: number[]\n ai_configured?: boolean\n notification_configured?: boolean\n headless_mode?: boolean\n running_in_docker?: boolean\n login_state_file: {\n exists: boolean\n path: string\n }\n env_file: {\n exists: boolean\n openai_api_key_set: boolean\n openai_base_url_set: boolean\n openai_model_name_set: boolean\n ntfy_topic_url_set: boolean\n gotify_url_set: boolean\n gotify_token_set: boolean\n bark_url_set: boolean\n wx_bot_url_set: boolean\n telegram_bot_token_set: boolean\n telegram_chat_id_set: boolean\n webhook_url_set: boolean\n webhook_headers_set: boolean\n }\n configured_notification_channels?: string[]\n}\n\nexport async function getNotificationSettings(): Promise {\n return await http('/api/settings/notifications')\n}\n\nexport async function updateNotificationSettings(settings: NotificationSettingsUpdate): Promise<{ message: string; configured_channels: string[] }> {\n return await http('/api/settings/notifications', {\n method: 'PUT',\n headers: { 'Content-Type': 'application/json' },\n body: JSON.stringify(settings)\n })\n}\n\nexport async function testNotificationSettings(\n payload: { channel?: string; settings: NotificationSettingsUpdate }\n): Promise {\n return await http('/api/settings/notifications/test', {\n method: 'POST',\n headers: { 'Content-Type': 'application/json' },\n body: JSON.stringify(payload)\n })\n}\n\nexport async function getAiSettings(): Promise {\n return await http('/api/settings/ai')\n}\n\nexport async function updateAiSettings(settings: AiSettings): Promise {\n await http('/api/settings/ai', {\n method: 'PUT',\n headers: { 'Content-Type': 'application/json' },\n body: JSON.stringify(settings)\n })\n}\n\nexport async function getRotationSettings(): Promise {\n return await http('/api/settings/rotation')\n}\n\nexport async function updateRotationSettings(settings: RotationSettings): Promise {\n await http('/api/settings/rotation', {\n method: 'PUT',\n headers: { 'Content-Type': 'application/json' },\n body: JSON.stringify(settings)\n })\n}\n\nexport async function testAiSettings(settings: AiSettings): Promise<{ success: boolean; message: string; response?: string }> {\n return await http('/api/settings/ai/test', {\n method: 'POST',\n headers: { 'Content-Type': 'application/json' },\n body: JSON.stringify(settings)\n })\n}\n\nexport async function getSystemStatus(): Promise {\n return await http('/api/settings/status')\n}\n\nexport async function updateLoginState(content: string): Promise<{ message: string }> {\n return await http('/api/login-state', {\n method: 'POST',\n headers: { 'Content-Type': 'application/json' },\n body: JSON.stringify({ content })\n })\n}\n\nexport async function deleteLoginState(): Promise<{ message: string }> {\n return await http('/api/login-state', { method: 'DELETE' })\n}\n" }, { "path": "web-ui/src/api/tasks.ts", "content": "import type {\n Task,\n TaskCreateResponse,\n TaskGenerateRequest,\n TaskGenerationJob,\n TaskUpdate,\n} from '@/types/task.d.ts'\nimport { http } from '@/lib/http'\n\nexport async function getAllTasks(): Promise {\n return await http('/api/tasks')\n}\n\nexport async function createTaskWithAI(data: TaskGenerateRequest): Promise {\n return await http('/api/tasks/generate', {\n method: 'POST',\n headers: {\n 'Content-Type': 'application/json',\n },\n body: JSON.stringify(data),\n })\n}\n\nexport async function getTaskGenerationJob(jobId: string): Promise {\n const result = await http(`/api/tasks/generate-jobs/${jobId}`)\n return result.job\n}\n\nexport async function updateTask(taskId: number, data: TaskUpdate): Promise {\n const result = await http(`/api/tasks/${taskId}`, {\n method: 'PATCH',\n headers: {\n 'Content-Type': 'application/json',\n },\n body: JSON.stringify(data),\n })\n return result.task\n}\n\nexport async function startTask(taskId: number): Promise {\n await http(`/api/tasks/start/${taskId}`, { method: 'POST' })\n}\n\nexport async function stopTask(taskId: number): Promise {\n await http(`/api/tasks/stop/${taskId}`, { method: 'POST' })\n}\n\nexport async function deleteTask(taskId: number): Promise {\n await http(`/api/tasks/${taskId}`, { method: 'DELETE' })\n}\n" }, { "path": "web-ui/src/assets/main.css", "content": "@tailwind base;\n@tailwind components;\n@tailwind utilities;\n\n@layer base {\n :root {\n --background: 0 0% 100%;\n --foreground: 222.2 84% 4.9%;\n\n --card: 0 0% 100%;\n --card-foreground: 222.2 84% 4.9%;\n\n --popover: 0 0% 100%;\n --popover-foreground: 222.2 84% 4.9%;\n\n --primary: 222.2 47.4% 11.2%;\n --primary-foreground: 210 40% 98%;\n\n --secondary: 210 40% 96.1%;\n --secondary-foreground: 222.2 47.4% 11.2%;\n\n --muted: 210 40% 96.1%;\n --muted-foreground: 215.4 16.3% 46.9%;\n\n --accent: 210 40% 96.1%;\n --accent-foreground: 222.2 47.4% 11.2%;\n\n --destructive: 0 84.2% 60.2%;\n --destructive-foreground: 210 40% 98%;\n\n --border: 214.3 31.8% 91.4%;\n --input: 214.3 31.8% 91.4%;\n --ring: 222.2 84% 4.9%;\n\n --radius: 0.5rem;\n }\n\n .dark {\n --background: 222.2 84% 4.9%;\n --foreground: 210 40% 98%;\n\n --card: 222.2 84% 4.9%;\n --card-foreground: 210 40% 98%;\n\n --popover: 222.2 84% 4.9%;\n --popover-foreground: 210 40% 98%;\n\n --primary: 210 40% 98%;\n --primary-foreground: 222.2 47.4% 11.2%;\n\n --secondary: 217.2 32.6% 17.5%;\n --secondary-foreground: 210 40% 98%;\n\n --muted: 217.2 32.6% 17.5%;\n --muted-foreground: 215 20.2% 65.1%;\n\n --accent: 217.2 32.6% 17.5%;\n --accent-foreground: 210 40% 98%;\n\n --destructive: 0 62.8% 30.6%;\n --destructive-foreground: 210 40% 98%;\n\n --border: 217.2 32.6% 17.5%;\n --input: 217.2 32.6% 17.5%;\n --ring: 212.7 26.8% 83.9%;\n }\n}\n\n@layer base {\n * {\n @apply border-border;\n }\n body {\n @apply bg-background text-foreground;\n }\n}" }, { "path": "web-ui/src/components/HelloWorld.vue", "content": "\n\n\n\n\n" }, { "path": "web-ui/src/components/layout/DashboardTaskSearch.vue", "content": "\n\n\n\n\n" }, { "path": "web-ui/src/components/layout/LocaleToggle.vue", "content": "\n\n\n" }, { "path": "web-ui/src/components/layout/TheHeader.vue", "content": "\n\n\n" }, { "path": "web-ui/src/components/layout/TheSidebar.vue", "content": "\n\n\n\n\n" }, { "path": "web-ui/src/components/results/PriceTrendChart.vue", "content": "\n\n\n" }, { "path": "web-ui/src/components/results/ResultCard.vue", "content": "\n\n\n" }, { "path": "web-ui/src/components/results/ResultsFilterBar.vue", "content": "\n\n\n" }, { "path": "web-ui/src/components/results/ResultsGrid.vue", "content": "\n\n\n" }, { "path": "web-ui/src/components/results/ResultsInsightsPanel.vue", "content": "\n\n\n" }, { "path": "web-ui/src/components/settings/NotificationSettingsPanel.vue", "content": "\n\n