From 43ad54ef74c7939ff6f4c91b5acf9c489d60badf Mon Sep 17 00:00:00 2001 From: bobo Date: Mon, 6 Jul 2026 10:50:50 +0800 Subject: [PATCH] Remove twitter-digest browser collection --- twitter-digest/DATA_COLLECTION.md | 299 ++------ twitter-digest/FUNCTION_RULES_FLOW.md | 259 +------ twitter-digest/README.md | 216 ++---- twitter-digest/RUNBOOK.md | 429 ++--------- twitter-digest/SKILL.md | 332 +++----- twitter-digest/agents/openai.yaml | 2 +- twitter-digest/install.sh | 9 +- twitter-digest/references/x-twitter-digest.md | 260 +------ twitter-digest/scripts/api_x_digest.py | 12 +- twitter-digest/scripts/browser_lifecycle.py | 129 ---- twitter-digest/scripts/browser_x_digest.py | 315 -------- twitter-digest/scripts/cdp_client.py | 165 ---- twitter-digest/scripts/collector_commands.py | 63 -- twitter-digest/scripts/configure_api.py | 4 +- twitter-digest/scripts/digest_context.py | 18 +- twitter-digest/scripts/digest_io.py | 8 +- twitter-digest/scripts/dm_scraper.py | 725 ------------------ twitter-digest/scripts/dom_script_loader.py | 15 - twitter-digest/scripts/dom_scripts/README.md | 13 - .../scripts/dom_scripts/count_dm_messages.js | 42 - .../scripts/dom_scripts/detect_handle.js | 75 -- .../dom_scripts/dm_loaded_beyond_window.js | 107 --- .../dom_scripts/dm_page_loading_state.js | 45 -- .../scripts/dom_scripts/extract_articles.js | 104 --- .../extract_dm_conversation_text.js | 4 - .../dom_scripts/extract_dm_messages.js | 109 --- .../dom_scripts/extract_dm_thread_targets.js | 155 ---- .../scripts/dom_scripts/extract_main_text.js | 4 - .../dom_scripts/get_dm_history_state.js | 43 -- .../dom_scripts/scroll_dm_messages_up.js | 19 - .../dom_scripts/scroll_dm_thread_list_down.js | 41 - twitter-digest/scripts/install.py | 42 +- twitter-digest/scripts/public_scraper.py | 160 ---- twitter-digest/scripts/run_daily_digest.py | 181 ++--- 34 files changed, 446 insertions(+), 3958 deletions(-) delete mode 100644 twitter-digest/scripts/browser_lifecycle.py delete mode 100644 twitter-digest/scripts/browser_x_digest.py delete mode 100644 twitter-digest/scripts/cdp_client.py delete mode 100644 twitter-digest/scripts/dm_scraper.py delete mode 100644 twitter-digest/scripts/dom_script_loader.py delete mode 100644 twitter-digest/scripts/dom_scripts/README.md delete mode 100644 twitter-digest/scripts/dom_scripts/count_dm_messages.js delete mode 100644 twitter-digest/scripts/dom_scripts/detect_handle.js delete mode 100644 twitter-digest/scripts/dom_scripts/dm_loaded_beyond_window.js delete mode 100644 twitter-digest/scripts/dom_scripts/dm_page_loading_state.js delete mode 100644 twitter-digest/scripts/dom_scripts/extract_articles.js delete mode 100644 twitter-digest/scripts/dom_scripts/extract_dm_conversation_text.js delete mode 100644 twitter-digest/scripts/dom_scripts/extract_dm_messages.js delete mode 100644 twitter-digest/scripts/dom_scripts/extract_dm_thread_targets.js delete mode 100644 twitter-digest/scripts/dom_scripts/extract_main_text.js delete mode 100644 twitter-digest/scripts/dom_scripts/get_dm_history_state.js delete mode 100644 twitter-digest/scripts/dom_scripts/scroll_dm_messages_up.js delete mode 100644 twitter-digest/scripts/dom_scripts/scroll_dm_thread_list_down.js delete mode 100644 twitter-digest/scripts/public_scraper.py diff --git a/twitter-digest/DATA_COLLECTION.md b/twitter-digest/DATA_COLLECTION.md index da1dc09..667e07d 100644 --- a/twitter-digest/DATA_COLLECTION.md +++ b/twitter-digest/DATA_COLLECTION.md @@ -1,283 +1,92 @@ -# X/Twitter Digest Data Collection Design +# Twitter Digest Data Collection -## 目标 +`twitter-digest` collects data through the X API only. Local profiles, cookies, and X page automation are not part of this skill. -抓数据层要稳定、可替换、可诊断。 +## Supported Collector -核心约束： - -- 抓取脚本只负责生成标准化的 `digest-input.*`。 -- 上层摘要逻辑只读取 `digest-context.md`。 -- 不同数据源必须产出相同的页面结构，方便后续处理复用。 -- 用户操作从对话里触发。Agent 负责运行配置、授权、抓取和总结脚本；用户只在弹出的系统输入框或浏览器授权页里输入/确认。 -- 首次配置成功后，凭据保存在本地 `.state/api_config.json`；后续生成日报不再要求用户重复输入或授权，除非 token 被撤销、过期且无法刷新，或用户主动清除配置。 - -## 三层脚本 - -### 1. 浏览器抓取脚本 - -```text -scripts/browser_x_digest.py -``` - -职责： - -- 使用本地专用 Chromium profile 登录 X。 -- 读取 `home`、`own_profile`、`mentions_search`、`mentions_notifications`。 -- 读取 X Chat 左侧会话列表。 -- 只打开 `waiting_reply` DM 会话，并尽量完整加载这些会话。 -- 输出 `digest-input.json` 和 `digest-input.md`。 - -适合： - -- 需要读取 X Chat / DM 内容的场景。 -- 用户显式选择浏览器模式时使用。 - -### 2. API 抓取脚本 - -```text -scripts/api_x_digest.py -``` - -职责： - -- 使用已保存的 OAuth2 user-context token，或 `X_BEARER_TOKEN` / `TWITTER_BEARER_TOKEN` / `--bearer-token` 传入的 OAuth2 user token。 -- 主路径是 OAuth2 PKCE：用户准备 X App 的 `Client ID`，使用 `python3 ~/.claude/skills/twitter-digest/scripts/run_daily_digest.py --configure-api` 让脚本打开授权页并通过本地 callback 换取 user access token / refresh token。 -- 如果用户已经有 OAuth2 user access token，使用 `python3 ~/.claude/skills/twitter-digest/scripts/run_daily_digest.py --configure-api-token` 由脚本安全保存。 -- OAuth1 不再作为日报 API 配置路径，因为它不能可靠读取 DM；需要完整 DM 时必须显式使用浏览器模式，API DM 拿不到时不能自动补采浏览器。 -- 读取 home timeline：`/2/users/:id/timelines/reverse_chronological`。 -- 读取用户公开发帖。 -- 读取 mentions。 -- 读取 recent search 中的 @ 提及。 -- 读取显式配置的 keyword search。 -- 输出和浏览器脚本同形状的 `digest-input.json` 和 `digest-input.md`。 - -适合： - -- 普通日报默认公开数据采集。 -- 定时任务中减少浏览器页面变化影响。 - -限制： - -- `run_daily_digest.py` 的正常日报路径不使用 API DM 做最终判断；DM / X Chat 只在显式浏览器模式中读取。 -- `api_x_digest.py --include-dms` 保留为 TODO/调试路径：它会尝试 `/2/dm_events`，但结果只作为 API 可见事件参考。 -- Home timeline endpoint 需要可访问该用户上下文的 token；如果账号权限、套餐或 token 类型不支持，会在 `home` 页面写入具体 `collection_error`。 -- App-only API key / app-only bearer token 不等于用户授权，不能保证读取用户 home timeline 或 DM。 -- DM API 需要 X App / token 有 `dm.read` 相关权限；权限不足、tier 不支持、rate limit、返回 0 条或无法确认是否需要回复时，会写入 `api_dm_todo` 和 data gap。 -- API DM 不等同于网页 X Chat。XChat / 加密私信可能不会出现在 `/2/dm_events`。现阶段 API DM 标记为 TODO，等待 X 修复或明确文档；日报 DM 以浏览器脚本为准。 -- API 权限不足、额度不足或 endpoint 不可用时，会把错误写入对应页面的 `collection_error`。 - -### 3. 上层入口脚本 - -```text -scripts/run_daily_digest.py -``` - -职责： - -- 统一入口。 -- 提供 `--configure-api`，由 Agent 在对话里触发 OAuth 配置。 -- 默认 `--source auto`。 -- 默认自动选择来源：已配置 API 时走 API；未配置 API 时走浏览器；API 已配置但认证失效时先触发 API 重配置，配置成功后继续 API 抓取。 -- 用户主动要求浏览器或命令显式传入 `--source browser` 时，强制浏览器抓取。 -- API source 已选择时不启动浏览器，不自动回退浏览器；认证类错误会触发一次重配，其他 API 不可用情况失败或写入 data gap。 -- 如果 OAuth2 access token 快过期且保存了 refresh token，自动刷新后再抓取。 -- 抓取完成后调用 `digest_context.py` 生成 `digest-context.*`。 - -选择逻辑： - -```text ---source api -> 强制 API，只采公开数据，不启动浏览器 ---source browser -> 强制浏览器，采公开网页和可见 X Chat / DM ---source auto -> 默认自动模式；已配置 API 时走 API；未配置 API 时走浏览器；API 已配置但认证坏了就配置后继续 API；不自动回退浏览器 -``` - -隔离规则： - -- API 来源只运行 `api_x_digest.py`，不启动浏览器、不读取浏览器 profile、不用浏览器补采 API 缺口。 -- 浏览器来源只运行 `browser_x_digest.py`，不读取 API token、不合并 API collector 输出。 -- 默认 `--source auto` 每次只选择一个来源，不合并 API 和浏览器两边的数据。 -- 来源不是会话状态，不能继承上一轮。上一次用浏览器不代表下一次普通“日报”继续浏览器；上一次用 API 也不要求特殊命令。每次新日报都重新运行默认入口，由脚本按当前 API 配置选择来源。 -- API 输出里的“需要浏览器确认 DM”只能作为 data gap 提示，不表示本次已经读取了浏览器 DM。 - -## 对话触发流程 - -底层只保留两个抓取脚本： - -```text -scripts/api_x_digest.py -> 官方 API 抓取：home timeline、mentions、profile；API DM 仅作为 TODO/调试 -scripts/browser_x_digest.py -> 浏览器抓取：公开网页 + X Chat / 加密 DM -``` - -普通日报： - -```text -用户：生成 X 日报 -Agent：运行 scripts/run_daily_digest.py -脚本：默认自动选择来源；已配置 API 时走 API，未配置 API 时走浏览器；API 已配置但认证失效时先重配 API，然后继续 API 采集；生成 digest-context.md -Agent：读取 digest-context.md 写中文日报 +```bash +python3 twitter-digest/scripts/api_x_digest.py ``` -配置 API，已有 token： +The chat-facing wrapper is: -```text -用户：输入 X token / 我已经有 token -Agent：运行 `python3 ~/.claude/skills/twitter-digest/scripts/run_daily_digest.py --configure-api-token` -脚本：弹出隐藏输入框 -用户：粘贴 user access token -脚本：保存 token 到 .state/api_config.json -后续普通日报仍运行 `python3 ~/.claude/skills/twitter-digest/scripts/run_daily_digest.py`，并自动走 API；要临时使用浏览器时显式运行 `--source browser` +```bash +python3 twitter-digest/scripts/run_daily_digest.py ``` -配置 API，OAuth2 授权： +The wrapper uses API directly. -```text -用户：配置 X API -Agent：运行 `python3 ~/.claude/skills/twitter-digest/scripts/run_daily_digest.py --configure-api` -脚本：直接进入 OAuth2 配置 -用户：输入 Client ID 和可选 Client Secret -脚本：打开 X 授权页 -用户：在浏览器里授权 app -脚本：通过本地 callback 换取 access token / refresh token 并保存 -后续普通日报仍运行 `python3 ~/.claude/skills/twitter-digest/scripts/run_daily_digest.py`，并自动走 API；要临时使用浏览器时显式运行 `--source browser` -``` +## Configuration -后续运行： +API credentials are saved in: ```text -用户：生成 X 日报 -Agent：运行 scripts/run_daily_digest.py -脚本：默认 auto；已配置 API 时使用 API，未配置 API 时使用浏览器；API 已配置但认证失效时先重配 API，然后继续 API 采集；生成 digest-context.md -脚本：只有显式 `--source browser` 时才强制浏览器 -Agent：读取 digest-context.md 写中文日报 +twitter-digest/.state/api_config.json ``` -如果凭据失效： +OAuth2 PKCE setup: -```text -脚本：认证类错误会打开 API 配置并重试一次 -脚本：权限、tier、限流或 endpoint 不可用时写入 data gap 或失败，不回退浏览器 -``` - -清除 API： - -```text -用户：清除 X API 配置 -Agent：运行 `python3 ~/.claude/skills/twitter-digest/scripts/configure_api.py --clear` -后续：`python3 ~/.claude/skills/twitter-digest/scripts/run_daily_digest.py` 因为没有 API 配置会走浏览器；如需重新走 API，用户需要再次配置 API +```bash +python3 twitter-digest/scripts/run_daily_digest.py --configure-api ``` -## 标准输出结构 - -两种抓取方式都输出： +Existing token setup: -```text -digest-input.json -digest-input.md +```bash +python3 twitter-digest/scripts/run_daily_digest.py --configure-api-token ``` -核心 JSON 结构： +Verification: -```json -{ - "generated_at": "...", - "source": "browser|api", - "handle": "...", - "keywords": [], - "pages": [ - { - "kind": "home|own_profile|mentions_search|mentions_notifications|messages", - "url": "...", - "items": [] - } - ] -} +```bash +python3 twitter-digest/scripts/configure_api.py --verify ``` -公开 item 尽量包含： - -- `text` -- `url` -- `time` -- `authorUrl` -- `externalLinks` -- `media` -- `cards` +## Runtime Contract -DM page 尽量包含： +- Normal daily digest requests run the wrapper with no source override. +- API credentials are required for every digest run. +- If saved API credentials exist, they are used. +- If credentials are missing or invalid, the wrapper opens API configuration. After configuration succeeds, run the digest command again. +- API errors are reported as failures or data gaps. +- The collector never switches to another data source. -- `dm_status` -- `dm_note` -- `dm_visible_thread_count` -- `dm_replied_thread_count` -- `dm_unreplied_thread_count` -- `dm_captured_message_count` -- `dm_threads` +OAuth setup can open the X authorization page. That is authorization only, not collection. -API DM TODO / 调试规则： +## Captured Public Data -- 正常日报不使用 API DM 做最终判断；`run_daily_digest.py` 不会用浏览器补 DM，除非显式传入 `--source browser`。 -- 只有直接运行 `api_x_digest.py --include-dms` 做调试时，才调用 `/2/dm_events`。 -- 按 `dm_conversation_id` 分组。 -- 只把最后一条不是用户发出的会话正文放进 `dm_threads`，用于判断是否需要回复。 -- 最后一条是用户发出的会话只计数，不展开正文。 -- 如果 `/2/dm_events` 返回权限、tier、认证或限流错误，写入 `api_dm_todo` data gap，不把失败当作“无私信”。 -- API 来源不启动浏览器，也不补采浏览器 DM；如果需要完整 DM，必须显式使用 `--source browser`。 +The API collector attempts to collect: -## 稳定性策略 +- Authenticated user profile. +- Home timeline. +- Own recent posts/profile activity. +- Mentions and handle search. +- Optional keyword search. -### 浏览器抓取 +Public items are normalized into the current-run context. Final facts are filtered to the local 24-hour window. -- 使用专用 profile，不读用户常用浏览器。 -- 通过 `auth_token` cookie 判断登录态。 -- 页面结构变动时，失败会进入 data gap，而不是生成伪结论。 -- DM 会话只打开 `waiting_reply`。 -- 需要回复的 DM 会话尽量滚到顶部。 -- 未完整加载时记录 `dm_thread_incomplete`。 +## DM Data -### API 抓取 +The API may expose DM events for some accounts, but X Chat/encrypted messages are often incomplete or absent. -- 使用标准 HTTP API。 -- 每个页面独立采集，某个 endpoint 失败不会阻塞其他页面。 -- API 错误写入页面级 `collection_error`。 -- Home timeline 会先尝试官方 timeline endpoint，只有 endpoint 返回权限/额度/可用性错误时才进入 data gap。 -- 没有 API token 且没有 API 配置时，上层入口走浏览器；已有 API 配置但认证失效时，上层入口触发 API 重配置并继续 API 采集；不自动回退浏览器。 +Rules: -### 上层摘要 +- Never claim "no DMs" from zero API DM events. +- Treat API DM failures as data gaps. +- Non-API DM collection is not part of this skill. -- 不直接读取原始页面。 -- 只读取 `digest-context.md`。 -- `digest-context.md` 包含 data gaps。 -- 模型必须把 data gaps 明确写进日报。 +## Output Shape -## 常用命令 +Current-run files: -默认自动模式： - -```bash -python3 twitter-digest/scripts/run_daily_digest.py -``` - -强制浏览器： - -```bash -python3 twitter-digest/scripts/run_daily_digest.py --source browser -``` - -强制 API： - -```bash -X_BEARER_TOKEN=... python3 twitter-digest/scripts/run_daily_digest.py --source api --handle -``` - -只跑浏览器抓取： - -```bash -python3 twitter-digest/scripts/browser_x_digest.py --include-dms +```text +.state/run/digest-input.json +.state/run/digest-input.md +.state/run/digest-context.md +.state/run/digest-context.json +.state/run/digest-context-timeline.md +.state/run/digest-context-mentions.md +.state/run/digest-context-dm.md ``` -只跑 API 抓取： - -```bash -X_BEARER_TOKEN=... python3 twitter-digest/scripts/api_x_digest.py --handle -``` +No long-term memory or daily archive is written. diff --git a/twitter-digest/FUNCTION_RULES_FLOW.md b/twitter-digest/FUNCTION_RULES_FLOW.md index 01b2467..25e090a 100644 --- a/twitter-digest/FUNCTION_RULES_FLOW.md +++ b/twitter-digest/FUNCTION_RULES_FLOW.md @@ -1,251 +1,52 @@ -# X/Twitter Digest 功能 / 规则 / 流程说明 +# Twitter Digest Function Rules Flow -面向 PM/用户的完整运行流程说明见： +## Source Rule -```text -twitter-digest/RUNBOOK.md -``` - -## 1. 功能 - -`twitter-digest` 默认 `--source auto`：已配置 API 时通过 API 读取公开数据并生成中文日报；未配置 API 时使用浏览器采集。API 已配置但认证失效时，会先触发 API 重配置，配置成功后继续 API 采集。用户主动要求浏览器或显式传入 `--source browser` 时强制浏览器。 - -采集内容按来源区分： - -- Home timeline：时间线热点 -- Mentions search：搜索谁 @ 了当前账号 -- Own profile：当前账号主页动态 -- API 模式：公开数据为主，包括 home timeline、mentions / @ 搜索、own profile；不以 DM 作为日报依据。 -- 浏览器模式：包含 home timeline、mentions search、mentions notifications、own profile，以及私信页面可见会话。 - -默认不采集关键词。只有显式传 `--keywords` 时才做关键词搜索。 - -公开页采集范围： - -- timeline、profile、mentions 默认按日报目标加载近 24 小时内容。 -- API public 默认最多保留 300 条公开帖子；浏览器公开页默认最多滚动 40 次，并最多保留 100 条公开帖子进入 `digest-context`。 -- 如果已加载帖子的时间戳显示已经超过 24 小时窗口，会提前停止滚动。 -- 公开帖子数量表示“本次浏览器加载到的帖子”，不是完整 X 历史。 - -## 2. 数据保存位置 - -安装脚本会按当前工具选择 skills 目录：Codex 使用 `~/.codex/skills`，Claude Code 使用 `~/.claude/skills`。旧的 `twitter-briefing`、`twitter-briefing.bak` 或已有 `twitter-digest` 安装会迁移到所选 skills 目录的 `.backups/`，并把备份里的 `SKILL.md` 改成 `SKILL.md.disabled`，避免当前工具加载重复旧 skill。 - -登录状态保存在： - -```text -twitter-digest/.state/chrome-profile -``` - -这不是 `/tmp`。只要这个目录不删除、X 登录态不过期，后续运行会复用。 - -每次运行的临时采集结果保存在： - -```text -twitter-digest/.state/run/ -``` - -主要文件： +`twitter-digest` is API-only. ```text -twitter-digest/.state/run/digest-input.json -twitter-digest/.state/run/digest-input.md -twitter-digest/.state/run/digest-context.json -twitter-digest/.state/run/digest-context.md -``` - -用途： - -- `digest-context.md/json`：最终总结主输入，包含本次采集归一化后的 `Final Summary Facts`。 -- `digest-input.md/json`：原始采集结果，只在需要核对细节或排查抓取问题时使用。 - -不生成长期 `memory.json`，不生成 `daily/` 历史归档。 -`twitter-digest/.state/run/` 会尽量设置为 700 权限，避免把当次 DM 原文放到全局可读的 `/tmp`。 - -## 3. 运行规则 - -- 默认入口 `run_daily_digest.py`：自动选择来源。已配置 API 时走 API；未配置 API 时走浏览器；API 已配置但认证失效时自动进入配置流程，配置成功后继续 API 采集。 -- 用户主动要求浏览器或命令显式 `--source browser` 时强制浏览器；API 模式只抓公开数据，不打开浏览器。 -- DM / X Chat 以本地浏览器抓取为准；API 模式不用于判断是否有私信。 -- API 认证类错误会触发一次重配；权限不足、tier 不支持、限流或其他 API 错误记录数据缺口或失败，不回退浏览器路径。 -- 不要求用户复制 cookie 或 token。 -- 默认 headless 运行。 -- 第一次没有登录态时，会自动打开可见浏览器让用户登录。 -- 浏览器模式默认读取 DM，但只读取浏览器页面上可见的内容；API 模式不以 DM 作为日报依据。 -- 支持 `--non-interactive`，定时任务遇到 passcode 时跳过 DM 恢复并记录数据缺口，不阻塞等待。 -- 不自动发送消息、回复、点赞、关注、拉黑、打开可疑链接或接受 DM 请求。 -- 只生成摘要和建议回复草稿。 -- DM 原文只用于当次总结，不写入长期状态文件。 - -## 4. 首次运行流程 - -1. 用户运行： - - ```bash - python3 twitter-digest/scripts/run_daily_digest.py - ``` - -2. 脚本检查 API 配置；如果缺失，走浏览器；如果已配置但认证失效，打开 API 重配置流程。 - -3. 用户在配置流程里输入 Client ID / Secret 并完成 X OAuth 授权。 - -4. 脚本保存并验证 API 配置。 - -5. 脚本继续用 API 采集 timeline、mentions、own profile 等公开数据。 - -如果用户显式使用 `--source browser`： - -1. 脚本先尝试用 headless 浏览器读取保存的登录态。 -2. 如果没有登录态，脚本自动打开可见浏览器窗口。 -3. 用户在该浏览器窗口里登录 X。 -4. 脚本自动识别当前 X 账号 handle；如果 headless 识别失败，会打开可见浏览器重试。 -5. 如果仍无法识别 handle，脚本直接停止并要求用户带 `--handle <用户名>` 重跑，不能生成缺 `@你` 和自己动态的日报。 -6. 识别成功后，脚本采集 timeline、mentions、own profile、DM。 - -9. 脚本生成 `twitter-digest/.state/run/*` 当次采集文件。 - -10. Agent 只读取 `twitter-digest/.state/run/digest-context.md` 生成中文日报。 - -11. `digest-input.*` 只在排查抓取问题时使用。 - -## 5. 后续运行流程 - -1. 用户再次运行： - - ```bash - python3 twitter-digest/scripts/run_daily_digest.py - ``` - -2. 脚本默认 headless 启动。 - -3. 脚本复用： - - ```text - twitter-digest/.state/chrome-profile - ``` - -4. 如果登录态有效，不弹浏览器窗口。 - -5. 脚本直接采集并生成日报输入文件。 - -6. Agent 生成中文日报。 - -## 6. 需要人工介入的情况 - -以下情况会打开可见浏览器窗口： - -- 第一次使用，还没有登录态。 -- X session 过期。 -- 用户退出了 X。 -- X 要求 CAPTCHA 或风控验证。 -- X Chat 要求设置或输入 passcode，此时会自动打开可见浏览器窗口，等待用户处理后继续。 -- `twitter-digest/.state/chrome-profile` 被删除。 - -## 7. DM 规则 - -DM 默认读取。 - -读取范围： - -- 只读 X Messages 通过浏览器加载出来的内容；打开等我回复会话后会自动向上滚动加载更多消息。 -- 默认先向下扫描 X Chat 左侧会话列表，最多 20 轮，尽量覆盖今天的可见会话，而不是只看首屏。 -- 只统计今天可见会话数量、最后我发出的数量、等我回复的数量；列表里更早的历史会话不计入日报会话数。 -- “最后我发出”只看 X Chat 列表最后预览是否是 `You:` / `You sent` / `你:`，不是指会话历史里曾经回复过。 -- 消息数量单独统计，只来自已打开的等我回复会话里的消息气泡，不能和会话数量混用。 -- 默认每个等我回复会话会尽量向上滚到对话顶部，完整捕获浏览器可加载的会话历史。 -- 默认安全上限是向上滚动 200 次，并最多保留 2000 条消息气泡；`--dm-window-hours 0` 表示不按 24 小时窗口截断 DM 历史。 -- 如果没有滚到顶部或命中消息上限，会在 `digest-context` 的数据缺口里记录 `dm_thread_incomplete`，不能假装完整。 -- 如果会话列表没有扫到底，会在 `digest-context` 的数据缺口里记录 `dm_list_incomplete`，日报需要说明可能还有会话未覆盖。 -- 如果 X Messages 显示骨架屏、占位条或右侧 `Start Conversation` 但左侧会话列表仍未真实加载，脚本会自动重载 `/messages`，最多重试 3 次；仍失败时记录 `dm_page_loading_timeout`，不能写成“没有私信”。 -- `digest-context.md` 会为需要总结的等我回复会话输出 `DM Thread Context`，最多带 2000 条已加载消息，并保留 raw label、URL、加载状态等原始信息，方便模型理解复杂上下文。 -- 默认只打开今天等我回复的会话。 -- 发信人以 `participant` / `会话对象` 和消息气泡方向为准，不能把引用帖、转发卡片、链接预览里的作者当作 DM 发信人。 -- 如果今天可见会话最后一条都是我发出的，会记录 `no_unreplied_threads`，日报应写“今天可见私信会话最后一条都是我发出的，无需处理”，不能写“没有私信”。 -- 如果能看到会话列表但打不开等我回复的正文，会记录 `visible_threads_unopened`。 -- 只有 `captured_unreplied_threads` 的私信，才进入 DM 摘要。 -- 等我回复的私信也要挑重点总结；垃圾、钓鱼、低质营销、重复无关内容只计数并归为忽略，不要展开正文。 -- 如果 inbox 为空，会记录 `no_visible_threads`。 -- 如果 X Chat 要求 passcode，headless 会自动切到可见浏览器窗口，等待用户输入或完成设置后重试 DM 采集。 - -隐私规则： - -- 不长期保存 DM 原文。 -- 不写 `memory.json`。 -- 不写 `daily/` 历史归档。 -- 不保存 DM thread 状态签名。 - -不保存： - -- DM 原文 -- X cookie -- token -- passcode -- 浏览器截图 - -## 8. 常用命令 - -生成今日 X 日报： - -```bash -python3 twitter-digest/scripts/run_daily_digest.py +normal digest -> run_daily_digest.py -> api_x_digest.py -> digest_context.py ``` -跳过 DM： +The wrapper uses API directly. -```bash -python3 twitter-digest/scripts/run_daily_digest.py --source browser --no-dms -``` +## Agent Rules -强制显示浏览器窗口： +- For "生成X日报", "日报", "要", "继续", or similar short confirmations, run the installed `run_daily_digest.py`. +- Do not infer another source from prior conversation. +- Do not propose another collector as a fallback for missing DMs or API gaps. +- If the user asks for a non-API source, say this skill only supports API collection. -```bash -python3 twitter-digest/scripts/run_daily_digest.py --source browser --headed -``` +## API Configuration Flow -无人值守定时运行： +1. Wrapper checks saved API config. +2. If token exists, refresh it if needed. +3. If no usable token exists, open API configuration. +4. After configuration succeeds, rerun API collection. +5. If collection fails because of permission, tier, endpoint, or rate-limit issues, report the gap or failure. -```bash -python3 twitter-digest/scripts/run_daily_digest.py --non-interactive -``` +OAuth authorization may open the X authorization page. No page data is collected. -增加滚动覆盖： - -```bash -python3 twitter-digest/scripts/run_daily_digest.py --source browser --scrolls 5 -``` +## 24-Hour Window -增加关键词搜索： +Use the user's local timezone. Final facts must fall inside: -```bash -python3 twitter-digest/scripts/run_daily_digest.py --keywords "AI,crypto" -``` - -清理登录状态并重新登录： - -```bash -rm -rf twitter-digest/.state/chrome-profile +```text +[now - 24h, now] ``` -## 10. 预期日报结构 - -日报默认用中文输出： +Older mentions must not be listed as current reply tasks. -```markdown -## 🐦 X 日报 - YYYY-MM-DD +## Reply State -**📌 今日总结** +- `already_replied` / `handled`: do not show as pending. +- `reply_unverified`: show as `回复状态未确认`. +- Missing source: report as data gap. -**✅ 该处理** +## DM Rule -**◆ 谁 @ 了你** +API DM data is not authoritative for X Chat. If unavailable or zero, report a data gap and avoid saying "没有私信". -**◆ 私信（DM）** +## Privacy -**◆ 时间线热点** - -**◆ 你的动态** - -**✍️ 建议回复草稿** - -**⚠️ 数据缺口** -``` +Use file Read on `digest-context.md` and slices. Do not shell-print private run context during normal summarization. diff --git a/twitter-digest/README.md b/twitter-digest/README.md index caec438..14fe2df 100644 --- a/twitter-digest/README.md +++ b/twitter-digest/README.md @@ -1,226 +1,146 @@ # X/Twitter Digest -Skill for generating a Chinese daily digest from a user's own X/Twitter account. The main command is automatic: it uses saved X API credentials when configured, otherwise it uses browser collection. Browser collection can also be requested explicitly. +Skill for generating a Chinese daily digest from a user's own X/Twitter account. This version is API-only. ## Quick Install -From a fresh checkout: +Codex: ```bash -git clone git@github.com:BofAI/skills.git -cd skills -python3 twitter-digest/scripts/install.py +curl -fsSL https://raw.githubusercontent.com/BofAI/skills/v1.5.13/twitter-digest/install.sh | env TWITTER_DIGEST_INSTALL_CLIENT=codex sh ``` -To ask Codex to install this skill for itself, paste this into Codex: +Claude Code: ```bash -curl -fsSL https://raw.githubusercontent.com/BofAI/skills/v1.5.12-beta.12/twitter-digest/install.sh | env TWITTER_DIGEST_INSTALL_CLIENT=codex sh +curl -fsSL https://raw.githubusercontent.com/BofAI/skills/v1.5.13/twitter-digest/install.sh | env TWITTER_DIGEST_INSTALL_CLIENT=claude TWITTER_DIGEST_ALLOW_CLAUDE_COMMANDS=1 TWITTER_DIGEST_ALLOW_CLAUDE_STATE_READ=1 sh ``` -When this one-line installer is launched from Codex, Claude Code, or another non-interactive agent on macOS, it opens a real Terminal window and re-runs the full installation there. This avoids agent permission/inspect prompts during `git clone`, Python checks, browser checks, and installer writes. Set `TWITTER_DIGEST_OPEN_TERMINAL=0` only when intentionally running in an already interactive Terminal or CI. - -Or use the natural-language prompt: - -```text -请帮我安装这个 Codex skill： - -git clone git@github.com:BofAI/skills.git bofai-skills \ - && cd bofai-skills \ - && python3 twitter-digest/scripts/install.py --client codex - -安装后请确认 ~/.codex/skills/twitter-digest 存在。首次运行日报时，如果缺少 X API 配置，请打开配置流程并让我完成授权；只有我明确要求浏览器模式时才打开 X 登录浏览器。 -``` - -To ask Claude Code to install this skill for itself, paste this into Claude Code: +From a checkout: ```bash -curl -fsSL https://raw.githubusercontent.com/BofAI/skills/v1.5.12-beta.12/twitter-digest/install.sh | env TWITTER_DIGEST_INSTALL_CLIENT=claude TWITTER_DIGEST_ALLOW_CLAUDE_COMMANDS=1 TWITTER_DIGEST_ALLOW_CLAUDE_STATE_READ=1 sh +python3 twitter-digest/scripts/install.py ``` -Or use the natural-language prompt: +The installer opens a real macOS Terminal when launched from Codex, Claude Code, or another non-interactive agent. Set `TWITTER_DIGEST_OPEN_TERMINAL=0` only when intentionally running inside an interactive Terminal or CI. -```text -请帮我安装这个 Claude Code skill： - -git clone git@github.com:BofAI/skills.git bofai-skills \ - && cd bofai-skills \ - && python3 twitter-digest/scripts/install.py --client claude --allow-claude-commands --allow-claude-state-read - -安装后请确认 ~/.claude/skills/twitter-digest 存在。首次运行日报时，如果 Claude Code 弹出 Bash 授权，批准 `python3 ~/.claude/skills/twitter-digest/scripts/run_daily_digest.py`；如果缺少 X API 配置，请打开配置流程并让我完成授权。只有我明确要求浏览器模式时才打开 X 登录浏览器。 -``` - -The installer chooses the target by client: +Install targets: ```text Codex: ~/.codex/skills/twitter-digest Claude Code: ~/.claude/skills/twitter-digest ``` -It also checks for Python 3.9+ and a supported Chromium browser: Google Chrome, Chromium, Microsoft Edge, or Brave. The installer does not install or upgrade Python or browsers; if a prerequisite is missing, install it and rerun the same install command. -Reinstalling is the upgrade path: it replaces the skill code and preserves the existing installed `.state` directory, including saved API and browser-session settings. The installer still excludes `.state` from the development checkout. -After installation, run the installed copy. If a script is accidentally started from a temporary clone while an installed copy exists, it automatically re-runs the installed copy so state is saved under `~/.claude/skills/twitter-digest/.state` or `~/.codex/skills/twitter-digest/.state`. - -## Uninstall - -Safe uninstall moves the installed skill to `.backups/` and preserves `.state`: +The installer requires Python 3.9+. Reinstalling is the upgrade path: existing code is replaced and the installed `.state` directory is preserved. -```bash -~/.codex/skills/twitter-digest/uninstall.sh --client codex -~/.claude/skills/twitter-digest/uninstall.sh --client claude -``` +## Run -To permanently remove the installed skill, `.state`, and matching `.backups` entries, add `--purge-state`. - -Use one stable installed command form for normal runs. This prevents repeated Claude Code Bash permission prompts across different projects: +Use the installed command: ```text Claude Code: python3 ~/.claude/skills/twitter-digest/scripts/run_daily_digest.py Codex: python3 ~/.codex/skills/twitter-digest/scripts/run_daily_digest.py ``` -Claude Code cannot let a skill silently grant itself Bash permission or file access outside the project. Either approve the first visible `run_daily_digest.py` prompt and any file-access prompt, or install with `--allow-claude-commands --allow-claude-state-read` to explicitly add one global command allow rule and one `.state` read directory. - -## First Run +`run_daily_digest.py` uses API. A valid API configuration is required before a digest can be generated. If API credentials are missing or expired, the wrapper opens the API configuration flow. After configuration succeeds, run the digest command again. -```bash -python3 ~/.claude/skills/twitter-digest/scripts/run_daily_digest.py -``` +## Required API Configuration -On first run, the script checks X API configuration. If API credentials are missing, it uses browser collection. If API credentials are configured, a normal digest request or short follow-up such as "要", "日报", "继续", or "生成" uses API. Use browser source explicitly when you want local browser collection, visible DMs, or X Chat after API has been configured. - -## Data Collection Sources - -There are three collection entry points: +From chat or Terminal: ```bash -# Browser-only collector -python3 twitter-digest/scripts/browser_x_digest.py --include-dms - -# API-only collector, requires OAuth2 user-context credentials or --bearer-token -X_BEARER_TOKEN=... python3 twitter-digest/scripts/api_x_digest.py --handle - -# Recommended upper-level wrapper -python3 ~/.claude/skills/twitter-digest/scripts/run_daily_digest.py +python3 ~/.claude/skills/twitter-digest/scripts/run_daily_digest.py --configure-api +python3 ~/.codex/skills/twitter-digest/scripts/run_daily_digest.py --configure-api ``` -`run_daily_digest.py` defaults to `--source auto`: - -- A normal daily run uses API when API credentials are configured. -- If no API credentials are configured, a normal daily run uses browser collection. -- If API credentials are configured but authentication is broken, it opens API configuration and then retries API collection once. -- Use `--source browser` when the user explicitly wants browser collection after API has been configured. +The setup flow is required for first use. It uses OAuth2 PKCE, asks for the X Developer App Client ID, opens the X authorization page, waits for the local callback, and saves a user-context access token plus refresh token. -Source isolation: +Scopes: -- API mode only runs `api_x_digest.py`; it never opens a browser or reads the browser profile. -- Browser mode only runs `browser_x_digest.py`; it does not use API tokens or API collector output. -- Default `--source auto` picks one source for the run and does not merge API and browser data. -- API DM data gaps are notes only; they do not mean browser DM data was collected. +```text +tweet.read users.read offline.access dm.read +``` -Force API source: +If the user already has an OAuth2 user access token: ```bash -X_BEARER_TOKEN=... python3 ~/.claude/skills/twitter-digest/scripts/run_daily_digest.py --source api --handle +python3 ~/.claude/skills/twitter-digest/scripts/run_daily_digest.py --configure-api-token +python3 ~/.codex/skills/twitter-digest/scripts/run_daily_digest.py --configure-api-token ``` -API mode is for stable public-data collection, including the official home timeline endpoint when the configured token has user-context timeline access. API mode never starts a browser, never reads the browser profile, and never collects X Chat / DM content from the browser. Use browser mode when visible X Chat / DM is required. API DM lookup is marked TODO because XChat / encrypted DMs may not appear in `/2/dm_events`; do not use API DM to conclude there are no private messages. App-only API keys are not enough for user-context data. - -## Configure API In Chat - -Users should trigger every flow from chat. They do not need to export environment variables manually. Ask the agent to run: +Verify: ```bash -python3 ~/.claude/skills/twitter-digest/scripts/run_daily_digest.py --configure-api +python3 ~/.claude/skills/twitter-digest/scripts/configure_api.py --verify +python3 ~/.codex/skills/twitter-digest/scripts/configure_api.py --verify ``` -For a local user-owned X Developer App, the supported API setup path is OAuth2 Authorization Code with PKCE. Ask the agent to run: +Clear: ```bash -python3 ~/.claude/skills/twitter-digest/scripts/run_daily_digest.py --configure-api +python3 ~/.claude/skills/twitter-digest/scripts/configure_api.py --clear +python3 ~/.codex/skills/twitter-digest/scripts/configure_api.py --clear ``` -The script uses OAuth2 directly. It asks for the X Developer App `Client ID`, opens the X authorization page, waits for the user to authorize the account, receives the local callback, exchanges it for a user-context access token and refresh token, then saves it. In a non-interactive agent session, it opens a real Terminal window for Client ID / Secret input and closes it after the flow ends. +## Data Source -If the user already has an OAuth2 user access token, use the direct token path: +Only one collector is supported: ```bash -python3 ~/.claude/skills/twitter-digest/scripts/run_daily_digest.py --configure-api-token +python3 twitter-digest/scripts/api_x_digest.py ``` -The script opens a hidden system prompt for the token, then asks for optional handle/user id and saves the config locally. +API source collects: -OAuth1 is no longer exposed as a normal setup path for this skill because it did not reliably return DM data during validation. Use OAuth2 with user-context scopes for API collection; use explicit browser mode when visible X Chat / DM collection is required. +- Home timeline when the token has user-context access. +- Mentions and recent search results. +- Own profile activity. +- Optional keyword searches. -The app's callback URL in X Developer Portal must match the redirect URI shown by the script, by default: +DM / X Chat limitation: -```text -http://127.0.0.1:8765/callback -``` +- API DM results are incomplete for many accounts. +- Do not conclude "no DMs" from zero API DM events. +- Non-API DM collection is not part of this skill. -For API DM lookup through OAuth2, include these scopes when configuring the X App / OAuth flow: +All final facts are filtered to the user's current local 24-hour window. -```text -dm.read tweet.read users.read offline.access -``` - -Use `dm.write` only if the app will send or delete messages; the digest skill only reads. - -On macOS prompts appear as system dialogs; non-GUI terminals fall back to hidden terminal input. The token is saved to: +## Outputs ```text -twitter-digest/.state/api_config.json +/.state/run/digest-context.md +/.state/run/digest-context.json +/.state/run/digest-context-timeline.md +/.state/run/digest-context-mentions.md +/.state/run/digest-context-dm.md +/.state/run/digest-input.md +/.state/run/digest-input.json ``` -The file is created with owner-only permissions where supported. Later normal runs read this saved config and use API without opening the browser. If this file is absent, normal runs use browser collection. Use `run_daily_digest.py --source browser` when the user explicitly wants browser collection after API has been configured. To clear API config: - -```bash -python3 ~/.claude/skills/twitter-digest/scripts/configure_api.py --clear -``` +Use `digest-context.md` as the normal AI input. Use `digest-input.*` only for debugging. -If OAuth returns a refresh token, later API-source runs refresh the saved access token automatically before collection. If refresh/authentication fails, the main collection command reopens API configuration and retries once. - -Chat flow summary: - -```text -生成 X 日报 -> agent runs python3 ~/.claude/skills/twitter-digest/scripts/run_daily_digest.py -输入 X token -> agent runs python3 ~/.claude/skills/twitter-digest/scripts/run_daily_digest.py --configure-api-token -配置 X API -> agent runs python3 ~/.claude/skills/twitter-digest/scripts/run_daily_digest.py --configure-api -清除 X API 配置 -> agent runs python3 ~/.claude/skills/twitter-digest/scripts/configure_api.py --clear -调试浏览器 -> agent runs python3 ~/.claude/skills/twitter-digest/scripts/run_daily_digest.py --source browser --headed -``` +## Uninstall -## Main Outputs +Safe uninstall moves the installed skill to `.backups/` and preserves `.state`: -```text -twitter-digest/.state/run/digest-context.md -twitter-digest/.state/run/digest-context.json -twitter-digest/.state/run/digest-context-timeline.md -twitter-digest/.state/run/digest-context-mentions.md -twitter-digest/.state/run/digest-context-dm.md -twitter-digest/.state/run/digest-input.md -twitter-digest/.state/run/digest-input.json +```bash +~/.codex/skills/twitter-digest/uninstall.sh --client codex +~/.claude/skills/twitter-digest/uninstall.sh --client claude ``` -Use `digest-context.md` as the normal AI input. Use the split context files when the agent needs only one section: timeline/profile, mentions, or DM. `digest-input.*` is raw collector capture for debugging. +To permanently remove the installed skill, `.state`, and matching `.backups` entries: -During analysis/summary writing, the agent should use its file Read tool to read: - -```text -~/.claude/skills/twitter-digest/.state/run/digest-context.md -~/.claude/skills/twitter-digest/.state/run/digest-context-timeline.md -~/.claude/skills/twitter-digest/.state/run/digest-context-mentions.md -~/.claude/skills/twitter-digest/.state/run/digest-context-dm.md +```bash +~/.codex/skills/twitter-digest/uninstall.sh --client codex --purge-state +~/.claude/skills/twitter-digest/uninstall.sh --client claude --purge-state ``` -Do not use `cat`, `head`, `grep`, `python3 -c`, or temporary scripts to read context files during normal summary generation; those shell reads create extra Claude Code permission prompts. - -## More Details +## Details See: -```text -twitter-digest/DATA_COLLECTION.md -twitter-digest/RUNBOOK.md -twitter-digest/FUNCTION_RULES_FLOW.md -``` +- `SKILL.md` +- `RUNBOOK.md` +- `DATA_COLLECTION.md` +- `FUNCTION_RULES_FLOW.md` +- `references/x-twitter-digest.md` diff --git a/twitter-digest/RUNBOOK.md b/twitter-digest/RUNBOOK.md index 0db8a83..50b734e 100644 --- a/twitter-digest/RUNBOOK.md +++ b/twitter-digest/RUNBOOK.md @@ -1,417 +1,102 @@ -# X/Twitter Digest Skill Runbook +# Twitter Digest Runbook -## 目标 +`twitter-digest` is API-only. -`twitter-digest` 读取用户自己的 X/Twitter 数据并生成中文日报。抓数据层支持 API 和本地已登录浏览器两种来源。 +## Entry Points -默认入口是 `scripts/run_daily_digest.py`，默认 `--source auto`。普通日报每次重新判断来源：已配置 API 时走 API；没有 API 配置时走浏览器；已配置 API 但 token 刷新失败或认证失效时触发 API 重配置，配置成功后继续 API 采集。上一轮使用浏览器或 API 不会成为下一轮的默认来源。只有用户主动要求浏览器，或命令显式传入 `--source browser` 时，才强制浏览器。API 模式只抓公开数据，不打开浏览器。 - -核心链路： - -```text -run_daily_digest.py --> 选择 API 或浏览器 collector --> 生成 digest-input.* --> 归一化生成 digest-context.md --> Agent 只基于 digest-context.md 写中文日报 -``` - -## 抓数据脚本分层 - -当前有三层脚本： - -```text -scripts/browser_x_digest.py - 通过本地浏览器抓取数据。 - -scripts/api_x_digest.py - 通过 X API 抓取数据。 - -scripts/run_daily_digest.py - 上层入口。默认 --source auto： - - 已配置 API 时走 API。 - - 没有 API 配置时走浏览器。 - - 已配置 API 但 token 刷新失败或认证失效时触发 API 重配置。 - - 重配置成功后继续 API 采集。 - - 用户主动要求浏览器或显式 `--source browser` 时强制浏览器。 - - API source 已选择时不回退浏览器；认证类错误只重配一次，其他 API 失败就报错或写 data gap。 - - 使用 API 且保存了 refresh token 时自动刷新过期 access token。 -``` - -强制浏览器： +Normal digest: ```bash -python3 twitter-digest/scripts/run_daily_digest.py --source browser +python3 ~/.claude/skills/twitter-digest/scripts/run_daily_digest.py +python3 ~/.codex/skills/twitter-digest/scripts/run_daily_digest.py ``` -强制 API： - -```bash -X_BEARER_TOKEN=... python3 twitter-digest/scripts/run_daily_digest.py --source api --handle -``` - -API 模式重点用于更稳定地抓公开数据。API 模式不会打开浏览器。X Chat / DM 内容只在显式浏览器模式中读取；API 模式不使用 DM 作为日报判断依据。 - -来源隔离规则： - -- API 模式只运行 `api_x_digest.py`，不启动浏览器、不读取浏览器 profile、不用浏览器补采 API 缺口。 -- 浏览器模式只运行 `browser_x_digest.py`，不读取 API token、不合并 API collector 输出。 -- 默认 `--source auto` 每次只选择一个来源，不合并 API 和浏览器两边的数据。 -- API 模式输出的 DM 浏览器确认提示只是 data gap，不代表浏览器数据已被采集。 - -## 对话内 API 授权 - -用户不需要自己 export 环境变量。用户在对话里说“配置 X API”时，Agent 运行： +Configure API: ```bash python3 ~/.claude/skills/twitter-digest/scripts/run_daily_digest.py --configure-api +python3 ~/.codex/skills/twitter-digest/scripts/run_daily_digest.py --configure-api ``` -如果当前对话环境没有可交互 TTY，脚本会自动打开一个真实 Terminal 窗口来输入 Client ID / Secret 和等待 OAuth callback。不要把 `configure_api.py --oauth` 放到后台 shell 里跑；如果误从临时 clone 目录运行，脚本也会自动切回已安装的 skill 目录，避免把配置写到 `/tmp`。 - -推荐路径是 OAuth2 PKCE，适合用户自己申请的本地 X Developer App： - -1. 脚本直接进入 OAuth2 配置。 -2. 脚本提示输入 X App 的 Client ID。 -3. 脚本提示输入 X App 的 Client Secret；public PKCE app 可留空。 -4. 脚本打开 X 授权页。 -5. 用户在浏览器里授权 app。 -6. 脚本通过本地 callback 收到授权码。 -7. 脚本换取 user access token 和 refresh token。 -8. token 保存到已安装 skill 的 `.state/api_config.json`，文件权限尽量设为 owner-only。 -9. 后续普通日报通过默认 auto 命令自动走 API；要临时使用浏览器时显式运行 `--source browser`。 - -用户只需要准备： - -```text -CLIENT_ID -CLIENT_SECRET（如 App 要求） -``` - -`ACCESS_TOKEN` 和 `REFRESH_TOKEN` 由授权流程生成。 - -配置成功后，后续日报不再要求用户输入 Client ID、Secret 或重新授权。普通 `run_daily_digest.py` 会读取 `.state/api_config.json` 并默认走 API；如果没有这个 API 配置文件，普通日报会走浏览器： - -- OAuth2：如果保存了 refresh token，access token 快过期时自动 refresh。 -- 如果 token 被撤销、过期且无法刷新，或 API 返回 401 等认证错误，普通日报命令会重新打开配置并重试一次。 -- 如果权限变更、tier 不支持、限流或 endpoint 不可用，脚本把 endpoint 错误写入 data gap 或失败，不自动切到浏览器。 - -OAuth1 PIN 不再作为正常配置路径：验证中它不能可靠读取 DM，不适合这个日报 skill 的核心目标。 - -X Developer App 里需要配置 callback URL，默认： - -```text -http://127.0.0.1:8765/callback -``` - -如果用户已有 user-context access token，可以单独运行 `python3 ~/.claude/skills/twitter-digest/scripts/run_daily_digest.py --configure-api-token`。不要要求用户手动配置 shell 环境变量；这是调试路径，不是主流程。 - -## 安装与依赖检查 - -安装命令： +Verify API: ```bash -python3 twitter-digest/scripts/install.py +python3 ~/.claude/skills/twitter-digest/scripts/configure_api.py --verify +python3 ~/.codex/skills/twitter-digest/scripts/configure_api.py --verify ``` -安装脚本默认安装到当前工具对应的 skills 目录： - -- 在 Codex 里运行：`~/.codex/skills/twitter-digest` -- 在 Claude Code 里运行：`~/.claude/skills/twitter-digest` +The wrapper uses API directly. -如需显式指定： +## Runtime Flow -```bash -python3 twitter-digest/scripts/install.py --client codex -python3 twitter-digest/scripts/install.py --client claude -``` +1. `run_daily_digest.py` loads saved `.state/api_config.json`. +2. If an OAuth refresh token exists, it refreshes the access token when needed. +3. If no usable token exists, it opens API configuration in a real Terminal when necessary, then exits the current command with `api_configuration_required`. +4. After the user finishes API configuration, run the digest command again. +5. It runs `api_x_digest.py` only after a usable API token exists. +6. It builds current-run context files with `digest_context.py`. -安装脚本会检查： +OAuth authorization may open the X authorization page, but that is not data collection. -- Python 3.9+ -- Google Chrome、Chromium、Microsoft Edge 或 Brave +## Upgrade Behavior -如果缺少支持的 Chromium 浏览器，安装脚本会停止并提示先安装浏览器。浏览器用于未配置 API 时的默认本地 X 页面采集、显式 `--source browser` 采集，以及 OAuth 授权页打开。已配置 API 的默认日报不会启动浏览器采集。 - -如果浏览器会稍后安装，可以显式跳过检查： - -```bash -python3 twitter-digest/scripts/install.py --skip-browser-check -``` +Reinstalling is upgrading: -安装脚本不会复制 `.state/`，因此不会把开发机器上的 X 登录态、运行结果或 DM 原文复制到用户的 skill 安装目录。 +- Existing installed skill code is moved to `.backups/`. +- Backup `SKILL.md` files are disabled so agents do not load old versions. +- The active installed `.state` directory is restored into the new install. -## 显式浏览器模式与登录状态 +Safe uninstall preserves `.state`; `--purge-state` removes active state and matching backups. -以下流程只适用于用户主动要求浏览器采集，或命令显式传入 `--source browser`。 - -首次运行： - -1. 脚本启动专用浏览器 profile。 -2. 如果没有 X 登录态，打开可见浏览器窗口。 -3. 用户在窗口里正常登录 X。 -4. 登录态保存在 `twitter-digest/.state/chrome-profile`。 - -后续浏览器运行： - -1. 默认 headless 启动浏览器。 -2. 复用 `twitter-digest/.state/chrome-profile`。 -3. 如果登录态有效，不弹浏览器。 -4. 如果登录失效，再打开可见浏览器让用户重新登录。 - -浏览器日报必须识别当前登录账号的 handle。脚本会先从 X 的账号切换器、Profile 导航和账号相关 DOM 自动识别；如果 headless 识别不到，会打开可见浏览器重试。仍识别不到时直接停止，不生成日报。此时让用户确认可见浏览器里登录的是正确账号，或显式运行： - -```bash -python3 twitter-digest/scripts/run_daily_digest.py --handle -``` +## API Data -不会读取用户常用浏览器 profile，也不会要求用户复制 cookie/token。 +The API collector writes the same current-run output shape on every run: -## 默认采集范围 +- `digest-input.json` +- `digest-input.md` +- `digest-context.md` +- `digest-context.json` +- `digest-context-timeline.md` +- `digest-context-mentions.md` +- `digest-context-dm.md` -API 默认页面： +Use the agent file Read tool to read `digest-context.md`. Do not inspect private context with shell commands during normal summarization. -- `home`：首页时间线，用于看热点。 -- `own_profile`：自己的主页，用于看自己最近发帖/互动。 -- `mentions` / `mentions_search`：谁 @ 了当前账号。 +## Time Window -显式浏览器模式页面： - -- `home`：首页时间线，用于看热点。 -- `own_profile`：自己的主页，用于看自己最近发帖/互动。 -- `mentions_search`：搜索 `@当前账号`。 -- `mentions_notifications`：通知里的 @ 提及。 -- `messages`：X Chat / DM。 - -默认不采集关键词。只有显式传 `--keywords` 时才增加关键词搜索页。 - -## 公开页采集逻辑 - -公开页包括： - -- `home` -- `own_profile` -- `mentions_search` -- `mentions_notifications` - -默认参数： +Final digest facts use the user's local timezone and the window: ```text ---scrolls 40 ---min-public-scrolls 5 ---max-public-items 100 ---public-window-hours 24 +[now - 24 hours, now] ``` -流程： - -1. 打开公开页面。 -2. 读取已加载的 `article`。 -3. 滚动页面并继续读取。 -4. 去重。 -5. 最多保留 100 条浏览器公开帖子。API public 默认最多保留 300 条。 -6. 如果已加载帖子的时间戳显示超过 24 小时窗口，提前停止。 - -每条公开帖子会尽量提取： +Items outside the window must not appear as current action items. Items without parseable timestamps are excluded from final facts and reported as data gaps. -- 帖子文本 -- 时间 -- tweet URL -- 作者 URL -- 图片 / 视频缩略图 URL -- 外部链接 -- 卡片 / 引用链接 +## Mentions -公开帖子数量表示“本次浏览器加载到的帖子”，不是完整 X 历史。 +Mention sources: -## DM 采集逻辑 +- Direct mention/notification data when available. +- Recent search for the authenticated handle. -DM 只在显式浏览器模式中读取。API 模式不采集 DM，也不根据 API DM 结果判断“没有私信”。 +Do not treat stale mentions as current. Do not mark an already-replied mention as pending. If reply state cannot be verified from the current API run, label it `回复状态未确认`. -先读取 X Chat 左侧会话列表，并统计今天可见会话： +## DM / X Chat -```text -today visible = last_from_me + waiting_reply -``` - -定义： - -- `last_from_me`：列表最后预览是 `You:` / `You sent` / `你:`，表示最后一条是用户发出的。 -- `waiting_reply`：列表最后预览不是用户发出的，表示最后一条来自对方，等待用户处理。 +API DM coverage is incomplete. If API DM returns zero or fails, report a data gap. Do not claim there are no DMs from API results alone. -规则： +Non-API DM collection is not part of this skill. -- `last_from_me` 只计数，不打开正文。 -- `waiting_reply` 会打开正文。 -- 只有 `waiting_reply` 会进入 DM 摘要判断。 +## Install Prerequisites -## DM 会话完整性 +- `git` +- `python3` 3.9+ -对每个 `waiting_reply` 会话，脚本会尽量完整加载浏览器可读取的会话历史。 - -默认参数： - -```text ---dm-scrolls 200 ---dm-list-scrolls 20 ---dm-max-messages 2000 ---dm-window-hours 0 -``` +Only `git` and Python 3.9+ are required. -含义： +## Troubleshooting -- 最多向上滚动 200 次。 -- 最多保留 2000 条消息气泡。 -- `--dm-list-scrolls 20` 表示先向下扫描左侧会话列表，尽量覆盖今天会话。 -- `--dm-window-hours 0` 表示不按 24 小时截断 DM 历史。 -- 会尽量滚到对话顶部。 -- 如果没有滚到顶部或命中消息上限，会在 `digest-context` 里记录 `dm_thread_incomplete`。 - -DM 每条消息会尽量提取： - -- 发送方向：`me` / `other` -- 时间 -- 文本 -- 链接 -- 图片 / 视频 / 卡片 metadata - -## Passcode 与加密私信 - -如果 X Chat 要求 passcode 或加密恢复： - -1. headless 采集会检测到 passcode 页面。 -2. 脚本自动打开可见浏览器窗口。 -3. 用户自己输入或完成 passcode 设置。 -4. 脚本等待 Messages 页面真正可读。 -5. 继续采集 DM。 -6. 完成后可回到 headless。 - -脚本不会帮用户设置、输入或保存 passcode。 - -## 输出文件 - -每次运行输出在： - -```text -twitter-digest/.state/run/ -``` - -主要文件： - -```text -digest-input.json -digest-input.md -digest-context.json -digest-context.md -``` - -用途： - -- `digest-input.*`：浏览器原始采集结果，只用于 debug。 -- `digest-context.*`：给 Agent 写日报的正式输入。 - -正常总结只读： - -```text -twitter-digest/.state/run/digest-context.md -``` - -不生成长期 `memory.json`，不生成 `daily/` 历史归档。 - -## digest-context 内容 - -`digest-context.md` 包括： - -- Run 信息：日期、生成时间、时区、当前账号。 -- DM Facts：DM 状态、今日可见会话数、最后我发出数量、等我回复数量、捕获消息数量。 -- DM Thread Context：只包含需要回复的会话，带原始 label、URL、加载状态和消息上下文。 -- Public Counts：每个公开页抓到多少帖子。 -- Public Items：公开帖子文本、URL、媒体、外链、卡片。 -- Data Gaps：任何没读到或没读完整的数据缺口。 - -## 日报生成规则 - -Agent 只基于 `digest-context.md` 生成中文日报。 - -如果需要检查本次采集计数、DM 状态或数据缺口，不要临时写 `python3 -c` 或 shell 片段遍历 JSON。使用固定检查脚本： - -```bash -python3 twitter-digest/scripts/inspect_digest.py -``` - -该脚本只输出计数、加载状态和数据缺口，不输出 DM 正文。 - -默认结构： - -```markdown -## X 日报 - YYYY-MM-DD - -**今日总结** - -**该处理** - -**谁 @ 了你** - -**私信（DM）** - -**时间线热点** - -**你的动态** - -**建议回复草稿** - -**数据缺口** -``` - -总结原则： - -- 先判断今天有没有需要处理的风险、机会或回复。 -- @ 提及按重要性分组。 -- DM 只重点总结 `waiting_reply`。 -- `last_from_me` 不当作待处理。 -- 垃圾、钓鱼、低质营销只计数，不展开。 -- 私信内容只做必要摘要，不贴完整隐私历史。 -- 图片/链接信息只作为理解上下文，不能凭空描述图片内容。 -- 不自动发帖、不自动回复、不点赞、不关注。 - -## 用户可以获得的信息 - -用户可以问： - -- 生成今日 X 日报。 -- 今天谁 @ 我了？ -- 哪些 DM 等我回复？ -- 有没有重要私信？ -- 哪些私信是垃圾/钓鱼？ -- 今天时间线有什么热点？ -- 我的账号今天有没有互动？ -- 帮我起草回复，但不要发送。 -- 这次哪些数据没读到？ -- 给我看 `digest-context`。 -- 跳过 DM 生成日报。 - -## 常用命令 - -完整日报： - -```bash -python3 twitter-digest/scripts/run_daily_digest.py -``` - -跳过 DM： - -```bash -python3 twitter-digest/scripts/run_daily_digest.py --source browser --no-dms -``` - -强制显示浏览器： - -```bash -python3 twitter-digest/scripts/run_daily_digest.py --source browser --headed -``` - -清理登录态并重新登录： - -```bash -rm -rf twitter-digest/.state/chrome-profile -``` +- Missing config: API configuration is required; run `run_daily_digest.py --configure-api`. +- Token refresh failure: rerun configuration. +- 401/403/rate limit/API tier issues: report the API data gap. +- Non-API source request: explain that this skill only supports API collection. diff --git a/twitter-digest/SKILL.md b/twitter-digest/SKILL.md index a52da34..292adcb 100644 --- a/twitter-digest/SKILL.md +++ b/twitter-digest/SKILL.md @@ -1,323 +1,209 @@ --- name: twitter-digest -description: Use when the user asks to generate an X/Twitter daily digest or says phrases such as "生成X日报", "生成 x 日报", "X日报", "推特日报", "Twitter digest", or wants an agent to analyze their own X/Twitter mentions, home timeline, reply opportunities, and daily social-media summaries. Default collection uses saved API credentials when configured; otherwise it uses browser collection. Browser can also be requested explicitly. +description: Use when the user asks to generate an X/Twitter daily digest or says phrases such as "生成X日报", "生成 x 日报", "X日报", "推特日报", "Twitter digest", or wants an agent to analyze their own X/Twitter mentions, home timeline, reply opportunities, and daily social-media summaries. This skill is API-only. --- # X/Twitter Digest ## Overview -Use this skill to produce a concise Chinese daily digest from the user's own X/Twitter account. Use `scripts/run_daily_digest.py`, which defaults to `--source auto`: saved API credentials are used when configured; if no API credentials are configured, browser collection is used. Installing this skill does not configure X API and must not open an API configuration Terminal. Once API source is selected by default or explicitly, non-auth API failures should fail or report data gaps instead of falling back to browser. +Use this skill to produce a concise Chinese daily digest from the user's own X/Twitter account. The data source is API-only. -## Source Selection Contract - -Default source is automatic. If saved API credentials exist, automatic runs use API. If no API credentials exist, automatic runs use browser collection. If the latest user message does not explicitly request a source, run exactly: +Normal daily runs use: ```bash RUN_DAILY_DIGEST ``` -Source is not sticky across turns or runs. A previous browser run must never make later normal "日报" requests use browser. A previous API run also does not require a special command; later normal "日报" requests still run `RUN_DAILY_DIGEST`, and the script selects API when saved API credentials exist. For every new digest request, run the collection command again before reading `digest-context.*`; do not reuse a previous browser or API context as if it were fresh. - -Short follow-up replies such as "要", "日报", "生成", "继续", "好", or "可以" inherit only the user's intent to generate a digest. They do not inherit browser source from prior assistant suggestions, data-gap explanations, or API/browser comparisons. Never choose browser because it might be "more complete", because API excludes DMs, because API output says DMs need browser confirmation, or because an earlier assistant message offered browser mode. If API is configured, run API and report the data gap after collection; if API is not configured, the automatic command uses browser. - -When API is configured, only use browser source when the latest user message itself explicitly asks for browser mode, visible private messages / DMs, X Chat, or local browser collection. When API is not configured, the default automatic run uses browser. - -After installation, configuration and daily runs should use the installed skill copy, not a temporary clone/source checkout. Installed locations are `~/.claude/skills/twitter-digest` for Claude Code and `~/.codex/skills/twitter-digest` for Codex. If `run_daily_digest.py` or `configure_api.py` is accidentally run from a source checkout while an installed copy exists, the script automatically re-runs the installed copy so `.state` is written to the installed skill directory. - -Use a stable installed command form for normal chat-triggered runs. This keeps Claude Code's Bash permission prompt stable and lets the user's first "don't ask again" approval apply to future runs from different projects: +`RUN_DAILY_DIGEST` means the installed command for the current agent: - Claude Code: `python3 ~/.claude/skills/twitter-digest/scripts/run_daily_digest.py` - Codex: `python3 ~/.codex/skills/twitter-digest/scripts/run_daily_digest.py` -In command examples below, `RUN_DAILY_DIGEST` means the matching installed command above for the current agent. Do not rely on `python3 twitter-digest/scripts/run_daily_digest.py` after installation unless you are intentionally working inside a source checkout. - -For API maintenance commands, `CONFIGURE_API` means the matching installed configure command: +For API maintenance, `CONFIGURE_API` means: - Claude Code: `python3 ~/.claude/skills/twitter-digest/scripts/configure_api.py` - Codex: `python3 ~/.codex/skills/twitter-digest/scripts/configure_api.py` -For summary writing, read the installed current-run context with the agent's file Read tool, not Bash. The normal context path is: +## Source Contract -- Claude Code: `~/.claude/skills/twitter-digest/.state/run/digest-context.md` -- Codex: `~/.codex/skills/twitter-digest/.state/run/digest-context.md` - -Do not use `cat`, `head`, `tail`, `grep`, `sed`, `python3 -c`, or temporary scripts to read or inspect `digest-context.*` / `digest-input.*` during normal analysis. Those shell reads cause extra Claude Code Bash permission prompts and may expose private DM text in command output. - -Load `references/x-twitter-digest.md` when you need implementation details, API/browser workflow rules, current-run context behavior, or the scoring rubric. - -## Data Collection - -There are three collection scripts: +`twitter-digest` has only one supported collector: ```bash -python3 twitter-digest/scripts/browser_x_digest.py # browser collector -python3 twitter-digest/scripts/api_x_digest.py # API collector -python3 twitter-digest/scripts/run_daily_digest.py # upper wrapper, default auto source +python3 twitter-digest/scripts/api_x_digest.py ``` -For chat usage, run the wrapper: +The wrapper `scripts/run_daily_digest.py` uses API directly. -```bash -RUN_DAILY_DIGEST -``` +Source rules: + +- A normal "生成日报" / "日报" / "要" request always runs `RUN_DAILY_DIGEST`. +- API credentials are required before a digest can be generated. +- If API credentials are already saved, the run uses API and refreshes OAuth tokens when possible. +- If API credentials are missing or expired, the wrapper starts API configuration. The digest must be rerun after configuration succeeds. +- When configuration is opened in Terminal, do not ask the user to paste Client ID, Client Secret, tokens, or app credentials in chat. Tell the user to finish the Terminal flow, then rerun `RUN_DAILY_DIGEST`. +- Do not switch to another data source on API errors, missing DM coverage, rate limits, permission errors, or user requests for "more complete" data. +- If the user asks for a non-API source, visible DMs, X Chat, or cookies, explain that this skill only supports API collection and continue only if they still want an API digest. + +API source isolation is strict: -`run_daily_digest.py` defaults to `--source auto`. Auto uses API only when saved OAuth2 user-context credentials, `X_BEARER_TOKEN`, or `TWITTER_BEARER_TOKEN` are present. If no API configuration exists, auto uses browser collection. If API configuration exists but authentication is broken, it triggers API reconfiguration and then retries API collection once. It must not fall back to browser on API errors. Treat API DM lookup as unavailable; do not use API output to decide whether the user has private messages. A normal "生成日报" / "要" follow-up must run plain `RUN_DAILY_DIGEST`. +- It runs only `api_x_digest.py`. +- It never reads a local profile or cookies. +- It never supplements missing API data with another collector. +- It never asks the user to copy cookies. -Source isolation is strict: +OAuth setup may open the X authorization page. That is only for authorization and is not data collection. -- API source runs only `api_x_digest.py`. It never starts a browser, never opens X pages, never reads the browser profile, and never supplements missing API data with browser data. -- Browser source runs only `browser_x_digest.py`. It uses the dedicated browser profile and does not use saved API tokens or API collector output. -- Default source is auto. It uses saved API credentials when configured. If API is not configured, it uses browser collection. If API is configured, browser is selected only when the latest user message explicitly asks for browser mode or equivalent visible-DM/browser wording. -- Do not infer browser mode from words like "完整", "私信最完整", or an earlier comparison between API and browser. Use browser only when the user's latest instruction explicitly asks for browser mode, visible DMs, X Chat, or local browser collection. -- If API output says DM needs browser confirmation, treat that as a data gap note only. It does not mean browser data was collected. +## Required API Configuration -If the user asks to configure API access, trigger the OAuth/user-token setup from chat: +API access is required. If the user asks to configure API access, run: ```bash RUN_DAILY_DIGEST --configure-api ``` -This is an agent-triggered flow. It supports OAuth2 user authorization: +This is the primary setup flow. It uses OAuth2 PKCE with an X Developer App Client ID and local callback URL. Request scopes: -- OAuth2 path: if the user has an X Developer App OAuth2 `Client ID` / `Client Secret` and local callback URL, run `RUN_DAILY_DIGEST --configure-api`. It goes directly into OAuth2 setup. Request `dm.read tweet.read users.read offline.access`. -- Existing token path: if the user says they already have an OAuth2 user access token, run `RUN_DAILY_DIGEST --configure-api-token`. -- If the agent is not inside an interactive Terminal, do not background `configure_api.py` yourself. Use `run_daily_digest.py --configure-api`; it opens a real Terminal window for secure input and OAuth callback handling. `configure_api.py --oauth` also self-opens Terminal when invoked non-interactively, but the wrapper is the primary path. -- OAuth1 PIN is not a supported normal setup path for this skill because it did not reliably return DM data during validation. Do not guide users to Consumer Key / Consumer Secret / PIN unless they are explicitly debugging legacy API behavior. -- Do not write ad-hoc inline Python or shell snippets to verify tokens. Use the built-in verifier: `CONFIGURE_API --verify`. It calls `/users/me`, backfills `handle` / `user_id`, and does not print the token. - -If a refresh token is saved, API-source runs refresh the access token automatically. Do not ask the user to export environment variables manually. App-only API keys are not enough for user-context home timeline access. - -After API setup succeeds once, future daily digest runs should not ask the user for credentials again. Normal `RUN_DAILY_DIGEST` reads `.state/api_config.json` automatically and uses API by default. OAuth2 credentials are refreshed automatically when a refresh token is saved. If API credentials are missing, normal `RUN_DAILY_DIGEST` uses browser. If API credentials exist but authentication is broken during API collection, `RUN_DAILY_DIGEST` opens configuration and retries once. Use browser source only when the latest user message explicitly wants browser collection. +```text +tweet.read users.read offline.access dm.read +``` -If the user asks to clear API access, run: +If the user already has an OAuth2 user access token, run: ```bash -CONFIGURE_API --clear +RUN_DAILY_DIGEST --configure-api-token ``` -All normal flows should be triggered from chat by the agent: - -- X 日报 / 生成日报 / 要: run `RUN_DAILY_DIGEST`; this uses API when configured, otherwise browser. -- API 日报 / 用 API 生成日报: run `RUN_DAILY_DIGEST --source api`; after this, later plain "日报" requests still run `RUN_DAILY_DIGEST` and use API if credentials are saved. -- 用户已有 token / 输入 X token: run `RUN_DAILY_DIGEST --configure-api-token`. -- 配置 X API / 给 app 授权: run `RUN_DAILY_DIGEST --configure-api`. -- 验证 X API 配置: run `CONFIGURE_API --verify`. -- 检查本次采集计数 / JSON 结构: run `scripts/inspect_digest.py`. -- 清除 X API 配置: run `CONFIGURE_API --clear`. -- 调试浏览器: only when the latest user message explicitly asks to debug browser collection, run `RUN_DAILY_DIGEST --source browser --headed`. +If the agent is not inside an interactive Terminal, use the wrapper. It opens a real Terminal window for secure credential input and OAuth callback handling. After that command reports `api_configuration_required`, stop and tell the user to finish the Terminal flow. When the user says configuration is done, rerun `RUN_DAILY_DIGEST`. -Explicit browser/debug source only: +Verify saved API configuration with: ```bash -X_BEARER_TOKEN=... RUN_DAILY_DIGEST --source api --handle +CONFIGURE_API --verify ``` -API-source runs do not touch the browser profile and are used automatically when API is configured unless the latest user instruction explicitly selects browser collection. If API is not configured, auto uses browser collection. Browser-source runs use a dedicated browser profile at `twitter-digest/.state/chrome-profile`. The user logs in to X once in that browser. Later browser runs default to headless collection and reuse the saved local browser session. If the saved login is unavailable during a browser-source run, the script automatically opens a visible browser window for manual login. The skill has two collector scripts: `scripts/api_x_digest.py` for official API public data, and `scripts/browser_x_digest.py` for browser-visible X Chat / encrypted DM content. API-visible DM events remain TODO-only until X fixes or documents reliable XChat coverage. - -DM reading is enabled by default only for browser-source runs and only reads visible local browser content. API-source runs do not start a browser, even when `--include-dms` is passed. To skip DMs for a browser run: +Clear saved API configuration with: ```bash -RUN_DAILY_DIGEST --source browser --no-dms +CONFIGURE_API --clear ``` +Do not write ad-hoc token verification scripts. Do not ask the user to export bearer tokens manually unless they explicitly want to use environment variables. + +## Data Collection + +For every new digest request, run collection again before reading `digest-context.*`. Do not reuse previous run files as if they were fresh. + Default scope: - Mentions of the authenticated handle. - Home timeline hotspots. - Own profile activity. -- Browser source only: today's visible DM conversations, with only conversations whose latest preview is not from the user opened for content. - Optional keyword searches only when the user explicitly passes `--keywords`. -Public timeline/profile/mentions pages use the same daily-window loading model as DMs. API public collection keeps up to 300 items; browser public collection scrolls each public page up to 40 rounds, keeps up to 100 public items, waits for DOM growth after each scroll, and only allows early stop after at least 5 scroll rounds when loaded post timestamps show content beyond the 24-hour digest window (`--scrolls 40`, `--min-public-scrolls 5`, `--max-public-items 100`, `--public-window-hours 24`). The generated digest context applies a second strict filter: public timeline/profile/mention items must be inside `[now - 24 hours, now]` in the user's current local timezone. Items with missing or unparseable timestamps are excluded from final-summary facts and reported as `time-unverified` data gaps. +DM / X Chat caveat: -Mention handling is strict: +- API DM access is incomplete for many accounts and may return zero events even when the user has X Chat messages. +- Do not say "没有私信" based only on API DM results. +- If API DM results are unavailable, report a data gap. Do not suggest another collector as a fallback inside this skill. -- Both mention sources must be considered when available: direct mention/notification collection and handle search (`@handle` / equivalent search page or API recent search). Do not conclude "no current mentions" from only one source unless the other source was attempted and failed or is unavailable. -- Do not include mentions older than the local 24-hour window in `该处理`, `谁 @ 了你`, or reply drafts. -- Do not present an already-replied mention as needing reply. Use own-profile/API own posts, browser-visible reply context, or context metadata when available to decide whether the authenticated account already replied after the mention timestamp. -- When `digest-context.md` marks a mention with `reply_state=already_replied` or `action_state=handled`, it must not appear as a pending reply opportunity. It may be omitted or summarized as already handled. -- When `digest-context.md` marks `reply_state=reply_unverified`, do not say the user "needs to reply"; write `回复状态未确认` and suggest review only if the content is important. -- If reply status cannot be verified from the current run's data, label the item as `回复状态未确认` instead of claiming the user still needs to reply. +Time window rules: -Read the installed `digest-context.md` with the file Read tool when writing the Chinese digest. Its `Final Summary Facts` section is the content source for the final summary. If more focused context is needed, use the file Read tool on the split current-run files: `digest-context-timeline.md` for home/profile/timeline items, `digest-context-mentions.md` for @ mentions, and `digest-context-dm.md` for visible DM conversations. Use `digest-input.md` only when debugging collection issues, not during normal summarization. Do not add content from older runs. Do not write ad-hoc `python3 -c`, shell, `cat`, `head`, `tail`, `grep`, or temporary scripts to inspect context structure during normal summarization. If counts or non-content structure must be checked, run the built-in `scripts/inspect_digest.py`, which does not print DM bodies. +- Final summary facts must use only items inside `[now - 24 hours, now]` in the user's current local timezone. +- Items with missing or unparseable timestamps are excluded from final-summary facts and reported as data gaps. +- Mentions older than the 24-hour window must not appear as pending reply opportunities. -If `digest-context.md` or `digest-context-mentions.md` shows missing mention sources, collection errors for `mentions_search` / `mentions_notifications`, or only stale mention data, report that as a data gap. Do not turn stale mentions into action items. +Mention handling: -## Install - -From the repository `skills/` directory: - -```bash -python3 twitter-digest/scripts/install.py -``` +- Consider both direct mention/notification data and recent search results when available. +- Do not present an already-replied mention as needing reply. +- If reply status cannot be verified from current API data, label it `回复状态未确认` instead of claiming the user must reply. -For one-line installs through `twitter-digest/install.sh`, Codex, Claude Code, and other non-interactive macOS agents open a real Terminal window and re-run the full installation there. This keeps `git clone`, Python/browser prerequisite checks, and skill-directory writes out of the agent permission sandbox. To force direct in-process installation from an already interactive terminal or CI, set `TWITTER_DIGEST_OPEN_TERMINAL=0`. +## Writing The Digest -Default install targets the current agent client: Codex installs to `~/.codex/skills/twitter-digest`, Claude Code installs to `~/.claude/skills/twitter-digest`. Use `--client codex`, `--client claude`, or `--skills-dir` to override. Local development can use `--symlink`. +After collection, read the installed current-run context with the agent's file Read tool, not shell text commands. -For Claude Code, the skill cannot silently grant itself Bash permission or file access outside the project. On first use, approve the visible `run_daily_digest.py` command and choose "don't ask again" if appropriate. For a global opt-in during install, run: +Normal context paths: -```bash -python3 twitter-digest/scripts/install.py --client claude --allow-claude-commands --allow-claude-state-read -``` +- Claude Code: `~/.claude/skills/twitter-digest/.state/run/digest-context.md` +- Codex: `~/.codex/skills/twitter-digest/.state/run/digest-context.md` -This explicitly adds one Claude Code Bash allow rule for `python3 ~/.claude/skills/twitter-digest/scripts/run_daily_digest.py:*` and adds `~/.claude/skills/twitter-digest/.state` to Claude Code `additionalDirectories` so the Read tool can read `digest-context.md` without a separate file-access prompt. It does not bypass permissions globally and does not grant arbitrary Bash access. +Focused slices: -The installer checks for Python 3.9+ and a supported Chromium browser before installing. Supported browsers are Google Chrome, Chromium, Microsoft Edge, and Brave. If a prerequisite is missing, install it and rerun the same install command; the installer does not install or upgrade Python or browsers. If the browser will be installed later, use `--skip-browser-check`. +- `digest-context-timeline.md` +- `digest-context-mentions.md` +- `digest-context-dm.md` -Reinstalling is upgrading. The installer moves old `twitter-briefing`, `twitter-briefing.bak`, or existing `twitter-digest` installs into the selected skills directory's `.backups/` folder and disables their `SKILL.md` files so the current agent does not load duplicate old skills. It preserves `.state` from an existing installed `twitter-digest` copy during reinstall, but does not copy `.state` from the development checkout. +Use `digest-context.md` and its `Final Summary Facts` as the content source for the Chinese digest. Use `digest-input.*` only for debugging collection issues. -To uninstall, run the installed script: +Do not use `cat`, `head`, `tail`, `grep`, `sed`, `python3 -c`, or temporary scripts to inspect private context during normal summarization. If counts or structure must be checked, run: ```bash -~/.codex/skills/twitter-digest/uninstall.sh --client codex -~/.claude/skills/twitter-digest/uninstall.sh --client claude +python3 ~/.claude/skills/twitter-digest/scripts/inspect_digest.py +python3 ~/.codex/skills/twitter-digest/scripts/inspect_digest.py ``` -Default uninstall moves the installed skill to `.backups/` and preserves `.state`. Use `--purge-state` only when the user explicitly wants API config, browser profile, current-run files, and matching `.backups` entries removed permanently. - -Claude Code or other agents can use the installed skill by running the same installed commands. - -## Run Outputs - -`scripts/run_daily_digest.py` does not write long-term memory. Each run writes only current-run files: +Adjust the path to the current agent. -- `twitter-digest/.state/config.json`: account defaults and preferences. -- `/.state/run/digest-context.md`: the normal top-level input for AI daily-summary writing. -- `/.state/run/digest-context.json`: machine-readable version of the same normalized facts. -- `/.state/run/digest-context-timeline.md`: focused current-run timeline/profile context for file Read. -- `/.state/run/digest-context-mentions.md`: focused current-run @ mention context for file Read. -- `/.state/run/digest-context-dm.md`: focused current-run DM context for file Read. -- `/.state/run/digest-input.md`: raw collector capture for debugging only. -- `/.state/run/digest-input.json`: raw machine-readable collector capture for debugging only. +Digest format: -No `memory.json` or `daily/` archive is produced. Raw DM text or DM excerpts may exist only in the current run's private `twitter-digest/.state/run/digest-input.*` and `digest-context.*` files for immediate summarization/debugging. The run directory is created with owner-only permissions where supported. Run dates use the user's local timezone. +- 今日总结. +- 该处理. +- 谁 @ 了你. +- 时间线热点. +- 你的动态. +- 数据缺口. +- 建议回复草稿. -## Workflow +Never automatically post, reply, like, follow, block, open suspicious links, accept requests, or send DMs. Replies are drafts only unless the user explicitly asks to send after reviewing. -### 1. Collect - -When the user asks for an X daily digest or X 日报, run: - -```bash -RUN_DAILY_DIGEST -``` - -When the user's latest message is a short confirmation such as "要", "日报", "生成", "继续", "好", or "可以", run the same default command above. Do not treat an earlier assistant suggestion like "browser is more complete" as source selection. - -Do not treat an earlier browser-source result, a visible DM result, or a previous assistant mention of "浏览器源" as a user request for browser on the next digest. The latest user message must explicitly ask for browser/DM/X Chat/local browser before adding `--source browser`. - -If they ask to skip DMs during a normal API daily digest, still run the normal API path because API mode does not collect DMs: - -```bash -RUN_DAILY_DIGEST -``` +## Install -If they explicitly ask for browser mode but want to skip DMs: +From a checked-out repository: ```bash -RUN_DAILY_DIGEST --source browser --no-dms +python3 twitter-digest/scripts/install.py ``` -If the authenticated handle is not detected or the user corrects it: +The installer checks Python 3.9+ and installs the skill into the target agent skill directory. -```bash -RUN_DAILY_DIGEST --handle --account-name "<显示名>" --save-default -``` +Default install targets the current agent client: -Browser-source daily digests must identify the authenticated handle before collecting public pages. If automatic handle detection fails, the collector opens a visible browser window and retries. If the handle still cannot be identified, the run must stop and the agent must not generate a daily digest from partial browser data; ask the user to rerun with `--handle ` or confirm the correct account in the visible X window. +- Codex: `~/.codex/skills/twitter-digest` +- Claude Code: `~/.claude/skills/twitter-digest` -For debugging or manual inspection: +Use `--client codex`, `--client claude`, or `--skills-dir` to override. Local development can use `--symlink`. -```bash -RUN_DAILY_DIGEST --source browser --headed -``` +Reinstalling is upgrading. The installer moves the existing installed skill to `.backups/`, disables backup `SKILL.md` files so agents do not load old duplicate skills, and preserves the active installed `.state` directory. -For unattended scheduled runs that should not block on passcode recovery: +Uninstall: ```bash -RUN_DAILY_DIGEST --non-interactive +~/.codex/skills/twitter-digest/uninstall.sh --client codex +~/.claude/skills/twitter-digest/uninstall.sh --client claude ``` -Do not ask the user to copy cookies or configure another service. If the script opens a visible browser window, tell the user to log in or resolve the visible X challenge there. - -### 2. Protect Privacy - -Treat browser sessions, cookies observed internally by the script, DMs, phone numbers, emails, private handles, and screenshots as sensitive. Do not post, reply, like, follow, block, open suspicious links, accept DM requests, or send DMs unless the user explicitly asks after reviewing a draft. +Use `--purge-state` only when the user explicitly wants API config and current-run files removed permanently. -Browser DM collection only reads message content visible in the logged-in local browser. If X Chat shows a passcode setup, passcode entry, or end-to-end-encryption recovery screen during headless collection, the script should automatically reopen X Messages in a visible browser window, wait for the user to complete it, then retry DM collection. In `--non-interactive` mode, record the DM data gap and continue without blocking. Do not choose, enter, or store a passcode for the user. - -### 3. Analyze - -Group mentions by reason to care: - -- Direct asks: questions, requests, invitations, support requests. -- Influence: high-signal accounts or high engagement. -- Risk: complaints, misinformation, scams, impersonation, security-sensitive posts. -- Opportunity: partnership, hiring, customer lead, investor/media attention, community praise. -- Noise: spam, generic tags, low-context reposts. - -Before putting a mention into `✅ 该处理` or `🔴 值得回 / 需要处理`, confirm it is within the local 24-hour window and not already replied to. If it is already replied to, either omit it from action items or mark it as already handled. If the current run cannot verify reply status, mark `回复状态未确认` and avoid phrasing it as definitely needing a reply. For mentions, treat `reply_state` / `action_state` in `digest-context.md` as authoritative over the text of the mention. - -Classify DMs as: - -- `urgent`: time-sensitive, business-critical, safety/security, money, reputation, or deadline. -- `important`: meaningful relationship, opportunity, unresolved issue, or action needed. -- `routine`: informational, friendly, low-risk, or easy acknowledgement. -- `ignore`: spam, phishing, harassment, or irrelevant bulk outreach. - -Always report DM conversation counts and message counts separately when available. Conversation counts come from today's X Chat list items only: today visible conversations, conversations whose latest preview is from the user (`You:` / `You sent` / `你:`), and conversations waiting for the user's reply. The browser collector first scans the DM conversation list downward up to 20 rounds (`--dm-list-scrolls 20`) so it is not limited to the first visible screen; older conversations found in the list are ignored for the daily count. Message counts come only from opened waiting-reply conversations and represent captured message bubbles. The collector now tries to load each opened waiting-reply conversation completely by scrolling upward until the browser reaches the thread top; defaults are a 200-scroll safety limit and up to 2000 kept message bubbles (`--dm-scrolls 200`, `--dm-max-messages 2000`, `--dm-window-hours 0`). If the conversation list does not finish scanning, the digest context records `dm_list_incomplete`; if a thread does not reach the top or hits the message cap, it records `dm_thread_incomplete`. Only summarize DM bodies from `dm_status: captured_unreplied_threads`. If today's visible conversations all have latest previews from the user, report `no_unreplied_threads` as “今天可见私信会话最后一条都是我发出的，无需处理”, not “没有私信”. If `no_today_threads` appears, say there were visible older conversations but no today conversations. If `visible_threads_unopened` appears, say the conversation list was visible but waiting-reply message bodies were not opened. - -If X Messages shows a skeleton/loading conversation list or `Start Conversation` while the left list is still placeholder content, do not treat it as an empty inbox. The browser collector detects this state, reloads `/messages` up to 3 times, and records `dm_page_loading_timeout` if the list never becomes readable. - -For waiting-reply DMs, still summarize selectively. Count all waiting-reply conversations, but only include DMs with action value, relationship value, risk, money/security implications, or clear user relevance in the digest. Obvious spam, phishing, generic promotion, low-context links, or repeated junk should be counted and classified as ignore/noise without copying the content into the main summary. - -For DM sender attribution, use the thread `participant` / `会话对象` and message bubble direction. Do not treat authors inside quoted posts, repost cards, link previews, or embedded tweet text as the DM sender. If a DM contains a shared post by `Marco` inside a conversation with `@jerry`, the DM is from the conversation participant, not from `Marco`. - -When `digest-context.md` includes `### DM Thread Context`, use that section to understand the recent conversation history for waiting-reply DMs. It may include up to 2000 loaded message bubbles per summarized thread, plus raw thread label, URL, and load metadata, so the model can understand complex context before deciding whether and how to mention the DM. Keep the final digest concise; do not paste the full DM history into the report. - -Use media and link metadata when present. Public items and DM message context may include `media`, `link`, and `card` lines with image/video URLs, alt text, shared-post links, and external links. Treat these as context for understanding the item, but do not open suspicious links or overstate image contents beyond the available alt/text/URL signals. - -For private messages, summarize minimally. Quote only the short phrase needed to justify classification, and omit sensitive personal data unless the user specifically needs it. - -### 4. Produce The Daily Summary - -除非用户另有要求，最终输出必须用中文，并使用这个结构： - -```markdown -## 🐦 X 日报 - YYYY-MM-DD - -**📌 今日总结** -用 2-4 句中文先给判断：今天最值得知道的事、是否有需要处理的风险/机会、建议用户怎么做。 - -**✅ 该处理** -| 优先级 | 来自 | 为什么重要 | 建议动作 | -|---|---|---|---| - -**◆ 谁 @ 了你** -- 🔴 值得回 / 需要处理 -- 🟡 一般互动 -- ⚪ 噪音折叠统计 -- 只包含本地 24 小时窗口内的 mentions；已回复过的 mentions 不再作为待回复提醒 - -**◆ 私信（DM）** -- 会话统计：今日可见会话 N 个，最后我发出 N 个，等我回复 N 个 -- 消息统计：已打开等我回复会话中捕获消息 N 条 -- 仅挑重点总结等我回复的私信；垃圾、钓鱼、低质营销只计数并归为忽略 -- 🔴 重要 / 🟡 一般 / ⚪ 忽略 +## Run Outputs -**◆ 时间线热点** -1. 热点：一句话解释 + 代表观点 + 和用户的关系 +Each run writes only current-run files: -**◆ 你的动态** -- 近 24h 自己发帖/互动概况 +- `/.state/config.json` +- `/.state/api_config.json` +- `/.state/run/digest-context.md` +- `/.state/run/digest-context.json` +- `/.state/run/digest-context-timeline.md` +- `/.state/run/digest-context-mentions.md` +- `/.state/run/digest-context-dm.md` +- `/.state/run/digest-input.md` +- `/.state/run/digest-input.json` -**✍️ 建议回复草稿** -- 只给草稿，不自动发送。 +No long-term memory or daily archive is produced. Run dates use the user's local timezone. -**⚠️ 数据缺口** -- 哪些页面没读到、DM 是否不可见、是否只扫描了前 N 条。 -``` +## Troubleshooting -保持简洁、有判断、可执行。价值在总结和行动建议，不是把页面内容全文搬运。 +- Missing API config: run `RUN_DAILY_DIGEST --configure-api`. +- Token refresh failed: the wrapper opens API configuration and retries once. +- API permission/tier/rate-limit errors: report the data gap or failure. +- Non-API source requests: unsupported in this skill. diff --git a/twitter-digest/agents/openai.yaml b/twitter-digest/agents/openai.yaml index bad2c2b..37d8db9 100644 --- a/twitter-digest/agents/openai.yaml +++ b/twitter-digest/agents/openai.yaml @@ -1,4 +1,4 @@ interface: display_name: "Twitter Digest" short_description: "Generate X/Twitter daily digests from own account" - default_prompt: "Use $twitter-digest to generate my X daily digest via the default automatic flow: use saved API credentials when configured, otherwise use browser collection. Include mentions, timeline hotspots, own-account activity, and action items. For ordinary follow-ups like 要, 日报, 生成, or 继续, run the default command again; do not reuse a prior browser/API context and do not inherit the previous run's source. If API is configured, do not use browser/DM collection unless my latest message explicitly asks for browser mode, visible DMs, X Chat, or local browser collection." + default_prompt: "Use $twitter-digest to generate my X daily digest through the API-only flow. API credentials are required; if they are missing or expired, run the built-in API configuration flow first. If a Terminal opens for configuration, do not ask for credentials in chat; tell me to finish the Terminal flow, then rerun the digest command after I say configuration is done. Include mentions, timeline hotspots, own-account activity, action items, and clearly marked data gaps. For ordinary follow-ups like 要, 日报, 生成, or 继续, run the default command again and do not reuse prior context." diff --git a/twitter-digest/install.sh b/twitter-digest/install.sh index f9acf99..cc4dce6 100755 --- a/twitter-digest/install.sh +++ b/twitter-digest/install.sh @@ -1,12 +1,11 @@ #!/bin/sh set -eu -TAG="${TWITTER_DIGEST_INSTALL_TAG:-v1.5.12-beta.12}" +TAG="${TWITTER_DIGEST_INSTALL_TAG:-v1.5.13}" REPO="${TWITTER_DIGEST_INSTALL_REPO:-https://github.com/BofAI/skills.git}" CLIENT="${TWITTER_DIGEST_INSTALL_CLIENT:-auto}" ALLOW_CLAUDE_COMMANDS="${TWITTER_DIGEST_ALLOW_CLAUDE_COMMANDS:-0}" ALLOW_CLAUDE_STATE_READ="${TWITTER_DIGEST_ALLOW_CLAUDE_STATE_READ:-0}" -SKIP_BROWSER_CHECK="${TWITTER_DIGEST_SKIP_BROWSER_CHECK:-0}" OPEN_TERMINAL="${TWITTER_DIGEST_OPEN_TERMINAL:-auto}" info() { @@ -78,7 +77,7 @@ open_self_in_terminal_and_exit() { for arg in "$@"; do args_text="${args_text} $(shell_quote "$arg")" done - command_text="cd ~ && TMPDIR=\"\$(mktemp -d)\" && INSTALL_SH=\"\$TMPDIR/twitter-digest-install.sh\" && curl -fsSL $(shell_quote "$installer_url") -o \"\$INSTALL_SH\" && chmod 700 \"\$INSTALL_SH\" && env TWITTER_DIGEST_TERMINAL_CHILD=1 TWITTER_DIGEST_OPEN_TERMINAL=0 TWITTER_DIGEST_INSTALL_TAG=$(shell_quote "$TAG") TWITTER_DIGEST_INSTALL_REPO=$(shell_quote "$REPO") TWITTER_DIGEST_INSTALL_CLIENT=$(shell_quote "$CLIENT") TWITTER_DIGEST_ALLOW_CLAUDE_COMMANDS=$(shell_quote "$ALLOW_CLAUDE_COMMANDS") TWITTER_DIGEST_ALLOW_CLAUDE_STATE_READ=$(shell_quote "$ALLOW_CLAUDE_STATE_READ") TWITTER_DIGEST_SKIP_BROWSER_CHECK=$(shell_quote "$SKIP_BROWSER_CHECK") /bin/sh \"\$INSTALL_SH\"${args_text}; printf '\\nPress Enter to close this window...'; IFS= read -r _" + command_text="cd ~ && TMPDIR=\"\$(mktemp -d)\" && INSTALL_SH=\"\$TMPDIR/twitter-digest-install.sh\" && curl -fsSL $(shell_quote "$installer_url") -o \"\$INSTALL_SH\" && chmod 700 \"\$INSTALL_SH\" && env TWITTER_DIGEST_TERMINAL_CHILD=1 TWITTER_DIGEST_OPEN_TERMINAL=0 TWITTER_DIGEST_INSTALL_TAG=$(shell_quote "$TAG") TWITTER_DIGEST_INSTALL_REPO=$(shell_quote "$REPO") TWITTER_DIGEST_INSTALL_CLIENT=$(shell_quote "$CLIENT") TWITTER_DIGEST_ALLOW_CLAUDE_COMMANDS=$(shell_quote "$ALLOW_CLAUDE_COMMANDS") TWITTER_DIGEST_ALLOW_CLAUDE_STATE_READ=$(shell_quote "$ALLOW_CLAUDE_STATE_READ") /bin/sh \"\$INSTALL_SH\"${args_text}; printf '\\nPress Enter to close this window...'; IFS= read -r _" osascript >/dev/null < -``` - -Outputs: - -- `twitter-digest/.state/run/digest-input.json`: raw collector capture. -- `twitter-digest/.state/run/digest-input.md`: readable raw collector capture. -- `twitter-digest/.state/run/digest-context.json`: normalized current-run facts. -- `twitter-digest/.state/run/digest-context.md`: primary final summary input. Read this first. - -Normal summary writing must not use ad-hoc `python3 -c`, shell, or temporary scripts to inspect JSON structure. If the agent needs a low-sensitive count/status view, use: - -```bash -python3 twitter-digest/scripts/inspect_digest.py -``` - -This prints counts, load status, and data gaps without printing DM message bodies. Use it instead of writing custom inspection snippets. +## Source Selection -## Browser Collection Rules +The wrapper uses API directly. -- Use the dedicated profile, not the user's normal browser profile. -- Do not ask the user to copy cookies or tokens. -- If X shows a login page, wait for the user to log in manually. -- If X shows CAPTCHA or account challenge, stop and ask the user to resolve it in the browser. -- Browser daily digest collection must identify the authenticated handle before collecting `own_profile`, `mentions_search`, and `mentions_notifications`. If headless detection fails, retry in a visible browser. If detection still fails, stop the run; do not let the agent summarize partial browser data as a daily digest. -- Do not post, like, follow, accept DM requests, open suspicious links, or reply. -- Keep scrolling bounded by the digest goal. API public collection defaults to `--max-public-items 300`; browser public collection defaults to `--scrolls 40`, `--max-public-items 100`, and `--public-window-hours 24`, stopping early when loaded post timestamps are clearly outside the daily window. -- Read only DM content that is visible in the local logged-in browser. Use `--no-dms` when the user does not want DMs processed. -- If X Chat shows passcode setup, passcode entry, or encryption-key recovery, automatically reopen X Messages in a visible browser window, wait for the user to complete it, then retry DM collection. In `--non-interactive` mode, skip DM recovery and record a data gap. The script must not choose, enter, or store the passcode. +API configuration is required. If API config is missing or broken, the wrapper opens API configuration and the current command stops with `api_configuration_required`. After the user finishes the Terminal flow, rerun the digest command. It never switches to another collector. -## Pages Collected +## Data Rules -Default browser pages: +- Use only the current run's context files. +- Filter final facts to the user's local 24-hour window. +- Do not include stale mentions as pending work. +- Do not show already-replied mentions as needing reply. +- If reply status is unclear, mark it `回复状态未确认`. -- `home`: home timeline for hotspot detection. -- `own_profile`: the authenticated account profile. -- `mentions_search`: live search for `@handle`. -- `mentions_notifications`: notifications mentions page. -- `messages`: X Messages. +## DM Rules -Optional pages: +API DM coverage may be incomplete. Treat failed or zero DM results as a data gap, not proof that there are no DMs. -- `keyword_N`: explicit search queries, only when `--keywords` is provided. +Non-API X Chat / encrypted DM collection is not part of this skill. -## Run Outputs +## Summary Rubric -Current-run files: +Produce a concise Chinese daily digest: -- `twitter-digest/.state/config.json`: account defaults and preferences. -- `twitter-digest/.state/run/digest-context.md`: the only normal input for AI daily-summary writing. -- `twitter-digest/.state/run/digest-context.json`: machine-readable current-run facts. -- `twitter-digest/.state/run/digest-input.*`: raw collector capture for debugging only. +- 今日总结. +- 该处理. +- 谁 @ 了你. +- 时间线热点. +- 你的动态. +- 数据缺口. +- 建议回复草稿. -Privacy rule: do not write long-term memory or daily archives. Raw DM text may exist only in the current run's private `twitter-digest/.state/run/digest-input.*` and `digest-context.*` files for immediate summarization/debugging. Use only `digest-context.md` for normal final summaries. - -Date rule: - -- Run dates use the user's local timezone, not UTC. -- Public timeline/profile/mentions facts use a strict `[now - 24 hours, now]` window in the user's current local timezone. -- Public items with missing or unparseable timestamps are excluded from final-summary facts and must be reported as `time-unverified` data gaps. - -## Hotspot Detection - -Cluster home timeline posts by repeated topics, hashtags, URLs, named entities, accounts, and semantic similarity. Optional keyword-search posts may be included only when the user explicitly passes `--keywords`. - -Public timeline, profile, and mentions collection follows the same bounded-window model as DM collection: load enough context for a 24-hour digest, cap the amount passed to the model, and stop early when timestamps show older content. Treat public item counts as browser-loaded items in this run, not exhaustive X history. The generated digest context applies a second strict local-time 24-hour filter before writing final-summary public facts; do not reintroduce older raw-capture items while summarizing. - -Mention handling is strict: - -- Direct mention/notification collection and handle search (`@handle` / equivalent search page or API recent search) are complementary. Use both sources when available before concluding there are no current mentions or no reply opportunities. -- If one mention source is missing, failed, unavailable, or only returned stale data, report the source status as a data gap and do not turn stale mentions into action items. -- Mentions older than the local 24-hour window must not appear in action items, reply drafts, or the `谁 @ 了你` section. -- Before labeling a mention as needing reply, verify from current-run data whether the authenticated account already replied after the mention timestamp. -- Already-replied mentions should be omitted from action items or marked as already handled. -- If `digest-context.md` includes `reply_state=already_replied` or `action_state=handled`, the mention is not a pending reply. Do not include it in `✅ 该处理` as a reply task. -- If `digest-context.md` includes `reply_state=reply_unverified`, report `回复状态未确认`; do not claim the user definitely needs to reply. -- If reply status cannot be verified, label the mention as `回复状态未确认` instead of claiming the user still needs to reply. - -A hotspot needs at least one of: - -- Multiple independent posts in the digest window. -- One high-signal post with unusually strong engagement. -- A topic that directly affects the user's projects, brand, customers, portfolio, or community. - -For each hotspot, include: - -- Topic name. -- Why it is surfacing now. -- Evidence from representative posts when available. -- User relevance. -- Suggested action: monitor, reply, quote, DM, ignore, or investigate. - -## Importance Scoring - -Score each item from 0 to 100. Use the score to rank, not as false precision. - -Suggested weights: - -- Direct action requested: +25 -- Time sensitivity or deadline: +20 -- Reputation, security, legal, finance, or customer risk: +20 -- High-relevance relationship or business opportunity: +15 -- High-signal author or engagement: +10 -- Repeated topic across independent sources: +10 -- Spam, vague tag, automated promo, or low-context mention: -20 -- Harassment, phishing, or obvious scam: classify separately as risk/ignore. - -Classification bands: - -- 80-100: urgent -- 60-79: important -- 35-59: routine -- 0-34: ignore/noise - -## DM Handling - -DMs are private. Count today's visible conversations, but summarize only conversations whose latest preview is not from the user and has real action value. Conversations whose latest preview is `You:` / `You sent` / `你:` should not be presented as action items. - -Count fields: - -- `dm_visible_thread_count`: today's visible conversation targets found in the DM list. -- `dm_replied_thread_count`: today's visible conversations whose latest list preview appears to be from the user, based on labels such as `You:` / `You sent` / `你:`. -- `dm_unreplied_thread_count`: today's visible conversations waiting for the user's reply, meaning the latest list preview is not from the user. -- `dm_captured_message_count`: captured message bubbles from opened waiting-reply conversations only. -- Per-thread `message_count`: captured message bubbles in that opened conversation. - -Do not compare conversation counts and message counts as if they were the same unit. A user can have 5 today visible conversations, 2 waiting-reply conversations, and 4 captured message bubbles across those opened conversations. - -The collector opens waiting-reply conversations and scrolls upward before extracting message bubbles. Defaults are `--dm-scrolls 200`, `--dm-max-messages 2000`, and `--dm-window-hours 0`, so it tries to load the full browser-available thread instead of stopping at the 24-hour digest window. If X does not reach the thread top or the message cap is hit, the context records `dm_thread_incomplete`. - -Waiting-reply does not automatically mean important. Count every waiting-reply conversation, but summarize only messages that are actionable, relationship-relevant, risky, money/security-sensitive, or clearly useful. Spam, phishing, generic promotion, low-context links, and repeated junk should be classified as ignore/noise and not copied into the main narrative. - -For waiting-reply DMs that should be summarized, `digest-context.md` includes a `### DM Thread Context` section with loaded message history, raw thread label, URL, and load metadata. It can include up to 2000 loaded message bubbles per thread. Use it to understand the conversation, but do not copy full private history into the final digest. Keep final DM summaries short and action-oriented. - -Media/link context: - -- Public items may include `media`, `link`, and `card` lines with image/video URLs, thumbnail URLs, alt text, shared-post links, or external links. -- DM message context may include per-message `link:` and `media:` lines. -- Use these fields to understand whether a post or DM includes a shared post, image, video, or external reference. -- Do not open suspicious links. Do not claim visual details that are not present in the text, alt text, thumbnail URL, or surrounding context. - -Status rules: - -- `captured_unreplied_threads`: summarize the captured waiting-reply DM bodies selectively and classify importance. -- `no_unreplied_threads`: report the counts and say today's visible DM conversations all have latest previews from the user; do not say there are no DMs. -- `no_today_threads`: say older conversations were visible, but no today conversations were found. -- `visible_threads_unopened`: say the conversation list was visible but waiting-reply message bodies could not be opened; do not infer content. -- `blocked_by_x_chat_passcode`: say DM content is unavailable until the user completes X Chat passcode recovery. - -Sender attribution: - -- Use the thread `participant` / `会话对象` field and message bubble direction as the DM sender signal. -- Do not treat authors inside quoted posts, repost cards, link previews, or embedded tweet text as DM senders. -- If a conversation with `@jerry` contains a shared post authored by `Marco`, attribute the DM conversation to `@jerry`; mention `Marco` only as the quoted/shared post author if relevant. - -Red flags: - -- Credential requests, wallet/private-key requests, suspicious links, payment pressure, impersonation. -- Legal threats, press inquiries, customer escalations, partner deadlines. -- People asking for personal data about someone else. - -Reply drafting rules: - -- Draft replies separately from the summary. -- Keep tone aligned with the user's recent posts if available. -- Do not promise commitments, prices, dates, investments, legal positions, or confidential details unless the user supplied them. -- Ask for approval before sending any reply. - -## 中文每日 Prompt - -```text -使用 $twitter-digest 按默认自动流程读取我最近 24 小时的 X/Twitter 动态，生成中文日报：已配置 API 时用 API，未配置 API 时用浏览器。重点总结谁 @ 了我、时间线热点、需要处理的公开互动、我的账号动态。已配置 API 时，除非我明确要求浏览器模式或可见私信，不要使用浏览器/DM 采集。先给今日总结和行动建议，再给明细。回复只生成草稿，不要自动发送。读不到的数据要明确标注。 -``` +Do not automatically send replies or take account actions. diff --git a/twitter-digest/scripts/api_x_digest.py b/twitter-digest/scripts/api_x_digest.py index 09f60d0..6bb6333 100644 --- a/twitter-digest/scripts/api_x_digest.py +++ b/twitter-digest/scripts/api_x_digest.py @@ -437,16 +437,16 @@ def collect_api(args: argparse.Namespace) -> dict[str, Any]: dm_summary = normalize_dm_events(dm_window_raw, dm_includes, user_id, max_threads=50) if dm_errors: dm_status = "api_dm_todo" - dm_note = "TODO: X API DM lookup failed. Do not treat this as an empty inbox; use browser DM collection if DM coverage is required." - dm_todo_detail = "API DM request failed; check OAuth2 dm.read scope, Project/API access, rate limits, or use browser DM collection." + dm_note = "TODO: X API DM lookup failed. Do not treat this as an empty inbox." + dm_todo_detail = "API DM request failed; check OAuth2 dm.read scope, Project/API access, or rate limits." elif dm_summary["visible_count"] == 0: dm_status = "api_dm_todo" - dm_note = "TODO: X API returned 0 DM events in the digest window. XChat/encrypted messages may not be exposed through DM Events API; use browser DM collection if DM coverage is required." - dm_todo_detail = "API DM returned 0 events; verify with browser X Chat before saying there are no DMs." + dm_note = "TODO: X API returned 0 DM events in the digest window. XChat/encrypted messages may not be exposed through DM Events API; do not claim there are no DMs." + dm_todo_detail = "API DM returned 0 events; do not say there are no DMs." elif dm_summary["unreplied_count"] == 0: dm_status = "api_dm_todo" - dm_note = "TODO: X API returned DM events but no waiting-reply conversation. Browser X Chat should be checked before concluding no DM action is needed." - dm_todo_detail = "API DM has events but no waiting-reply thread; browser DM collection remains authoritative for XChat/encrypted conversations." + dm_note = "TODO: X API returned DM events but no waiting-reply conversation. Do not conclude no DM action is needed from API alone." + dm_todo_detail = "API DM has events but no waiting-reply thread; XChat/encrypted conversations may be missing from API." else: dm_status = "captured_unreplied_threads" dm_note = "X API DM lookup captured recent waiting-reply conversations. API events are limited by X API retention, permissions, and rate limits." diff --git a/twitter-digest/scripts/browser_lifecycle.py b/twitter-digest/scripts/browser_lifecycle.py deleted file mode 100644 index 77a02bc..0000000 --- a/twitter-digest/scripts/browser_lifecycle.py +++ /dev/null @@ -1,129 +0,0 @@ -"""Browser process and login lifecycle helpers for X collection.""" - -from __future__ import annotations - -import shutil -import socket -import subprocess -import time -from pathlib import Path - -from cdp_client import cdp_get_all_cookies, wait_for_cdp, wait_for_cdp_page_ws - - -def find_chrome() -> str: - candidates = [ - "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome", - "/Applications/Chromium.app/Contents/MacOS/Chromium", - "/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge", - "/Applications/Brave Browser.app/Contents/MacOS/Brave Browser", - "google-chrome", - "google-chrome-stable", - "chromium", - "chromium-browser", - "microsoft-edge", - "brave-browser", - ] - for candidate in candidates: - path = Path(candidate) - if path.is_absolute() and path.exists(): - return candidate - resolved = shutil.which(candidate) - if resolved: - return resolved - raise SystemExit("No supported Chromium browser found. Install Chrome, Chromium, Edge, or Brave.") - -def get_free_port() -> int: - with socket.socket() as sock: - sock.bind(("127.0.0.1", 0)) - return int(sock.getsockname()[1]) - -def launch_browser(profile_dir: Path, start_url: str, headless: bool) -> tuple[subprocess.Popen[bytes], int]: - profile_dir.mkdir(parents=True, exist_ok=True) - port = get_free_port() - command = [ - find_chrome(), - f"--remote-debugging-port={port}", - f"--user-data-dir={profile_dir}", - "--no-first-run", - "--no-default-browser-check", - start_url, - ] - if headless: - command.extend(["--headless=new", "--disable-gpu", "--window-size=1440,1200"]) - proc = subprocess.Popen(command, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) - wait_for_cdp(port) - return proc, port - -def ensure_logged_in(profile_dir: Path, timeout_sec: int, force_headed: bool, non_interactive: bool) -> tuple[subprocess.Popen[bytes], int, bool, bool]: - if force_headed: - proc, port = launch_browser(profile_dir, "https://x.com/home", headless=False) - wait_for_login(port, timeout_sec, interactive=True) - return proc, port, False, True - - proc, port = launch_browser(profile_dir, "https://x.com/home", headless=True) - if is_logged_in(port): - print("X login detected in saved browser session. Continuing headless collection...") - return proc, port, True, True - - if non_interactive: - print("Saved X login was not available. Non-interactive mode will record a login data gap without opening a browser.") - return proc, port, True, False - - print("Saved X login was not available. Opening a visible browser window for one-time login...") - stop_browser(proc) - proc, port = launch_browser(profile_dir, "https://x.com/home", headless=False) - wait_for_login(port, timeout_sec, interactive=True) - print("X login completed. Returning to headless collection...") - stop_browser(proc) - proc, port = launch_browser(profile_dir, "https://x.com/home", headless=True) - wait_for_login(port, timeout_sec, interactive=False) - return proc, port, True, True - -def is_logged_in(port: int) -> bool: - try: - ws_url = wait_for_cdp_page_ws(port) - return has_x_login_cookie(ws_url) - except Exception: - return False - -def wait_for_login(port: int, timeout_sec: int, interactive: bool) -> None: - if interactive: - print("Waiting for X login in the opened browser window...") - else: - print("Waiting for X login...") - deadline = time.time() + timeout_sec - last_notice = 0.0 - while time.time() < deadline: - try: - ws_url = wait_for_cdp_page_ws(port) - if has_x_login_cookie(ws_url): - print("X login detected. Continuing with browser collection...") - return - except Exception: - pass - if time.time() - last_notice > 15: - if interactive: - print("Still waiting for X login. Log in once in the opened browser window.") - else: - print("Still waiting for X login.") - last_notice = time.time() - time.sleep(2) - raise SystemExit("Timed out waiting for X login.") - -def stop_browser(proc: subprocess.Popen[bytes]) -> None: - proc.terminate() - try: - proc.wait(timeout=5) - except subprocess.TimeoutExpired: - proc.kill() - -def has_x_login_cookie(ws_url: str) -> bool: - for cookie in cdp_get_all_cookies(ws_url): - if cookie.get("name") == "auth_token" and domain_matches_x(str(cookie.get("domain") or "")): - return True - return False - -def domain_matches_x(domain: str) -> bool: - domain = domain.lstrip(".").lower() - return domain == "x.com" or domain.endswith(".x.com") or domain == "twitter.com" or domain.endswith(".twitter.com") diff --git a/twitter-digest/scripts/browser_x_digest.py b/twitter-digest/scripts/browser_x_digest.py deleted file mode 100644 index f67b754..0000000 --- a/twitter-digest/scripts/browser_x_digest.py +++ /dev/null @@ -1,315 +0,0 @@ -#!/usr/bin/env python3 -"""Collect X/Twitter digest input through a persistent local browser session. - -Module map: -- CLI options: translate chat-facing requests into collector settings. -- Page routing: decide whether a page uses public scraping or DM scraping. -- Message retries: recover from X Messages loading/skeleton states. -- Main orchestration: browser lifecycle, passcode recovery, incremental output. -""" - -from __future__ import annotations - -import argparse -import datetime as dt -import json -import urllib.parse -from pathlib import Path -from typing import Any, Optional - -from browser_lifecycle import ensure_logged_in, launch_browser, stop_browser, wait_for_login -from cdp_client import cdp_call, cdp_error, wait_for_cdp_page_ws -from digest_io import write_digest_output -from dm_scraper import collect_messages_page, dm_collection_looks_premature, wait_for_dm_passcode_resolution, wait_for_dm_ready -from public_scraper import collect_public_items, detect_handle, wait_for_public_page_ready - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - default_state_dir = Path(__file__).resolve().parents[1] / ".state" - parser.add_argument("--handle", help="Your X handle, with or without @. If omitted, the script tries to detect it from the logged-in page.") - parser.add_argument("--keywords", default="", help="Comma-separated keywords or queries for hotspot search.") - parser.add_argument("--out", default=str(default_state_dir / "run"), help="Output directory.") - parser.add_argument("--profile-dir", default=str(default_state_dir / "chrome-profile"), help="Dedicated browser profile directory used to persist X login/session state.") - parser.add_argument("--scrolls", type=int, default=40, help="Maximum scroll rounds per public page.") - parser.add_argument("--min-public-scrolls", type=int, default=5, help="Minimum public-page scroll rounds before early stop rules can end collection.") - parser.add_argument("--max-public-items", type=int, default=100, help="Maximum public post items kept per browser run.") - parser.add_argument("--public-window-hours", type=int, default=24, help="Stop loading older public timeline items once posts beyond this window are detected.") - parser.add_argument("--login-timeout-sec", type=int, default=300) - parser.add_argument("--include-dms", action="store_true", help="Also visit X messages and capture visible conversation text.") - parser.add_argument("--dm-only", action="store_true", help="Collect only X messages. Used when API already collected public data.") - parser.add_argument("--dm-threads", type=int, default=5, help="Maximum recent DM threads to open when --include-dms is set.") - parser.add_argument("--dm-list-scrolls", type=int, default=20, help="Maximum downward scroll rounds used to scan today's DM conversation list.") - parser.add_argument("--dm-scrolls", type=int, default=200, help="Maximum upward scroll rounds per opened DM thread.") - parser.add_argument("--dm-max-messages", type=int, default=2000, help="Maximum message bubbles kept per opened DM thread.") - parser.add_argument("--dm-window-hours", type=int, default=0, help="Stop loading older DM history once messages beyond this window are detected. 0 means load the full thread available in the browser.") - parser.add_argument("--headless", action="store_true", help="Run without a visible browser window. This is the default after first login.") - parser.add_argument("--headed", action="store_true", help="Force a visible browser window for debugging or manual login.") - parser.add_argument("--non-interactive", action="store_true", help="Do not open a visible browser for DM passcode recovery; record a data gap instead.") - parser.add_argument("--keep-browser-open", action="store_true", help="Leave the launched browser process running after collection for manual debugging.") - return parser.parse_args() - - -def build_pages(handle: Optional[str], keywords: str, include_dms: bool, dm_only: bool = False) -> list[dict[str, str]]: - if dm_only: - return [{"kind": "messages", "url": "https://x.com/messages"}] if include_dms else [] - pages = [{"kind": "home", "url": "https://x.com/home"}] - if handle: - clean = handle.lstrip("@") - query = urllib.parse.quote(f"@{clean}") - pages.append({"kind": "own_profile", "url": f"https://x.com/{clean}"}) - pages.append({"kind": "mentions_search", "url": f"https://x.com/search?q={query}&src=typed_query&f=live"}) - pages.append({"kind": "mentions_notifications", "url": "https://x.com/notifications/mentions"}) - for index, keyword in enumerate([k.strip() for k in keywords.split(",") if k.strip()], start=1): - pages.append( - { - "kind": f"keyword_{index}", - "url": f"https://x.com/search?q={urllib.parse.quote(keyword)}&src=typed_query&f=live", - } - ) - if include_dms: - pages.append({"kind": "messages", "url": "https://x.com/messages"}) - return pages - - -def collect_page( - port: int, - page: dict[str, str], - scrolls: int, - dm_threads: int = 5, - dm_list_scrolls: int = 20, - dm_scrolls: int = 200, - dm_max_messages: int = 2000, - dm_window_hours: int = 0, - max_public_items: int = 100, - public_window_hours: int = 24, - min_public_scrolls: int = 5, -) -> dict[str, Any]: - ws_url = wait_for_cdp_page_ws(port) - cdp_call(ws_url, "Page.enable") - cdp_call(ws_url, "Runtime.enable") - navigate_result = cdp_call(ws_url, "Page.navigate", {"url": page["url"]}) - if cdp_error(navigate_result): - return { - "kind": page["kind"], - "url": page["url"], - "items": [], - "collection_status": "error", - "collection_error": navigate_result["_cdp_error"], - } - if page["kind"] == "messages": - extra = collect_messages_with_retries( - ws_url, - page["url"], - dm_threads, - dm_list_scrolls, - dm_scrolls, - dm_max_messages, - dm_window_hours, - ) - return {"kind": page["kind"], "url": page["url"], "items": [], **extra} - wait_for_public_page_ready(ws_url, timeout_sec=20) - extra = collect_public_items(ws_url, max_scrolls=scrolls, max_items=max_public_items, window_hours=public_window_hours, min_scrolls=min_public_scrolls) - return {"kind": page["kind"], "url": page["url"], **extra} - - -def collect_messages_with_retries( - ws_url: str, - url: str, - dm_threads: int, - dm_list_scrolls: int, - dm_scrolls: int, - dm_max_messages: int, - dm_window_hours: int, - max_attempts: int = 3, -) -> dict[str, Any]: - extra: dict[str, Any] = {} - for attempt in range(1, max_attempts + 1): - wait_for_dm_ready(ws_url, timeout_sec=25) - extra = collect_messages_page(ws_url, dm_threads, dm_scrolls, dm_max_messages, dm_window_hours, dm_list_scrolls) - if not dm_collection_looks_premature(extra): - break - extra["dm_retry_attempts"] = attempt - if attempt < max_attempts: - print(f"X Messages still appears to be loading or incomplete. Reloading messages page (attempt {attempt + 1}/{max_attempts})...", flush=True) - cdp_call(ws_url, "Page.navigate", {"url": url}) - wait_for_dm_ready(ws_url, timeout_sec=25) - if dm_collection_looks_premature(extra): - original_status = str(extra.get("dm_status") or "unknown") - extra["dm_original_status"] = original_status - extra["dm_status"] = "dm_page_loading_timeout" - extra["collection_status"] = "partial" - extra["collection_error"] = "X Messages did not finish loading before retry budget was exhausted." - extra["dm_note"] = ( - f"X Messages page stayed in a loading/skeleton state after {max_attempts} attempts. " - "DM content was not treated as empty; rerun later or use --headed to inspect the page." - ) - return extra - - -def error_page(page: dict[str, str], exc: BaseException) -> dict[str, Any]: - return { - "kind": page.get("kind") or "unknown", - "url": page.get("url") or "", - "items": [], - "collection_status": "error", - "collection_error": str(exc) or exc.__class__.__name__, - } - - -def resolve_required_handle( - args: argparse.Namespace, - profile_dir: Path, - proc: Any, - port: int, - headless: bool, -) -> tuple[str, Any, int, bool]: - explicit = args.handle.lstrip("@").strip() if args.handle else "" - if explicit: - return explicit, proc, port, headless - - handle = detect_handle(port) - if handle: - return handle, proc, port, headless - - if args.non_interactive: - raise SystemExit( - "Could not auto-detect the authenticated X handle from the saved browser session. " - "Rerun interactively so the browser can be inspected, or pass --handle . " - "The digest was stopped because mentions and own-profile collection require the authenticated account." - ) - - if headless and not args.headed: - print("Could not auto-detect X handle in headless mode. Opening a visible browser window to verify the logged-in account...", flush=True) - stop_browser(proc) - proc, port = launch_browser(profile_dir, "https://x.com/home", headless=False) - headless = False - wait_for_login(port, args.login_timeout_sec, interactive=True) - handle = detect_handle(port) - if handle: - return handle, proc, port, headless - - raise SystemExit( - "Could not auto-detect the authenticated X handle from the browser UI. " - "Open the visible X window, confirm the correct account is logged in and the left account/profile area is visible, " - "then rerun the digest or pass --handle . " - "The digest was stopped because @mentions and own-profile collection would be incomplete without this." - ) - - -def main() -> None: - args = parse_args() - if args.dm_only: - args.include_dms = True - profile_dir = Path(args.profile_dir).expanduser().resolve() - force_headed = bool(args.headed and not args.headless) - proc, port, headless, logged_in = ensure_logged_in(profile_dir, args.login_timeout_sec, force_headed, args.non_interactive) - try: - out_dir = Path(args.out).expanduser().resolve() - if not logged_in: - data = { - "generated_at": dt.datetime.now().astimezone().isoformat(), - "profile_dir": str(profile_dir), - "handle": args.handle.lstrip("@") if args.handle else None, - "keywords": [k.strip() for k in args.keywords.split(",") if k.strip()], - "pages": [ - { - "kind": "login", - "url": "https://x.com/home", - "items": [], - "collection_status": "skipped", - "collection_error": "Saved X login unavailable in non-interactive mode.", - } - ], - } - write_digest_output(out_dir, data) - print(json.dumps({"out_dir": str(out_dir), "pages": len(data["pages"]), "headless": headless, "login": "unavailable"}, indent=2)) - return - if args.dm_only: - handle = args.handle.lstrip("@") if args.handle else None - else: - handle, proc, port, headless = resolve_required_handle(args, profile_dir, proc, port, headless) - print(f"Using X handle: @{handle}") - pages = build_pages(handle, args.keywords, args.include_dms, args.dm_only) - data = { - "generated_at": dt.datetime.now().astimezone().isoformat(), - "profile_dir": str(profile_dir), - "handle": handle, - "keywords": [k.strip() for k in args.keywords.split(",") if k.strip()], - "pages": [], - } - for page in pages: - print(f"Collecting {page['kind']}: {page['url']}") - try: - if page["kind"] == "messages" and headless: - stop_browser(proc) - proc, port = launch_browser(profile_dir, "https://x.com/messages", headless=True) - wait_for_login(port, args.login_timeout_sec, interactive=False) - result = collect_page( - port, - page, - args.scrolls, - args.dm_threads, - args.dm_list_scrolls, - args.dm_scrolls, - args.dm_max_messages, - args.dm_window_hours, - args.max_public_items, - args.public_window_hours, - args.min_public_scrolls, - ) - if page["kind"] == "messages" and result.get("dm_status") == "blocked_by_x_chat_passcode": - if args.non_interactive: - result["dm_note"] = "X Chat passcode is required. Non-interactive mode skipped DM recovery for this run." - data["pages"].append(result) - write_digest_output(out_dir, data) - continue - resume_headless_after_passcode = bool(headless) - if headless: - print("DM passcode screen detected in headless mode. Reopening X Messages in a visible browser window...") - stop_browser(proc) - proc, port = launch_browser(profile_dir, "https://x.com/messages", headless=False) - headless = False - wait_for_login(port, args.login_timeout_sec, interactive=True) - if wait_for_dm_passcode_resolution(port, args.login_timeout_sec): - if resume_headless_after_passcode: - print("X Chat passcode was completed. Returning to headless mode before DM collection...") - stop_browser(proc) - proc, port = launch_browser(profile_dir, "https://x.com/messages", headless=True) - headless = True - wait_for_login(port, args.login_timeout_sec, interactive=False) - result = collect_page( - port, - page, - args.scrolls, - args.dm_threads, - args.dm_list_scrolls, - args.dm_scrolls, - args.dm_max_messages, - args.dm_window_hours, - args.max_public_items, - args.public_window_hours, - args.min_public_scrolls, - ) - else: - result["dm_note"] = ( - "Timed out waiting for X Messages to become readable after passcode handling. " - "Keep the visible browser window open, complete passcode setup or entry until the inbox is visible, then rerun the digest." - ) - except SystemExit as exc: - result = error_page(page, exc) - print(f"Collection failed for {page['kind']}: {result['collection_error']}", flush=True) - except Exception as exc: - result = error_page(page, exc) - print(f"Collection failed for {page['kind']}: {result['collection_error']}", flush=True) - data["pages"].append(result) - write_digest_output(out_dir, data) - write_digest_output(out_dir, data) - print(json.dumps({"out_dir": str(out_dir), "pages": len(data["pages"]), "headless": headless}, indent=2)) - finally: - if not args.keep_browser_open: - stop_browser(proc) - - -if __name__ == "__main__": - main() diff --git a/twitter-digest/scripts/cdp_client.py b/twitter-digest/scripts/cdp_client.py deleted file mode 100644 index 519d34a..0000000 --- a/twitter-digest/scripts/cdp_client.py +++ /dev/null @@ -1,165 +0,0 @@ -"""Chrome DevTools Protocol helpers for browser-based X collection.""" - -from __future__ import annotations - -import base64 -import hashlib -import json -import os -import socket -import struct -import time -import urllib.parse -import urllib.request -from typing import Any, Optional - - -def wait_for_cdp(port: int) -> None: - url = f"http://127.0.0.1:{port}/json/version" - deadline = time.time() + 30 - while time.time() < deadline: - try: - with urllib.request.urlopen(url, timeout=1) as response: - json.loads(response.read().decode("utf-8")) - return - except Exception: - time.sleep(0.2) - raise SystemExit("Timed out waiting for browser DevTools endpoint.") - -def wait_for_cdp_page_ws(port: int) -> str: - url = f"http://127.0.0.1:{port}/json/list" - deadline = time.time() + 30 - fallback: Optional[str] = None - while time.time() < deadline: - try: - with urllib.request.urlopen(url, timeout=1) as response: - targets = json.loads(response.read().decode("utf-8")) - except Exception: - time.sleep(0.2) - continue - if isinstance(targets, list): - for target in targets: - if not isinstance(target, dict): - continue - ws_url = target.get("webSocketDebuggerUrl") - target_type = target.get("type") - if isinstance(ws_url, str) and target_type == "page": - fallback = fallback or ws_url - target_url = str(target.get("url") or "") - if "x.com" in target_url or "twitter.com" in target_url: - return ws_url - if fallback: - return fallback - time.sleep(0.2) - raise SystemExit("Timed out waiting for browser page DevTools endpoint.") - -def cdp_get_all_cookies(ws_url: str) -> list[dict[str, object]]: - result = cdp_call(ws_url, "Network.getAllCookies") - if isinstance(result, dict) and isinstance(result.get("cookies"), list): - return [c for c in result["cookies"] if isinstance(c, dict)] - result = cdp_call(ws_url, "Storage.getCookies") - if isinstance(result, dict) and isinstance(result.get("cookies"), list): - return [c for c in result["cookies"] if isinstance(c, dict)] - return [] - -def cdp_eval(ws_url: str, expression: str) -> Any: - result = cdp_call(ws_url, "Runtime.evaluate", {"expression": expression, "returnByValue": True, "awaitPromise": True}) - if not isinstance(result, dict) or cdp_error(result): - return None - remote = result.get("result", {}) - if isinstance(remote, dict): - return remote.get("value") - return None - -def cdp_error(result: Any) -> bool: - return isinstance(result, dict) and isinstance(result.get("_cdp_error"), str) - -def cdp_call(ws_url: str, method: str, params: Optional[dict[str, Any]] = None, retries: int = 2) -> Any: - last_error = "" - for _ in range(max(retries, 1)): - sock: Optional[socket.socket] = None - try: - sock = websocket_connect(ws_url) - websocket_send_json(sock, {"id": 1, "method": method, "params": params or {}}) - deadline = time.time() + 20 - while time.time() < deadline: - message = websocket_recv_json(sock) - if message.get("id") == 1: - if "error" in message: - return {"_cdp_error": json.dumps(message.get("error"), ensure_ascii=False)} - return message.get("result", {}) - last_error = f"Timed out waiting for CDP response to {method}" - except Exception as exc: - last_error = str(exc) - time.sleep(0.4) - finally: - if sock is not None: - sock.close() - return {"_cdp_error": last_error or f"CDP call failed: {method}"} - -def websocket_connect(ws_url: str) -> socket.socket: - if not ws_url.startswith("ws://"): - raise RuntimeError("Only local ws:// DevTools endpoints are supported.") - without_scheme = ws_url[len("ws://") :] - host_port, path = without_scheme.split("/", 1) - path = "/" + path - host, port_s = host_port.rsplit(":", 1) - raw_sock = socket.create_connection((host, int(port_s)), timeout=5) - key = base64.b64encode(os.urandom(16)).decode("ascii") - request = ( - f"GET {path} HTTP/1.1\r\n" - f"Host: {host_port}\r\n" - "Upgrade: websocket\r\n" - "Connection: Upgrade\r\n" - f"Sec-WebSocket-Key: {key}\r\n" - "Sec-WebSocket-Version: 13\r\n\r\n" - ) - raw_sock.sendall(request.encode("ascii")) - response = raw_sock.recv(4096) - if b" 101 " not in response.split(b"\r\n", 1)[0]: - raw_sock.close() - raise RuntimeError("Could not open DevTools WebSocket.") - return raw_sock - -def websocket_send_json(sock: socket.socket, payload: dict[str, object]) -> None: - data = json.dumps(payload).encode("utf-8") - header = bytearray([0x81]) - if len(data) < 126: - header.append(0x80 | len(data)) - elif len(data) < 65536: - header.append(0x80 | 126) - header.extend(struct.pack("!H", len(data))) - else: - header.append(0x80 | 127) - header.extend(struct.pack("!Q", len(data))) - mask = os.urandom(4) - masked = bytes(byte ^ mask[index % 4] for index, byte in enumerate(data)) - sock.sendall(bytes(header) + mask + masked) - -def websocket_recv_json(sock: socket.socket) -> dict[str, object]: - first_two = recv_exact(sock, 2) - opcode = first_two[0] & 0x0F - if opcode == 0x8: - raise ConnectionError("DevTools WebSocket closed.") - length = first_two[1] & 0x7F - if length == 126: - length = struct.unpack("!H", recv_exact(sock, 2))[0] - elif length == 127: - length = struct.unpack("!Q", recv_exact(sock, 8))[0] - masked = bool(first_two[1] & 0x80) - mask = recv_exact(sock, 4) if masked else b"" - data = recv_exact(sock, length) - if masked: - data = bytes(byte ^ mask[index % 4] for index, byte in enumerate(data)) - if opcode != 0x1: - return {} - return json.loads(data.decode("utf-8")) - -def recv_exact(sock: socket.socket, size: int) -> bytes: - chunks = bytearray() - while len(chunks) < size: - chunk = sock.recv(size - len(chunks)) - if not chunk: - raise ConnectionError("Unexpected end of DevTools WebSocket stream.") - chunks.extend(chunk) - return bytes(chunks) diff --git a/twitter-digest/scripts/collector_commands.py b/twitter-digest/scripts/collector_commands.py index 65b3e4f..31b040e 100644 --- a/twitter-digest/scripts/collector_commands.py +++ b/twitter-digest/scripts/collector_commands.py @@ -63,66 +63,3 @@ def api_collector_command( if handle: cmd.extend(["--handle", handle.lstrip("@")]) return cmd - - -def browser_collector_command( - python_executable: str, - scripts_dir: Path, - out_dir: str | Path, - *, - keywords: str, - max_public_items: int, - public_window_hours: int, - min_public_scrolls: int, - scrolls: int, - dm_threads: int, - dm_list_scrolls: int, - dm_scrolls: int, - dm_max_messages: int, - dm_window_hours: int, - handle: str = "", - include_dms: bool = False, - dm_only: bool = False, - headed: bool = False, - headless: bool = False, - non_interactive: bool = False, -) -> list[str]: - cmd = [ - python_executable, - str(scripts_dir / "browser_x_digest.py"), - "--keywords", - keywords, - "--out", - str(out_dir), - "--max-public-items", - str(max_public_items), - "--public-window-hours", - str(public_window_hours), - "--min-public-scrolls", - str(min_public_scrolls), - "--scrolls", - str(scrolls), - "--dm-threads", - str(dm_threads), - "--dm-list-scrolls", - str(dm_list_scrolls), - "--dm-scrolls", - str(dm_scrolls), - "--dm-max-messages", - str(dm_max_messages), - "--dm-window-hours", - str(dm_window_hours), - ] - if handle: - cmd.extend(["--handle", handle.lstrip("@")]) - if include_dms: - cmd.append("--include-dms") - if dm_only: - cmd.append("--dm-only") - if headed: - cmd.append("--headed") - if headless: - cmd.append("--headless") - if non_interactive: - cmd.append("--non-interactive") - return cmd diff --git a/twitter-digest/scripts/configure_api.py b/twitter-digest/scripts/configure_api.py index 6bfa2ef..4a13311 100644 --- a/twitter-digest/scripts/configure_api.py +++ b/twitter-digest/scripts/configure_api.py @@ -274,7 +274,7 @@ def main() -> None: args=sys.argv[1:], cwd=Path(__file__).resolve().parents[1], heading="X API 配置向导", - description="请在这个 Terminal 窗口里输入 Client ID / Secret，并在浏览器里完成 X OAuth2 授权。", + description="请在这个 Terminal 窗口里输入 Client ID / Secret，并完成 X OAuth2 授权。", ) if opened: print("已打开 Terminal 窗口用于配置 X API。", flush=True) @@ -347,7 +347,7 @@ def main() -> None: "handle": saved.get("handle") or config["handle"], "user_id": saved.get("user_id") or config["user_id"], "verification": verification, - "next_step": "Normal scripts/run_daily_digest.py runs now use the saved API token. Use --source browser only when you explicitly want browser collection.", + "next_step": "Normal scripts/run_daily_digest.py runs now use the saved API token.", }, ensure_ascii=False, indent=2, diff --git a/twitter-digest/scripts/digest_context.py b/twitter-digest/scripts/digest_context.py index 7ac2c93..9d7515f 100644 --- a/twitter-digest/scripts/digest_context.py +++ b/twitter-digest/scripts/digest_context.py @@ -73,7 +73,7 @@ def summarize_current_run(data: dict[str, Any]) -> dict[str, Any]: return { "generated_at": generated_at, "date": generated_at[:10], - "source": str(data.get("source") or "browser"), + "source": str(data.get("source") or "api"), "handle": clean_handle(data.get("handle")), "post_counts": post_counts, "dm_status": dm_status, @@ -246,7 +246,7 @@ def build_digest_facts(data: dict[str, Any], summary: dict[str, Any]) -> dict[st { "source": "messages", "status": "api_dm_todo", - "detail": facts["dms"].get("note") or "API DM was inconclusive; use browser DM collection before making a final DM claim.", + "detail": facts["dms"].get("note") or "API DM was inconclusive; do not make a final DM claim.", } ) return facts @@ -457,10 +457,10 @@ def normalize_context_assets(value: Any) -> list[dict[str, str]]: def render_digest_input(data: dict[str, Any]) -> str: lines = [ - "# X 浏览器采集输入", + "# X API 采集输入", "", f"- 生成时间: `{data.get('generated_at')}`", - f"- 浏览器 profile: `{data.get('profile_dir')}`", + f"- 数据源: `{data.get('source') or 'api'}`", f"- 当前账号: `{data.get('handle') or ''}`", "", ] @@ -505,9 +505,9 @@ def render_digest_input(data: dict[str, Any]) -> str: [ "## 数据缺口", "", - "- 浏览器采集依赖 X 页面结构和已加载的可见内容。", + "- API 采集受 X API 权限、套餐、端点可用性和限流影响。", "- 公开内容标记为 `[current]`，最终日报只依赖本次采集。", - "- DM 属于私密内容，不写长期 memory 或 daily archive。", + "- DM / X Chat 可能不会完整出现在 API 结果中；不要把 0 条 API DM 当作没有私信。", ] ) return "\n".join(lines) + "\n" @@ -525,7 +525,7 @@ def render_digest_context(summary: dict[str, Any], facts: dict[str, Any]) -> str "", f"- date: `{summary.get('date')}`", f"- handle: `@{summary.get('handle') or ''}`", - f"- source: `{summary.get('source') or 'browser'}`", + f"- source: `{summary.get('source') or 'api'}`", f"- context policy: {summary.get('context_policy')}", f"- DM status: `{summary.get('dm_status')}`", ( @@ -562,7 +562,7 @@ def render_context_slice(summary: dict[str, Any], facts: dict[str, Any], slice_n "", f"- date: `{summary.get('date')}`", f"- handle: `@{summary.get('handle') or ''}`", - f"- source: `{summary.get('source') or 'browser'}`", + f"- source: `{summary.get('source') or 'api'}`", f"- context policy: {summary.get('context_policy')}", "", ] @@ -729,7 +729,7 @@ def render_digest_facts(facts: dict[str, Any]) -> str: "", f"- date: `{(facts.get('run') or {}).get('date')}`", f"- generated_at: `{(facts.get('run') or {}).get('generated_at')}`", - f"- source: `{(facts.get('run') or {}).get('source') or 'browser'}`", + f"- source: `{(facts.get('run') or {}).get('source') or 'api'}`", f"- timezone: `{(facts.get('run') or {}).get('timezone')}`", f"- window_start: `{(facts.get('run') or {}).get('window_start')}`", f"- window_end: `{(facts.get('run') or {}).get('window_end')}`", diff --git a/twitter-digest/scripts/digest_io.py b/twitter-digest/scripts/digest_io.py index 9208e49..2df9a44 100644 --- a/twitter-digest/scripts/digest_io.py +++ b/twitter-digest/scripts/digest_io.py @@ -20,7 +20,7 @@ def render_markdown(data: dict[str, Any]) -> str: "# X 采集输入", "", f"- 生成时间: `{data['generated_at']}`", - f"- 浏览器 profile: `{data.get('profile_dir') or ''}`", + f"- 数据源: `{data.get('source') or 'api'}`", f"- 当前账号: `{data.get('handle') or ''}`", "", ] @@ -64,9 +64,9 @@ def render_markdown(data: dict[str, Any]) -> str: [ "## 数据缺口", "", - "- 浏览器采集依赖 X 页面结构和已加载的可见内容。", - "- 日报默认使用较小滚动次数；需要更全覆盖时再提高 `--scrolls`。", - "- DM 属于私密内容。只有用户明确同意本地读取时才使用 `--include-dms`。", + "- API 采集受 X API 权限、套餐、端点可用性和限流影响。", + "- DM / X Chat 可能不会完整出现在 API 结果中；不要把 0 条 API DM 当作没有私信。", + "- DM 属于私密内容，不写长期 memory 或 daily archive。", ] ) return "\n".join(lines) + "\n" diff --git a/twitter-digest/scripts/dm_scraper.py b/twitter-digest/scripts/dm_scraper.py deleted file mode 100644 index bd9e6f3..0000000 --- a/twitter-digest/scripts/dm_scraper.py +++ /dev/null @@ -1,725 +0,0 @@ -"""X Direct Message scraping helpers. - -Module map: -- Page-level collection: read the X Messages page and decide the DM status. -- Conversation-list scanning: scan today's visible threads and classify reply state. -- Conversation loading: open waiting-reply threads and load enough history. -- Readiness detection: distinguish empty inbox, loading skeleton, and passcode screens. -- Target parsing: normalize DOM-extracted thread rows into Python dictionaries. -""" - -from __future__ import annotations - -import datetime as dt -import re -import time -from typing import Any, TypedDict - -from cdp_client import cdp_call, cdp_eval, wait_for_cdp_page_ws -from dom_script_loader import load_dom_script -from public_scraper import extract_main_text - - -class DmAsset(TypedDict, total=False): - url: str - type: str - poster: str - alt: str - label: str - - -class DmMessage(TypedDict, total=False): - sender: str - time: str - text: str - links: list[DmAsset] - media: list[DmAsset] - - -class DmTarget(TypedDict, total=False): - target_type: str - url: str - label: str - time_hint: str - replied: bool - reply_reason: str - today: bool - x: float - y: float - - -class DmThread(TypedDict, total=False): - url: str - label: str - participant: str - target_type: str - replied: bool - reply_reason: str - today: bool - message_count: int - dm_scrolls_used: int - dm_load_complete: bool - dm_window_exceeded: bool - dm_hit_message_cap: bool - messages: list[DmMessage] - text: str - collection_status: str - collection_error: str - - -class DmListInfo(TypedDict, total=False): - targets: list[DmTarget] - scrolls_used: int - load_complete: bool - stable_rounds: int - - -DM_EMPTY_MARKERS = ("no messages", "welcome to your inbox") -DM_CONVERSATION_MARKERS = ("you:", "you sent", "now", " min", "m ", "h ", "today", "今天") -DM_SELF_REPLY_RE = re.compile( - r"\byou\s*[:：]|\byou sent\b|\byou replied\b|\byou responded\b|你\s*[:：]|你已发送|你发送|您\s*[:：]", - re.IGNORECASE, -) -DM_TODAY_WORD_RE = re.compile(r"\b(now|just now|sec|secs|second|seconds|min|mins|minute|minutes|h|hr|hrs|hour|hours)\b", re.IGNORECASE) -DM_TODAY_AGE_RE = re.compile(r"\b\d+\s*(m|min|mins|minute|minutes|h|hr|hrs|hour|hours)\b", re.IGNORECASE) -DM_TODAY_CN_RE = re.compile(r"(刚刚|秒|分钟|小时|今天|今日|上午|下午|晚上|中午)") -DM_OLD_TIME_RE = re.compile(r"\b(yesterday|d|day|days|w|week|weeks|mo|month|months|y|year|years)\b|昨天|周|週|月|年", re.IGNORECASE) -DM_PASSCODE_PHRASES = ( - "create passcode", - "set passcode", - "enter passcode", - "your passcode is required", - "recover your encryption keys", - "encryption keys", -) -DM_LIST_STABLE_STOP_ROUNDS = 4 -DM_LIST_OLDER_AFTER_TODAY_STABLE_ROUNDS = 2 -DM_LIST_BOTTOM_CONFIRM_ROUNDS = 1 -DM_LIST_SCROLL_SETTLE_SEC = 1.0 -DM_ROW_RELOCATE_MAX_SCROLLS = 8 -DM_ROW_RELOCATE_SETTLE_SEC = 0.5 -DM_CONVERSATION_READY_TIMEOUT_SEC = 12 -DM_CONVERSATION_READY_MIN_WORDS = 3 -DM_CONVERSATION_READY_POLL_SEC = 0.5 -DM_READY_AFTER_NAV_TIMEOUT_SEC = 8 -DM_READY_POLL_SEC = 1.0 -DM_HISTORY_SCROLL_SETTLE_SEC = 0.8 -DM_HISTORY_TOP_CONFIRM_ROUNDS = 1 -DM_PASSCODE_POLL_SEC = 3.0 -DM_PASSCODE_NOTICE_INTERVAL_SEC = 15.0 - - -# Page-level collection - - -def collect_messages_page(ws_url: str, dm_threads: int, dm_scrolls: int, dm_max_messages: int, dm_window_hours: int, dm_list_scrolls: int = 20) -> dict[str, Any]: - extra: dict[str, Any] = {} - extra["visible_text"] = extract_main_text(ws_url) - loading_state = dm_page_loading_state(ws_url) - extra["dm_loading_state"] = loading_state - if bool(loading_state.get("loading")): - return {**extra, **dm_loading_timeout_result()} - extra.update( - collect_dm_threads( - ws_url, - max_threads=dm_threads, - dm_scrolls=dm_scrolls, - dm_max_messages=dm_max_messages, - dm_window_hours=dm_window_hours, - dm_list_scrolls=dm_list_scrolls, - ) - ) - return extra - -def dm_collection_looks_premature(extra: dict[str, Any]) -> bool: - status = str(extra.get("dm_status") or "") - text = " ".join(str(extra.get("visible_text") or "").lower().split()) - if status == "dm_page_loading_timeout": - return True - loading_state = extra.get("dm_loading_state") if isinstance(extra.get("dm_loading_state"), dict) else {} - if bool(loading_state.get("loading")): - return True - if status not in {"no_today_threads", "no_visible_threads", "visible_threads_unopened"}: - return False - if "start conversation" not in text: - return False - return not any(marker in text for marker in DM_CONVERSATION_MARKERS) - - -def dm_loading_timeout_result() -> dict[str, Any]: - return { - "dm_status": "dm_page_loading_timeout", - "dm_note": ( - "X Messages page still showed skeleton/loading placeholders or Start Conversation before the " - "conversation list became readable. The collector will retry before treating DMs as unavailable." - ), - "dm_threads": [], - **dm_counts([]), - } - - -def collect_dm_threads(ws_url: str, max_threads: int, dm_scrolls: int, dm_max_messages: int, dm_window_hours: int, dm_list_scrolls: int = 20) -> dict[str, Any]: - main_text = wait_for_dm_ready(ws_url) - if is_dm_passcode_screen(main_text): - return { - "dm_status": "blocked_by_x_chat_passcode", - "dm_note": "X Chat is asking for an encryption passcode before message content is visible.", - "dm_threads": [], - **dm_counts([]), - } - - list_info = load_dm_thread_list_targets(ws_url, max_scrolls=dm_list_scrolls) - all_targets = list_info["targets"] - today_targets = today_dm_targets(all_targets) - thread_targets = unreplied_dm_targets(today_targets) - counts = dm_counts(today_targets) - if not thread_targets: - return no_unreplied_targets_result(main_text, all_targets, today_targets, counts, list_info) - - threads: list[DmThread] = [] - seen_targets: set[str] = set() - for target in thread_targets[: max(max_threads, 0)]: - if dm_target_key(target) in seen_targets: - continue - seen_targets.add(dm_target_key(target)) - threads.append(open_dm_thread(ws_url, target, dm_scrolls, dm_max_messages, dm_window_hours)) - cdp_call(ws_url, "Page.navigate", {"url": "https://x.com/messages"}) - wait_for_dm_ready(ws_url, timeout_sec=DM_READY_AFTER_NAV_TIMEOUT_SEC) - - captured_message_count = total_dm_message_count(threads) - return with_dm_list_metadata( - { - "dm_status": "captured_unreplied_threads" if threads else "no_unreplied_threads", - "dm_note": ( - f"Today visible DM threads: {counts['dm_visible_thread_count']}; latest from you: {counts['dm_replied_thread_count']}; " - f"waiting for your reply: {counts['dm_unreplied_thread_count']}. Opened up to {max_threads} waiting-reply thread(s); " - f"scanned DM list with {list_info['scrolls_used']} downward scroll round(s); " - f"loaded up to {dm_max_messages} message bubbles per thread with {dm_scrolls} upward scroll round(s); " - f"captured message bubbles: {captured_message_count}." - ), - "dm_threads": threads, - "dm_captured_message_count": captured_message_count, - **counts, - }, - list_info, - len(all_targets), - ) - - -def no_unreplied_targets_result( - main_text: str, - all_targets: list[DmTarget], - today_targets: list[DmTarget], - counts: dict[str, int], - list_info: DmListInfo, -) -> dict[str, Any]: - if today_targets: - status = "no_unreplied_threads" - note = ( - f"DM conversation list was visible with {counts['dm_visible_thread_count']} today thread target(s), " - "but every latest preview appears to be from you." - ) - elif all_targets: - status = "no_today_threads" - note = f"DM conversation list was visible with {len(all_targets)} older thread target(s), but no today conversation targets were found." - elif looks_like_dm_list_text(main_text): - status = "visible_threads_unopened" - note = "DM conversation list text was visible, but no unreplied openable conversation link or row target could be detected." - else: - status = "no_visible_threads" - note = "No DM conversation links or clickable conversation rows were visible after waiting for the messages page." - return with_dm_list_metadata( - {"dm_status": status, "dm_note": note, "dm_threads": [], **counts}, - list_info, - len(all_targets), - ) - - -def with_dm_list_metadata(result: dict[str, Any], list_info: DmListInfo, target_count: int) -> dict[str, Any]: - return { - **result, - "dm_list_scrolls_used": int(list_info.get("scrolls_used") or 0), - "dm_list_load_complete": bool(list_info.get("load_complete")), - "dm_list_target_count": target_count, - } - - -def open_dm_thread(ws_url: str, target: DmTarget, dm_scrolls: int, dm_max_messages: int, dm_window_hours: int) -> DmThread: - if not open_dm_target(ws_url, target): - return skipped_dm_thread(target, "Could not relocate DM row after scanning the thread list.") - wait_for_dm_conversation_content(ws_url, timeout_sec=DM_CONVERSATION_READY_TIMEOUT_SEC) - load_info = load_dm_thread_history(ws_url, max_scrolls=dm_scrolls, target_messages=dm_max_messages, window_hours=dm_window_hours) - messages = extract_dm_messages(ws_url, max_messages=dm_max_messages) - thread_text = render_dm_messages(messages) or extract_dm_conversation_text(ws_url) - message_count = len(messages) if messages else count_dm_messages(ws_url) - return { - **dm_thread_identity(target), - "message_count": message_count, - "dm_scrolls_used": load_info.get("scrolls_used", 0), - "dm_load_complete": load_info.get("load_complete", False), - "dm_window_exceeded": load_info.get("window_exceeded", False), - "dm_hit_message_cap": load_info.get("hit_message_cap", False), - "messages": messages, - "text": thread_text, - } - - -def open_dm_target(ws_url: str, target: DmTarget) -> bool: - has_click_point = float(target.get("x") or 0) > 0 and float(target.get("y") or 0) > 0 - if has_click_point: - cdp_call(ws_url, "Page.navigate", {"url": "https://x.com/messages"}) - wait_for_dm_ready(ws_url, timeout_sec=DM_READY_AFTER_NAV_TIMEOUT_SEC) - if not restore_dm_thread_list_position(ws_url, target): - return False - click_point(ws_url, float(target.get("x") or 0), float(target.get("y") or 0)) - return True - if target.get("url"): - cdp_call(ws_url, "Page.navigate", {"url": str(target["url"])}) - return True - return False - - -def skipped_dm_thread(target: DmTarget, error: str) -> DmThread: - return { - **dm_thread_identity(target), - "message_count": 0, - "dm_scrolls_used": 0, - "dm_load_complete": False, - "dm_window_exceeded": False, - "dm_hit_message_cap": False, - "messages": [], - "text": "", - "collection_status": "skipped", - "collection_error": error, - } - - -def dm_thread_identity(target: DmTarget) -> DmThread: - return { - "url": str(target.get("url") or ""), - "label": str(target.get("label") or ""), - "participant": dm_participant(target), - "target_type": str(target.get("target_type") or ""), - "replied": bool(target.get("replied")), - "reply_reason": str(target.get("reply_reason") or ""), - "today": bool(target.get("today")), - } - - -# Conversation-list scanning - - -def load_dm_thread_list_targets(ws_url: str, max_scrolls: int = 20) -> DmListInfo: - scroll_limit = max(0, int(max_scrolls)) - targets = extract_dm_thread_targets(ws_url) - previous_today = len(today_dm_targets(targets)) - previous_total = len(targets) - stable_rounds = 0 - scrolls_used = 0 - at_bottom = False - - for _ in range(scroll_limit): - if at_bottom and stable_rounds >= DM_LIST_BOTTOM_CONFIRM_ROUNDS: - break - info = scroll_dm_thread_list_down(ws_url) - scrolls_used += 1 - time.sleep(DM_LIST_SCROLL_SETTLE_SEC) - current = dedupe_dm_targets([*targets, *extract_dm_thread_targets(ws_url)]) - current_today = len(today_dm_targets(current)) - current_total = len(current) - at_bottom = bool(info.get("at_bottom")) - if current_today <= previous_today and current_total <= previous_total: - stable_rounds += 1 - else: - stable_rounds = 0 - targets = current - previous_today = current_today - previous_total = current_total - if dm_list_scan_should_stop(current_total, current_today, stable_rounds): - break - - return { - "targets": targets, - "scrolls_used": scrolls_used, - "load_complete": at_bottom or stable_rounds >= DM_LIST_STABLE_STOP_ROUNDS, - "stable_rounds": stable_rounds, - } - -def scroll_dm_thread_list_down(ws_url: str) -> dict[str, Any]: - script = load_dom_script("scroll_dm_thread_list_down.js") - value = cdp_eval(ws_url, script) - return value if isinstance(value, dict) else {} - - -def dm_list_scan_should_stop(total_targets: int, today_targets_count: int, stable_rounds: int) -> bool: - if stable_rounds >= DM_LIST_STABLE_STOP_ROUNDS: - return True - reached_older_threads_after_today = total_targets > 0 and today_targets_count > 0 and today_targets_count < total_targets - return reached_older_threads_after_today and stable_rounds >= DM_LIST_OLDER_AFTER_TODAY_STABLE_ROUNDS - - -def restore_dm_thread_list_position(ws_url: str, target: DmTarget) -> bool: - key = dm_target_key(target) - for _ in range(DM_ROW_RELOCATE_MAX_SCROLLS): - candidates = {dm_target_key(item): item for item in extract_dm_thread_targets(ws_url)} - current = candidates.get(key) - if current: - target["x"] = current.get("x") or target.get("x") - target["y"] = current.get("y") or target.get("y") - return True - scroll_dm_thread_list_down(ws_url) - time.sleep(DM_ROW_RELOCATE_SETTLE_SEC) - return False - - -# Conversation loading and message extraction - - -def wait_for_dm_conversation_content(ws_url: str, timeout_sec: int = 12) -> None: - deadline = time.time() + timeout_sec - while time.time() < deadline: - if count_dm_messages(ws_url) > 0: - return - text = extract_dm_conversation_text(ws_url) - if text and not is_dm_passcode_screen(text) and len(text.split()) > DM_CONVERSATION_READY_MIN_WORDS: - return - time.sleep(DM_CONVERSATION_READY_POLL_SEC) - -def dm_target_key(target: DmTarget) -> str: - return str(target.get("url") or target.get("label") or f"{target.get('x')}:{target.get('y')}") - -def dm_participant(target: DmTarget) -> str: - label = " ".join(str(target.get("label") or "").split()) - handle = re.search(r"@([A-Za-z0-9_]{1,15})", label) - if handle: - return "@" + handle.group(1) - first_chunk = re.split( - r"(?:\s+You:|\s+You sent|\s+You replied|\s+sent you|\s+你[:：]|\s+你已发送|\s+你发送|\s+您[:：]|\s+\d+\s*(?:m|h|d|min|hour|day)\b)", - label, - maxsplit=1, - flags=re.IGNORECASE, - )[0] - return first_chunk.strip(" -·•|")[:80] or label[:80] - -def unreplied_dm_targets(targets: list[DmTarget]) -> list[DmTarget]: - return [target for target in targets if not bool(target.get("replied"))] - -def today_dm_targets(targets: list[DmTarget]) -> list[DmTarget]: - return [target for target in targets if bool(target.get("today"))] - -def dm_counts(targets: list[DmTarget]) -> dict[str, int]: - replied_count = len([target for target in targets if bool(target.get("replied"))]) - visible_count = len(targets) - return { - "dm_visible_thread_count": visible_count, - "dm_replied_thread_count": replied_count, - "dm_unreplied_thread_count": max(visible_count - replied_count, 0), - } - -def total_dm_message_count(threads: list[DmThread]) -> int: - return sum(int(thread.get("message_count") or 0) for thread in threads) - -def dm_target_has_self_reply(label: str) -> bool: - normalized = " ".join(label.split()) - return bool(DM_SELF_REPLY_RE.search(normalized)) - -def load_dm_thread_history(ws_url: str, max_scrolls: int, target_messages: int, window_hours: int) -> dict[str, Any]: - scroll_limit = max(0, int(max_scrolls)) - target = max(1, int(target_messages)) - state = get_dm_history_state(ws_url) - last_count = int(state.get("count") or 0) - last_top_signature = str(state.get("top_signature") or "") - stable_top_rounds = 0 - scrolls_used = 0 - reached_top = bool(state.get("at_top")) - hit_message_cap = last_count >= target - window_exceeded = dm_loaded_beyond_window(ws_url, window_hours) if int(window_hours) > 0 else False - - while scrolls_used < scroll_limit and not reached_top and not window_exceeded and not hit_message_cap: - info = scroll_dm_messages_up(ws_url) - scrolls_used += 1 - time.sleep(DM_HISTORY_SCROLL_SETTLE_SEC) - state = get_dm_history_state(ws_url) - current_count = int(state.get("count") or 0) - current_top_signature = str(state.get("top_signature") or "") - window_exceeded = dm_loaded_beyond_window(ws_url, window_hours) if int(window_hours) > 0 else False - reached_top = bool(info.get("at_top")) or bool(state.get("at_top")) - hit_message_cap = current_count >= target - if current_count <= last_count: - if current_top_signature and current_top_signature == last_top_signature: - stable_top_rounds += 1 - else: - stable_top_rounds = 0 - else: - stable_top_rounds = 0 - last_count = max(last_count, current_count) - last_top_signature = current_top_signature or last_top_signature - if reached_top and stable_top_rounds >= DM_HISTORY_TOP_CONFIRM_ROUNDS: - break - - return { - "scrolls_used": scrolls_used, - "loaded_messages": last_count, - "load_complete": reached_top and not hit_message_cap, - "window_exceeded": window_exceeded, - "hit_message_cap": hit_message_cap, - "top_signature": last_top_signature, - "target_messages": target, - "window_hours": max(0, int(window_hours)), - } - -def get_dm_history_state(ws_url: str) -> dict[str, Any]: - script = load_dom_script("get_dm_history_state.js") - value = cdp_eval(ws_url, script) - return value if isinstance(value, dict) else {} - -def scroll_dm_messages_up(ws_url: str) -> dict[str, Any]: - script = load_dom_script("scroll_dm_messages_up.js") - value = cdp_eval(ws_url, script) - return value if isinstance(value, dict) else {} - -def dm_loaded_beyond_window(ws_url: str, window_hours: int) -> bool: - script = load_dom_script("dm_loaded_beyond_window.js") % max(1, int(window_hours)) - return bool(cdp_eval(ws_url, script)) - -def count_dm_messages(ws_url: str) -> int: - script = load_dom_script("count_dm_messages.js") - value = cdp_eval(ws_url, script) - return int(value) if isinstance(value, (int, float)) else 0 - -def extract_dm_messages(ws_url: str, max_messages: int = 300) -> list[DmMessage]: - script = load_dom_script("extract_dm_messages.js") % max(1, int(max_messages)) - value = cdp_eval(ws_url, script) - if not isinstance(value, list): - return [] - messages: list[DmMessage] = [] - for item in value: - if not isinstance(item, dict): - continue - text = str(item.get("text") or "").strip() - if not text: - continue - messages.append( - { - "sender": "me" if str(item.get("sender") or "") == "me" else "other", - "time": str(item.get("time") or ""), - "text": text[:1000], - "links": normalize_assets(item.get("links"), kind="links"), - "media": normalize_assets(item.get("media"), kind="media"), - } - ) - return messages - -def normalize_assets(value: Any, kind: str) -> list[DmAsset]: - if not isinstance(value, list): - return [] - out: list[DmAsset] = [] - for item in value: - if not isinstance(item, dict): - continue - url = str(item.get("url") or "").strip() - if not url: - continue - normalized: DmAsset = {"url": url[:1200]} - if kind == "media": - normalized["type"] = str(item.get("type") or "media")[:40] - if item.get("poster"): - normalized["poster"] = str(item.get("poster") or "")[:1200] - if item.get("alt"): - normalized["alt"] = str(item.get("alt") or "")[:500] - else: - if item.get("label"): - normalized["label"] = str(item.get("label") or "")[:500] - out.append(normalized) - return out[:10] - -def render_dm_messages(messages: list[DmMessage]) -> str: - lines = [] - for message in messages: - sender = "me" if message.get("sender") == "me" else "other" - timestamp = f" {message.get('time')}" if message.get("time") else "" - suffixes = [] - if message.get("links"): - suffixes.append("links=" + ", ".join(asset.get("url", "") for asset in message.get("links", [])[:3])) - if message.get("media"): - suffixes.append("media=" + ", ".join(asset.get("url", "") for asset in message.get("media", [])[:3])) - suffix = f" [{' ; '.join(suffixes)}]" if suffixes else "" - lines.append(f"{sender}{timestamp}: {message.get('text') or ''}{suffix}") - return "\n".join(lines) - -def extract_dm_conversation_text(ws_url: str) -> str: - script = load_dom_script("extract_dm_conversation_text.js") - value = cdp_eval(ws_url, script) - return str(value or "") - - -# Page readiness and status detection - - -def wait_for_dm_ready(ws_url: str, timeout_sec: int = 20) -> str: - deadline = time.time() + timeout_sec - last_text = "" - while time.time() < deadline: - text = extract_main_text(ws_url) - last_text = text or last_text - if is_dm_passcode_screen(text): - return text - if extract_dm_thread_targets(ws_url): - return text - state = dm_page_loading_state(ws_url) - if bool(state.get("loading")): - time.sleep(DM_READY_POLL_SEC) - continue - normalized = " ".join(text.lower().split()) - if any(marker in normalized for marker in DM_EMPTY_MARKERS): - return text - time.sleep(DM_READY_POLL_SEC) - return last_text - -def dm_page_loading_state(ws_url: str) -> dict[str, Any]: - script = load_dom_script("dm_page_loading_state.js") - value = cdp_eval(ws_url, script) - return value if isinstance(value, dict) else {} - -def looks_like_dm_list_text(text: str) -> bool: - lines = [line.strip() for line in text.splitlines() if line.strip()] - if len(lines) < 4: - return False - joined = " ".join(lines).lower() - if not any(marker in joined for marker in ("chat", "messages", "search", "inbox")): - return False - conversation_markers = ("@", "you sent", "you:", "sent you", "min", "m", "h", "d") - return any(marker in joined for marker in conversation_markers) - -def is_dm_passcode_screen(text: str) -> bool: - normalized = " ".join(text.lower().split()) - if "passcode" not in normalized: - return False - return any(phrase in normalized for phrase in DM_PASSCODE_PHRASES) - -def wait_for_dm_passcode_resolution(port: int, timeout_sec: int) -> bool: - ws_url = wait_for_cdp_page_ws(port) - cdp_call(ws_url, "Page.enable") - cdp_call(ws_url, "Runtime.enable") - cdp_call(ws_url, "Page.navigate", {"url": "https://x.com/messages"}) - print("X Chat passcode is required. Complete it in the opened browser window; collection will resume automatically.") - deadline = time.time() + timeout_sec - last_notice = 0.0 - while time.time() < deadline: - time.sleep(DM_PASSCODE_POLL_SEC) - main_text = extract_main_text(ws_url) - if is_dm_passcode_screen(main_text): - if time.time() - last_notice > DM_PASSCODE_NOTICE_INTERVAL_SEC: - print("Still waiting for X Chat passcode to be completed in the opened browser window.") - last_notice = time.time() - continue - if dm_messages_page_is_readable(ws_url, main_text): - print("X Chat messages are readable. Resuming DM collection...") - return True - if time.time() - last_notice > DM_PASSCODE_NOTICE_INTERVAL_SEC: - print("Passcode screen is not readable yet or messages are still loading. Keep the visible browser open until X Messages shows the inbox.") - last_notice = time.time() - return False - -def dm_messages_page_is_readable(ws_url: str, text: str) -> bool: - if is_dm_passcode_screen(text): - return False - if extract_dm_thread_targets(ws_url): - return True - normalized = " ".join(text.lower().split()) - if any(marker in normalized for marker in DM_EMPTY_MARKERS): - return True - return False - - -# Conversation target parsing - - -def extract_dm_thread_targets(ws_url: str) -> list[DmTarget]: - script = load_dom_script("extract_dm_thread_targets.js") - value = cdp_eval(ws_url, script) - if not isinstance(value, list): - return [] - out: list[DmTarget] = [] - for item in value: - if isinstance(item, dict): - out.append(normalize_dm_target(item)) - return dedupe_dm_targets(out) - -def normalize_dm_target(item: dict[str, Any]) -> DmTarget: - label = str(item.get("label") or "") - time_hint = str(item.get("time_hint") or "") - return { - "target_type": str(item.get("target_type") or ""), - "url": str(item.get("url") or ""), - "label": label, - "time_hint": time_hint, - "replied": bool(item.get("replied")) or dm_target_has_self_reply(label), - "reply_reason": str(item.get("reply_reason") or ""), - "today": dm_target_is_today(label, time_hint), - "x": float(item.get("x") or 0), - "y": float(item.get("y") or 0), - } - -def dm_target_is_today(label: str, time_hint: str = "") -> bool: - combined = " ".join(part for part in (time_hint, label) if part) - if dm_time_hint_is_today(time_hint): - return True - normalized = " ".join(combined.lower().split()) - if not normalized: - return False - if DM_TODAY_WORD_RE.search(normalized): - return True - if DM_TODAY_AGE_RE.search(normalized): - return True - if DM_TODAY_CN_RE.search(normalized): - return True - if DM_OLD_TIME_RE.search(normalized): - return False - return False - -def dm_time_hint_is_today(value: str) -> bool: - if not value: - return False - today = dt.datetime.now().astimezone().date() - for match in re.findall(r"\d{4}-\d{2}-\d{2}(?:[T ][0-9:.+-Z]*)?", value): - try: - parsed = dt.datetime.fromisoformat(match.replace("Z", "+00:00")) - except ValueError: - continue - if parsed.tzinfo is None: - parsed = parsed.replace(tzinfo=dt.datetime.now().astimezone().tzinfo) - if parsed.astimezone().date() == today: - return True - return False - -def dedupe_dm_targets(targets: list[DmTarget]) -> list[DmTarget]: - deduped: dict[str, DmTarget] = {} - for target in targets: - key = str(target.get("url") or dm_participant(target) or target.get("label") or "").lower() - if not key: - key = f"{round(float(target.get('x') or 0))}:{round(float(target.get('y') or 0))}" - existing = deduped.get(key) - if existing is None: - deduped[key] = target - continue - existing["replied"] = bool(existing.get("replied")) or bool(target.get("replied")) - existing["today"] = bool(existing.get("today")) or bool(target.get("today")) - reasons = {part for part in str(existing.get("reply_reason") or "").split(",") if part} - reasons.update(part for part in str(target.get("reply_reason") or "").split(",") if part) - existing["reply_reason"] = ",".join(sorted(reasons)) - if not existing.get("time_hint") and target.get("time_hint"): - existing["time_hint"] = target["time_hint"] - if not existing.get("url") and target.get("url"): - existing["url"] = target["url"] - existing["target_type"] = target.get("target_type") or existing.get("target_type") - if len(str(target.get("label") or "")) > len(str(existing.get("label") or "")): - existing["label"] = target["label"] - return list(deduped.values()) - -def click_point(ws_url: str, x: float, y: float) -> None: - for event_type in ("mouseMoved", "mousePressed", "mouseReleased"): - params: dict[str, Any] = {"type": event_type, "x": x, "y": y, "button": "left", "clickCount": 1} - if event_type == "mousePressed": - params["buttons"] = 1 - cdp_call(ws_url, "Input.dispatchMouseEvent", params) diff --git a/twitter-digest/scripts/dom_script_loader.py b/twitter-digest/scripts/dom_script_loader.py deleted file mode 100644 index 96262f6..0000000 --- a/twitter-digest/scripts/dom_script_loader.py +++ /dev/null @@ -1,15 +0,0 @@ -"""Load browser-side DOM extraction scripts.""" - -from __future__ import annotations - -from functools import lru_cache -from pathlib import Path - - -DOM_SCRIPT_DIR = Path(__file__).with_name("dom_scripts") - - -@lru_cache(maxsize=64) -def load_dom_script(name: str) -> str: - path = DOM_SCRIPT_DIR / name - return path.read_text(encoding="utf-8") diff --git a/twitter-digest/scripts/dom_scripts/README.md b/twitter-digest/scripts/dom_scripts/README.md deleted file mode 100644 index e7bffe0..0000000 --- a/twitter-digest/scripts/dom_scripts/README.md +++ /dev/null @@ -1,13 +0,0 @@ -# DOM Scripts - -These JavaScript snippets are loaded by `dom_script_loader.py` and executed in the X page through Chrome DevTools Protocol `Runtime.evaluate`. - -Keep DOM-specific heuristics here instead of embedding long JavaScript strings inside Python. This keeps the Python collectors focused on browser orchestration, retries, normalization, and output, while the JavaScript files own page-specific selectors, layout thresholds, and text extraction. - -Maintenance rules: - -- Name visual thresholds and timing/window constants before using them. -- Keep selectors and i18n text markers close to the script that uses them. -- Return plain JSON-compatible values only: objects, arrays, strings, numbers, booleans, or null. -- Treat X DOM attributes such as `data-testid` as hints, not the only source of truth. -- Prefer readable helper predicates over long inline DOM conditions. diff --git a/twitter-digest/scripts/dom_scripts/count_dm_messages.js b/twitter-digest/scripts/dom_scripts/count_dm_messages.js deleted file mode 100644 index cfd1b01..0000000 --- a/twitter-digest/scripts/dom_scripts/count_dm_messages.js +++ /dev/null @@ -1,42 +0,0 @@ -(() => { - const panel = document.querySelector('[data-testid="dm-conversation-panel"]') - || document.querySelector('[data-testid="dm-conversation-content"]') - || document.querySelector('main') - || document.body; - const roots = messageRoots(panel); - return roots.filter((node) => { - const bubble = node.querySelector('[data-testid^="message-text-"]'); - const text = bubbleText(bubble || node); - const rect = node.getBoundingClientRect(); - return Boolean(text) && rect.width > 20 && rect.height > 12; - }).length; - - function bubbleText(node) { - if (!node) return ''; - const parts = []; - for (const child of Array.from(node.querySelectorAll('span, div[dir="auto"]'))) { - const text = clean(child.innerText || ''); - if (!text || isTimeText(text)) continue; - const style = getComputedStyle(child); - if (style.opacity === '0' || style.visibility === 'hidden' || style.display === 'none') continue; - parts.push(text); - } - return Array.from(new Set(parts)).join(' ').trim(); - } - function clean(text) { return (text || '').replace(/\s+/g, ' ').trim(); } - function isTimeText(text) { return /^(\d{1,2}:\d{2}\s?(AM|PM)?|\d{1,2}:\d{2}|上午\s*\d{1,2}:\d{2}|下午\s*\d{1,2}:\d{2})$/i.test(text); } - function messageRoots(root) { - const byMessage = Array.from(root.querySelectorAll('div[data-testid^="message-"]')) - .filter((node) => !String(node.getAttribute('data-testid') || '').startsWith('message-text-')); - if (byMessage.length) return byMessage; - const byText = Array.from(root.querySelectorAll('[data-testid^="message-text-"]')) - .map((node) => node.closest('li, [role="group"]') || node.parentElement) - .filter(Boolean); - if (byText.length) return Array.from(new Set(byText)); - return Array.from(root.querySelectorAll('li, [role="group"]')).filter((node) => { - const text = clean(node.innerText || ''); - const rect = node.getBoundingClientRect(); - return text.length > 0 && text.length < 4000 && rect.width > 40 && rect.height > 16; - }); - } -})() diff --git a/twitter-digest/scripts/dom_scripts/detect_handle.js b/twitter-digest/scripts/dom_scripts/detect_handle.js deleted file mode 100644 index 27ef266..0000000 --- a/twitter-digest/scripts/dom_scripts/detect_handle.js +++ /dev/null @@ -1,75 +0,0 @@ -(() => { - const RESERVED_PATHS = new Set([ - 'home', 'explore', 'notifications', 'messages', 'i', 'search', 'settings', - 'compose', 'jobs', 'premium', 'verified-orgs', 'privacy', 'tos', - 'login', 'signup', 'logout', 'download', 'intent', 'share', 'hashtag', - ]); - const validHandle = (value) => /^[A-Za-z0-9_]{1,15}$/.test(value || '') && !RESERVED_PATHS.has(String(value).toLowerCase()); - const cleanHandle = (value) => { - const text = String(value || ''); - const match = text.match(/@([A-Za-z0-9_]{1,15})/); - return match && validHandle(match[1]) ? match[1] : null; - }; - const handleFromHref = (href) => { - if (!href) return null; - let parsed; - try { - parsed = new URL(href, location.href); - } catch (_) { - return null; - } - if (!/(^|\.)x\.com$|(^|\.)twitter\.com$/.test(parsed.hostname)) return null; - const parts = parsed.pathname.split('/').filter(Boolean); - if (parts.length !== 1) return null; - return validHandle(parts[0]) ? parts[0] : null; - }; - const fromText = (...values) => { - for (const value of values) { - const handle = cleanHandle(value); - if (handle) return handle; - } - return null; - }; - - const account = document.querySelector('[data-testid="SideNav_AccountSwitcher_Button"]'); - const accountText = account ? account.innerText : ''; - const accountMatch = fromText(accountText, account && account.getAttribute('aria-label')); - if (accountMatch) return accountMatch; - - const profileLink = document.querySelector('[data-testid="AppTabBar_Profile_Link"]'); - const profileHref = profileLink ? profileLink.getAttribute('href') : ''; - const profileHrefMatch = handleFromHref(profileHref); - if (profileHrefMatch) return profileHrefMatch; - const profileTextMatch = fromText(profileLink && profileLink.innerText, profileLink && profileLink.getAttribute('aria-label')); - if (profileTextMatch) return profileTextMatch; - - const accountLikeNodes = Array.from(document.querySelectorAll([ - '[data-testid*="Account"]', - '[data-testid*="account"]', - '[aria-label*="account" i]', - '[aria-label*="profile" i]', - '[aria-label*="账号"]', - '[aria-label*="帳號"]', - '[aria-label*="账户"]', - '[aria-label*="个人资料"]', - '[aria-label*="個人資料"]', - ].join(','))); - for (const node of accountLikeNodes) { - const handle = fromText(node.innerText, node.getAttribute('aria-label'), node.getAttribute('title')); - if (handle) return handle; - const linkHandle = handleFromHref(node.getAttribute('href') || ''); - if (linkHandle) return linkHandle; - } - - const nav = document.querySelector('nav') || document.body; - const navProfileLinks = Array.from(nav.querySelectorAll('a[href]')).filter((link) => { - const text = `${link.innerText || ''} ${link.getAttribute('aria-label') || ''}`; - return /profile|account|账号|帳號|账户|个人资料|個人資料/i.test(text); - }); - for (const link of navProfileLinks) { - const handle = handleFromHref(link.getAttribute('href') || '') || fromText(link.innerText, link.getAttribute('aria-label')); - if (handle) return handle; - } - - return null; -})() diff --git a/twitter-digest/scripts/dom_scripts/dm_loaded_beyond_window.js b/twitter-digest/scripts/dom_scripts/dm_loaded_beyond_window.js deleted file mode 100644 index 41cae01..0000000 --- a/twitter-digest/scripts/dom_scripts/dm_loaded_beyond_window.js +++ /dev/null @@ -1,107 +0,0 @@ -(() => { - const windowHours = Math.max(1, %d); - const oldest = oldestLoadedMessageAgeHours(); - return Number.isFinite(oldest) && oldest > windowHours; - - function oldestLoadedMessageAgeHours() { - const panel = document.querySelector('[data-testid="dm-conversation-panel"]') - || document.querySelector('[data-testid="dm-conversation-content"]') - || document.querySelector('main') - || document.body; - const list = panel.querySelector('[data-testid="dm-message-list"]') || panel; - const items = Array.from(list.querySelectorAll('li')); - let currentDay = ''; - let oldest = -Infinity; - for (const item of items) { - const text = clean(item.innerText || ''); - if (!text) continue; - const roots = messageRoots(item); - if (!roots.length) { - const day = dayLabel(text); - if (day) currentDay = day; - continue; - } - for (const root of roots) { - const timeText = firstTimeText(root); - const when = parseMessageDate(currentDay, timeText); - if (!when) continue; - const age = (Date.now() - when.getTime()) / 36e5; - if (age > oldest) oldest = age; - } - } - return oldest; - } - function dayLabel(text) { - const value = clean(text); - if (/^(today|yesterday|今天|昨天)$/i.test(value)) return value; - if (/^(jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)[a-z]*\.?\s+\d{1,2}/i.test(value)) return value; - if (/^\d{4}[\/-]\d{1,2}[\/-]\d{1,2}$/.test(value)) return value; - if (/^\d{1,2}[\/-]\d{1,2}(?:[\/-]\d{2,4})?$/.test(value)) return value; - return ''; - } - function parseMessageDate(day, timeText) { - const time = parseTime(timeText); - if (!time) return null; - const base = parseDay(day); - if (!base) return null; - base.setHours(time.hours, time.minutes, 0, 0); - return base; - } - function parseDay(day) { - const now = new Date(); - const value = clean(day).toLowerCase(); - if (!value || value === 'today' || value === '今天') return new Date(now.getFullYear(), now.getMonth(), now.getDate()); - if (value === 'yesterday' || value === '昨天') return new Date(now.getFullYear(), now.getMonth(), now.getDate() - 1); - const parsed = new Date(day); - if (!Number.isNaN(parsed.getTime())) return new Date(parsed.getFullYear(), parsed.getMonth(), parsed.getDate()); - return null; - } - function firstTimeText(node) { - for (const child of Array.from(node.querySelectorAll('span, div'))) { - const text = clean(child.innerText || ''); - if (isTimeText(text)) return text; - } - const match = clean(node.innerText || '').match(/(\d{1,2}:\d{2}\s?(?:AM|PM)?|上午\s*\d{1,2}:\d{2}|下午\s*\d{1,2}:\d{2})/i); - return match ? match[1] : ''; - } - function parseTime(text) { - const value = clean(text); - let match = value.match(/^(\d{1,2}):(\d{2})\s?(AM|PM)$/i); - if (match) { - let hours = Number(match[1]); - const minutes = Number(match[2]); - const suffix = match[3].toUpperCase(); - if (suffix === 'PM' && hours < 12) hours += 12; - if (suffix === 'AM' && hours === 12) hours = 0; - return {hours, minutes}; - } - match = value.match(/^(上午|下午)\s*(\d{1,2}):(\d{2})$/); - if (match) { - let hours = Number(match[2]); - const minutes = Number(match[3]); - if (match[1] === '下午' && hours < 12) hours += 12; - if (match[1] === '上午' && hours === 12) hours = 0; - return {hours, minutes}; - } - match = value.match(/^(\d{1,2}):(\d{2})$/); - if (match) return {hours: Number(match[1]), minutes: Number(match[2])}; - return null; - } - function clean(text) { return (text || '').replace(/\s+/g, ' ').trim(); } - function isTimeText(text) { return /^(\d{1,2}:\d{2}\s?(AM|PM)?|\d{1,2}:\d{2}|上午\s*\d{1,2}:\d{2}|下午\s*\d{1,2}:\d{2})$/i.test(text); } - function messageRoots(root) { - const byMessage = Array.from(root.querySelectorAll('div[data-testid^="message-"]')) - .filter((node) => !String(node.getAttribute('data-testid') || '').startsWith('message-text-')); - if (byMessage.length) return byMessage; - const byText = Array.from(root.querySelectorAll('[data-testid^="message-text-"]')) - .map((node) => node.closest('li, [role="group"]') || node.parentElement) - .filter(Boolean); - if (byText.length) return Array.from(new Set(byText)); - if (root.matches && root.matches('li, [role="group"]')) { - const text = clean(root.innerText || ''); - const rect = root.getBoundingClientRect(); - if (text.length > 0 && text.length < 4000 && rect.width > 40 && rect.height > 16) return [root]; - } - return []; - } -})() diff --git a/twitter-digest/scripts/dom_scripts/dm_page_loading_state.js b/twitter-digest/scripts/dom_scripts/dm_page_loading_state.js deleted file mode 100644 index 75a89f5..0000000 --- a/twitter-digest/scripts/dom_scripts/dm_page_loading_state.js +++ /dev/null @@ -1,45 +0,0 @@ -(() => { - const main = document.querySelector('main') || document.body; - const text = clean(main.innerText || ''); - const normalized = text.toLowerCase(); - const targets = document.querySelectorAll('a[href^="/messages/"], a[href^="/i/chat/"], a[href*="x.com/messages/"], a[href*="x.com/i/chat/"]').length; - const skeletons = skeletonBlocks(main); - const hasChatShell = /\bchat\b|\bmessages\b|聊天|私信/.test(normalized); - const hasStartConversation = /start conversation|choose from your existing conversations|new chat/.test(normalized); - const explicitEmpty = /welcome to your inbox|no messages/.test(normalized); - return { - text: text.slice(0, 1000), - has_chat_shell: hasChatShell, - has_start_conversation: hasStartConversation, - explicit_empty: explicitEmpty, - thread_target_count: targets, - skeleton_count: skeletons.length, - loading: hasChatShell && targets === 0 && !explicitEmpty && (skeletons.length >= 3 || hasStartConversation), - }; - - function skeletonBlocks(root) { - const out = []; - for (const el of Array.from(root.querySelectorAll('div, span'))) { - const rect = el.getBoundingClientRect(); - if (rect.width < 35 || rect.height < 8 || rect.width > 420 || rect.height > 120) continue; - if (rect.left < 70 || rect.left > Math.min(620, window.innerWidth * 0.55)) continue; - const text = clean(el.innerText || ''); - if (text.length > 0) continue; - const style = getComputedStyle(el); - if (style.display === 'none' || style.visibility === 'hidden' || style.opacity === '0') continue; - const bg = style.backgroundColor || ''; - const radius = parseFloat(style.borderRadius || '0') || 0; - const lightBlock = /rgb\((23[0-9]|24[0-9]|25[0-5]),\s*(23[0-9]|24[0-9]|25[0-5]),\s*(23[0-9]|24[0-9]|25[0-5])\)/.test(bg) - || /rgba\((23[0-9]|24[0-9]|25[0-5]),\s*(23[0-9]|24[0-9]|25[0-5]),\s*(23[0-9]|24[0-9]|25[0-5]),/.test(bg); - if (!lightBlock && radius < 6) continue; - out.push({ - x: Math.round(rect.left), - y: Math.round(rect.top), - width: Math.round(rect.width), - height: Math.round(rect.height), - }); - } - return out; - } - function clean(value) { return (value || '').replace(/\s+/g, ' ').trim(); } -})() diff --git a/twitter-digest/scripts/dom_scripts/extract_articles.js b/twitter-digest/scripts/dom_scripts/extract_articles.js deleted file mode 100644 index d1818fa..0000000 --- a/twitter-digest/scripts/dom_scripts/extract_articles.js +++ /dev/null @@ -1,104 +0,0 @@ -(() => { - const statusUrl = (href) => { - try { - const url = new URL(href, location.href); - return /\/status\/\d+/.test(url.pathname) ? url.href : null; - } catch { return null; } - }; - const clean = (text) => (text || '').replace(/\s+/g, ' ').trim(); - const normalizeUrl = (value) => { - if (!value || value.startsWith('data:') || value.startsWith('blob:')) return ''; - try { - const url = new URL(value, location.href); - url.hash = ''; - return url.href; - } catch { return ''; } - }; - const linkInfo = (article, links) => { - const out = []; - for (const a of Array.from(article.querySelectorAll('a[href]'))) { - const url = normalizeUrl(a.getAttribute('href')); - if (!url) continue; - let parsed; - try { parsed = new URL(url); } catch { continue; } - if (/\/photo\/\d+/.test(parsed.pathname)) continue; - if (/\/analytics$|\/retweets$|\/likes$/.test(parsed.pathname)) continue; - const label = clean(a.innerText || a.getAttribute('aria-label') || ''); - const isStatus = /\/status\/\d+/.test(parsed.pathname); - const sameStatus = isStatus && links.map(statusUrl).includes(url); - const isProfile = /^\/[^/]+$/.test(parsed.pathname) && (parsed.hostname.endsWith('x.com') || parsed.hostname.endsWith('twitter.com')); - if (sameStatus || isProfile) continue; - if (!out.some((item) => item.url === url)) out.push({url, label}); - } - return out.slice(0, 12); - }; - const mediaInfo = (article) => { - const out = []; - for (const img of Array.from(article.querySelectorAll('img[src]'))) { - const url = normalizeUrl(img.getAttribute('src')); - if (!url) continue; - if (/profile_images|emoji|hashflags|abs\.twimg\.com\/responsive-web/i.test(url)) continue; - const alt = clean(img.getAttribute('alt') || img.getAttribute('aria-label') || ''); - if (!out.some((item) => item.url === url)) out.push({type: 'image', url, alt}); - } - for (const video of Array.from(article.querySelectorAll('video'))) { - const url = normalizeUrl(video.currentSrc || video.getAttribute('src')); - const poster = normalizeUrl(video.getAttribute('poster')); - if (url || poster) out.push({type: 'video', url, poster, alt: clean(video.getAttribute('aria-label') || '')}); - } - return out.slice(0, 8); - }; - const cardInfo = (article) => { - const cards = []; - for (const link of Array.from(article.querySelectorAll('a[href]'))) { - const text = clean(link.innerText || link.getAttribute('aria-label') || ''); - const href = normalizeUrl(link.getAttribute('href')); - if (!href || text.length < 8) continue; - try { - const parsed = new URL(href); - const isProfile = /^\/[^/]+$/.test(parsed.pathname) && (parsed.hostname.endsWith('x.com') || parsed.hostname.endsWith('twitter.com')); - if (isProfile) continue; - if (/\/status\/\d+/.test(parsed.pathname) && text.length < 80) continue; - } catch { continue; } - if (!cards.some((item) => item.url === href && item.text === text)) cards.push({url: href, text: text.slice(0, 500)}); - } - return cards.slice(0, 5); - }; - return Array.from(document.querySelectorAll('article')).map((article) => { - const text = (article.innerText || '').trim(); - const links = Array.from(article.querySelectorAll('a[href]')).map(a => a.href).filter(Boolean); - const status = links.map(statusUrl).find(Boolean) || null; - let tweetId = null; - if (status) { - try { - const match = new URL(status).pathname.match(/\/status\/(\d+)/); - tweetId = match ? match[1] : null; - } catch {} - } - const times = Array.from(article.querySelectorAll('time')).map(t => t.getAttribute('datetime')).filter(Boolean); - const authorLinks = links.filter(h => { - try { - const p = new URL(h).pathname; - return /^\/[^/]+$/.test(p) && !p.includes('/i/'); - } catch { return false; } - }); - let authorUsername = null; - if (authorLinks[0]) { - try { - authorUsername = new URL(authorLinks[0]).pathname.split('/').filter(Boolean)[0] || null; - } catch {} - } - return { - id: tweetId, - text, - url: status, - links, - externalLinks: linkInfo(article, links), - media: mediaInfo(article), - cards: cardInfo(article), - time: times[0] || null, - authorUrl: authorLinks[0] || null, - authorUsername - }; - }).filter(item => item.text); -})() diff --git a/twitter-digest/scripts/dom_scripts/extract_dm_conversation_text.js b/twitter-digest/scripts/dom_scripts/extract_dm_conversation_text.js deleted file mode 100644 index 1487cc6..0000000 --- a/twitter-digest/scripts/dom_scripts/extract_dm_conversation_text.js +++ /dev/null @@ -1,4 +0,0 @@ -(() => { - const panel = document.querySelector('[data-testid="dm-conversation-panel"]') || document.querySelector('[data-testid="conversationPanel"]') || document.querySelector('main') || document.body; - return (panel.innerText || '').trim().slice(0, 12000); -})() diff --git a/twitter-digest/scripts/dom_scripts/extract_dm_messages.js b/twitter-digest/scripts/dom_scripts/extract_dm_messages.js deleted file mode 100644 index 39dad55..0000000 --- a/twitter-digest/scripts/dom_scripts/extract_dm_messages.js +++ /dev/null @@ -1,109 +0,0 @@ -(() => { - const panel = document.querySelector('[data-testid="dm-conversation-panel"]') - || document.querySelector('[data-testid="dm-conversation-content"]') - || document.querySelector('main') - || document.body; - const roots = messageRoots(panel); - const out = []; - const seen = new Set(); - for (const node of roots) { - const bubble = node.querySelector('[data-testid^="message-text-"]'); - const text = bubbleText(bubble || node); - if (!text) continue; - const rect = node.getBoundingClientRect(); - if (rect.width < 20 || rect.height < 12) continue; - const key = `${Math.round(rect.top)}:${text.slice(0, 120)}`; - if (seen.has(key)) continue; - seen.add(key); - const time = firstTimeText(bubble || node); - const classText = String(node.className || ''); - const assets = messageAssets(node); - out.push({ - sender: classText.includes('justify-end') ? 'me' : 'other', - time, - text, - links: assets.links, - media: assets.media, - }); - } - return out.slice(-Math.max(1, %d)); - - function bubbleText(node) { - if (!node) return ''; - const leafParts = []; - for (const child of Array.from(node.querySelectorAll('span'))) { - const text = clean(child.innerText || ''); - if (!text || isTimeText(text)) continue; - const style = getComputedStyle(child); - if (style.opacity === '0' || style.visibility === 'hidden' || style.display === 'none') continue; - leafParts.push(text); - } - if (leafParts.length) return Array.from(new Set(leafParts)).join(' ').trim(); - const text = clean(node.innerText || ''); - return stripTrailingTimes(text); - } - function firstTimeText(node) { - if (!node) return ''; - for (const child of Array.from(node.querySelectorAll('span, div'))) { - const text = clean(child.innerText || ''); - if (isTimeText(text)) return text; - } - const match = clean(node.innerText || '').match(/(\d{1,2}:\d{2}\s?(?:AM|PM)?|上午\s*\d{1,2}:\d{2}|下午\s*\d{1,2}:\d{2})/i); - return match ? match[1] : ''; - } - function stripTrailingTimes(text) { - let value = clean(text); - for (let i = 0; i < 3; i += 1) { - value = value.replace(/\s+(\d{1,2}:\d{2}\s?(?:AM|PM)?|上午\s*\d{1,2}:\d{2}|下午\s*\d{1,2}:\d{2})$/i, '').trim(); - } - return value; - } - function clean(text) { return (text || '').replace(/\s+/g, ' ').trim(); } - function isTimeText(text) { return /^(\d{1,2}:\d{2}\s?(AM|PM)?|\d{1,2}:\d{2}|上午\s*\d{1,2}:\d{2}|下午\s*\d{1,2}:\d{2})$/i.test(text); } - function messageAssets(node) { - const links = []; - for (const a of Array.from(node.querySelectorAll('a[href]'))) { - const href = normalizeUrl(a.getAttribute('href')); - if (!href) continue; - const label = clean(a.innerText || a.getAttribute('aria-label') || ''); - if (!links.some((item) => item.url === href)) links.push({url: href, label}); - } - const media = []; - for (const img of Array.from(node.querySelectorAll('img[src]'))) { - const src = normalizeUrl(img.getAttribute('src')); - if (!src) continue; - const alt = clean(img.getAttribute('alt') || img.getAttribute('aria-label') || ''); - if (!media.some((item) => item.url === src)) media.push({type: 'image', url: src, alt}); - } - for (const video of Array.from(node.querySelectorAll('video'))) { - const src = normalizeUrl(video.currentSrc || video.getAttribute('src')); - const poster = normalizeUrl(video.getAttribute('poster')); - if (src || poster) media.push({type: 'video', url: src || '', poster: poster || '', alt: clean(video.getAttribute('aria-label') || '')}); - } - return {links: links.slice(0, 10), media: media.slice(0, 8)}; - } - function normalizeUrl(value) { - if (!value || value.startsWith('data:') || value.startsWith('blob:')) return ''; - try { - const url = new URL(value, location.href); - url.hash = ''; - return url.href; - } catch { - return ''; - } - } - function messageRoots(root) { - const byMessage = Array.from(root.querySelectorAll('div[data-testid^="message-"]')) - .filter((node) => !String(node.getAttribute('data-testid') || '').startsWith('message-text-')); - if (byMessage.length) return byMessage; - const byText = Array.from(root.querySelectorAll('[data-testid^="message-text-"]')) - .map((node) => node.closest('li, [role="group"]') || node.parentElement) - .filter(Boolean); - if (byText.length) return Array.from(new Set(byText)); - return Array.from(root.querySelectorAll('li, [role="group"]')).filter((node) => { - const text = clean(node.innerText || ''); - const rect = node.getBoundingClientRect(); - return text.length > 0 && text.length < 4000 && rect.width > 40 && rect.height > 16; - }); - } -})() diff --git a/twitter-digest/scripts/dom_scripts/extract_dm_thread_targets.js b/twitter-digest/scripts/dom_scripts/extract_dm_thread_targets.js deleted file mode 100644 index 8a116b3..0000000 --- a/twitter-digest/scripts/dom_scripts/extract_dm_thread_targets.js +++ /dev/null @@ -1,155 +0,0 @@ -(() => { - const MIN_VISIBLE_WIDTH_PX = 40; - const MIN_VISIBLE_HEIGHT_PX = 24; - const CLICK_X_MAX_OFFSET_PX = 280; - const MAX_LABEL_LENGTH = 600; - const DEDUPE_LABEL_PREFIX_LENGTH = 80; - const LEFT_NAV_CUTOFF_PX = 150; - const MAX_NON_LINK_ROW_HEIGHT_PX = 220; - const LIST_COLUMN_MAX_WIDTH_PX = 760; - const LIST_COLUMN_WIDTH_RATIO = 0.45; - const MIN_LIST_ROW_WIDTH_PX = 160; - const MIN_LIST_ROW_HEIGHT_PX = 36; - const MAX_LIST_ROW_HEIGHT_PX = 180; - const MAX_THREAD_TARGETS = 20; - const SUPPORTED_ROW_SELECTOR = [ - '[role="button"]', - '[role="link"]', - '[data-testid*="conversation" i]', - '[data-testid*="cell" i]', - '[data-testid="cellInnerDiv"]', - 'a[href*="/messages/"]', - 'div[aria-label]' - ].join(','); - const SELF_REPLY_RE = /\byou\s*[:：]|\byou sent\b|\byou replied\b|\byou responded\b|你\s*[:：]|你已发送|你发送|您\s*[:：]/i; - const RECENT_TIME_RE = /\b(now|just now|\d+\s*(m|min|mins|minute|minutes|h|hr|hrs|hour|hours))\b/i; - const RECENT_TIME_CN_RE = /(刚刚|\d+\s*(秒|分钟|小时)|今天|今日|上午|下午|晚上|中午)/; - const seen = new Set(); - const ignoredText = /^(messages|new message|message requests|search direct messages|search|settings|home|profile|notifications)$/i; - const ignoredShortText = /^(all|chat)$/i; - const visible = (el) => { - const rect = el.getBoundingClientRect(); - const style = getComputedStyle(el); - return rect.width > MIN_VISIBLE_WIDTH_PX && rect.height > MIN_VISIBLE_HEIGHT_PX && style.visibility !== 'hidden' && style.display !== 'none'; - }; - const clean = (text) => (text || '').replace(/\s+/g, ' ').trim(); - const clickX = (rect) => rect.left + Math.min(rect.width / 2, CLICK_X_MAX_OFFSET_PX); - const out = []; - - const replyMeta = (el, label) => { - const text = clean(label).toLowerCase(); - const aria = clean(el.getAttribute('aria-label') || '').toLowerCase(); - const combined = `${text} ${aria}`; - const reasons = []; - if (SELF_REPLY_RE.test(combined)) { - reasons.push('self_reply_label'); - } - return { replied: reasons.length > 0, reply_reason: reasons.join(',') }; - }; - const timeMeta = (el) => { - const parts = []; - for (const node of Array.from(el.querySelectorAll('time'))) { - parts.push(clean(node.getAttribute('datetime') || '')); - parts.push(clean(node.getAttribute('title') || '')); - parts.push(clean(node.getAttribute('aria-label') || '')); - parts.push(clean(node.innerText || '')); - } - parts.push(clean(el.getAttribute('title') || '')); - parts.push(clean(el.getAttribute('aria-label') || '')); - return parts.filter(Boolean).join(' '); - }; - - for (const a of document.querySelectorAll('a[href^="/messages/"], a[href^="/i/chat/"], a[href*="x.com/messages/"], a[href*="x.com/i/chat/"]')) { - if (!visible(a)) continue; - const url = new URL(a.getAttribute('href'), location.href); - url.search = ''; - url.hash = ''; - const label = clean(a.innerText || a.getAttribute('aria-label') || ''); - if (!/^https:\/\/(x|twitter)\.com\/(messages\/[^/]+|i\/chat\/[^/]+)/.test(url.href)) continue; - if (/\/messages\/compose$/.test(url.pathname)) continue; - if (!label || ignoredText.test(label) || ignoredShortText.test(label)) continue; - const key = url.href; - if (seen.has(key)) continue; - seen.add(key); - const meta = replyMeta(a, label); - const rect = a.getBoundingClientRect(); - out.push({ - target_type: 'link', - url: url.href, - label, - time_hint: timeMeta(a), - x: clickX(rect), - y: rect.top + rect.height / 2, - ...meta - }); - } - - const hasThreadMarker = (label) => ( - SELF_REPLY_RE.test(label) - || RECENT_TIME_RE.test(label) - || RECENT_TIME_CN_RE.test(label) - ); - const isSupportedInteractiveNode = (node) => node.matches(SUPPORTED_ROW_SELECTOR); - const looksLikeThreadListRow = (rect, label) => ( - rect.left < Math.min(LIST_COLUMN_MAX_WIDTH_PX, window.innerWidth * LIST_COLUMN_WIDTH_RATIO) - && rect.width > MIN_LIST_ROW_WIDTH_PX - && rect.height >= MIN_LIST_ROW_HEIGHT_PX - && rect.height < MAX_LIST_ROW_HEIGHT_PX - && hasThreadMarker(label) - ); - - const candidates = Array.from(document.querySelectorAll([ - '[role="button"]', - '[role="link"]', - '[data-testid*="conversation" i]', - '[data-testid*="cell" i]', - '[data-testid="cellInnerDiv"]', - 'a[href*="/messages/"]', - 'div[aria-label]', - 'section div', - 'aside div' - ].join(','))); - for (const node of candidates) { - if (!visible(node)) continue; - let label = clean(node.innerText || node.getAttribute('aria-label') || ''); - if (!label || label.length < 2 || ignoredText.test(label) || ignoredShortText.test(label)) continue; - if (label.length > MAX_LABEL_LENGTH) continue; - if (!/[A-Za-z0-9_\u4e00-\u9fff]/.test(label)) continue; - let rect = node.getBoundingClientRect(); - const link = node.querySelector && node.querySelector('a[href*="/messages/"], a[href*="/i/chat/"]'); - if (link) { - const linkLabel = clean(link.innerText || link.getAttribute('aria-label') || ''); - const linkRect = link.getBoundingClientRect(); - if (linkLabel && hasThreadMarker(linkLabel) && linkRect.width > MIN_VISIBLE_WIDTH_PX && linkRect.height > MIN_VISIBLE_HEIGHT_PX) { - label = linkLabel; - rect = linkRect; - } - } - if (!link && rect.left < LEFT_NAV_CUTOFF_PX) continue; - if (!link && rect.height > MAX_NON_LINK_ROW_HEIGHT_PX) continue; - const isLikelyListRow = looksLikeThreadListRow(rect, label); - if (!link && !hasThreadMarker(label)) continue; - if (!isSupportedInteractiveNode(node) && !isLikelyListRow) continue; - let url = ''; - if (link) { - const parsed = new URL(link.getAttribute('href'), location.href); - parsed.search = ''; - parsed.hash = ''; - url = parsed.href; - } - const key = url || `${Math.round(rect.left)}:${Math.round(rect.top)}:${label.slice(0, DEDUPE_LABEL_PREFIX_LENGTH)}`; - if (seen.has(key)) continue; - seen.add(key); - const meta = replyMeta(node, label); - out.push({ - target_type: url ? 'row_link' : 'row_click', - url, - label, - time_hint: timeMeta(node), - x: clickX(rect), - y: rect.top + rect.height / 2, - ...meta - }); - } - return out.slice(0, MAX_THREAD_TARGETS); -})() diff --git a/twitter-digest/scripts/dom_scripts/extract_main_text.js b/twitter-digest/scripts/dom_scripts/extract_main_text.js deleted file mode 100644 index 0d15d52..0000000 --- a/twitter-digest/scripts/dom_scripts/extract_main_text.js +++ /dev/null @@ -1,4 +0,0 @@ -(() => { - const main = document.querySelector('main') || document.body; - return (main.innerText || '').trim().slice(0, 12000); -})() diff --git a/twitter-digest/scripts/dom_scripts/get_dm_history_state.js b/twitter-digest/scripts/dom_scripts/get_dm_history_state.js deleted file mode 100644 index 6dc796f..0000000 --- a/twitter-digest/scripts/dom_scripts/get_dm_history_state.js +++ /dev/null @@ -1,43 +0,0 @@ -(() => { - const panel = document.querySelector('[data-testid="dm-conversation-panel"]') - || document.querySelector('[data-testid="dm-conversation-content"]') - || document.querySelector('main') - || document.body; - const scroller = findScroller(panel); - const roots = messageRoots(panel); - const first = roots[0]; - return { - count: roots.length, - at_top: scroller ? scroller.scrollTop <= 2 : window.scrollY <= 0, - scroll_top: scroller ? scroller.scrollTop : window.scrollY, - top_signature: first ? signature(first) : '', - }; - - function findScroller(root) { - const candidates = [root, ...Array.from(root.querySelectorAll('*'))].filter((el) => { - const rect = el.getBoundingClientRect(); - return rect.width > 200 && rect.height > 200 && el.scrollHeight > el.clientHeight + 40; - }); - return candidates.sort((a, b) => (b.scrollHeight - b.clientHeight) - (a.scrollHeight - a.clientHeight))[0] || null; - } - function signature(node) { - const text = (node.innerText || '').replace(/\s+/g, ' ').trim(); - const testid = node.getAttribute('data-testid') || ''; - const rect = node.getBoundingClientRect(); - return `${testid}:${Math.round(rect.top)}:${text.slice(0, 160)}`; - } - function messageRoots(root) { - const byMessage = Array.from(root.querySelectorAll('div[data-testid^="message-"]')) - .filter((node) => !String(node.getAttribute('data-testid') || '').startsWith('message-text-')); - if (byMessage.length) return byMessage; - const byText = Array.from(root.querySelectorAll('[data-testid^="message-text-"]')) - .map((node) => node.closest('li, [role="group"]') || node.parentElement) - .filter(Boolean); - if (byText.length) return Array.from(new Set(byText)); - return Array.from(root.querySelectorAll('li, [role="group"]')).filter((node) => { - const text = (node.innerText || '').replace(/\s+/g, ' ').trim(); - const rect = node.getBoundingClientRect(); - return text.length > 0 && text.length < 4000 && rect.width > 40 && rect.height > 16; - }); - } -})() diff --git a/twitter-digest/scripts/dom_scripts/scroll_dm_messages_up.js b/twitter-digest/scripts/dom_scripts/scroll_dm_messages_up.js deleted file mode 100644 index 48187c5..0000000 --- a/twitter-digest/scripts/dom_scripts/scroll_dm_messages_up.js +++ /dev/null @@ -1,19 +0,0 @@ -(() => { - const panel = document.querySelector('[data-testid="dm-conversation-panel"]') - || document.querySelector('[data-testid="dm-conversation-content"]') - || document.querySelector('main') - || document.body; - const candidates = [panel, ...Array.from(panel.querySelectorAll('*'))].filter((el) => { - const rect = el.getBoundingClientRect(); - return rect.width > 200 && rect.height > 200 && el.scrollHeight > el.clientHeight + 40; - }); - const scroller = candidates.sort((a, b) => (b.scrollHeight - b.clientHeight) - (a.scrollHeight - a.clientHeight))[0]; - if (!scroller) { - window.scrollBy(0, -Math.max(900, window.innerHeight * 0.9)); - return {found: false, at_top: window.scrollY <= 0, scroll_top: window.scrollY}; - } - const before = scroller.scrollTop; - scroller.scrollTop = Math.max(0, before - Math.max(900, scroller.clientHeight * 0.9)); - scroller.dispatchEvent(new Event('scroll', {bubbles: true})); - return {found: true, at_top: scroller.scrollTop <= 0, scroll_top: scroller.scrollTop, before}; -})() diff --git a/twitter-digest/scripts/dom_scripts/scroll_dm_thread_list_down.js b/twitter-digest/scripts/dom_scripts/scroll_dm_thread_list_down.js deleted file mode 100644 index c974c0a..0000000 --- a/twitter-digest/scripts/dom_scripts/scroll_dm_thread_list_down.js +++ /dev/null @@ -1,41 +0,0 @@ -(() => { - const scroller = findThreadListScroller(); - if (!scroller) { - const before = window.scrollY; - window.scrollBy(0, Math.max(700, window.innerHeight * 0.8)); - return {found: false, before, after: window.scrollY, at_bottom: window.innerHeight + window.scrollY >= document.body.scrollHeight - 4}; - } - const before = scroller.scrollTop; - const step = Math.max(500, scroller.clientHeight * 0.85); - scroller.scrollTop = Math.min(scroller.scrollHeight - scroller.clientHeight, before + step); - scroller.dispatchEvent(new Event('scroll', {bubbles: true})); - return { - found: true, - before, - after: scroller.scrollTop, - at_bottom: scroller.scrollTop + scroller.clientHeight >= scroller.scrollHeight - 4, - scroll_height: scroller.scrollHeight, - client_height: scroller.clientHeight, - }; - - function findThreadListScroller() { - const candidates = Array.from(document.querySelectorAll('main, aside, section, div')).filter((el) => { - const rect = el.getBoundingClientRect(); - if (rect.width < 220 || rect.height < 220) return false; - if (rect.left > Math.min(820, window.innerWidth * 0.55)) return false; - if (el.scrollHeight <= el.clientHeight + 40) return false; - const text = clean(el.innerText || ''); - const hasThreadLink = !!el.querySelector('a[href^="/messages/"], a[href^="/i/chat/"], a[href*="x.com/messages/"], a[href*="x.com/i/chat/"]'); - const looksLikeInbox = /\b(chat|messages|search)\b/i.test(text) || /(聊天|私信|搜索)/.test(text); - return hasThreadLink || looksLikeInbox; - }); - return candidates.sort((a, b) => { - const ar = a.getBoundingClientRect(); - const br = b.getBoundingClientRect(); - const aScore = (a.scrollHeight - a.clientHeight) - ar.left; - const bScore = (b.scrollHeight - b.clientHeight) - br.left; - return bScore - aScore; - })[0] || null; - } - function clean(text) { return (text || '').replace(/\s+/g, ' ').trim(); } -})() diff --git a/twitter-digest/scripts/install.py b/twitter-digest/scripts/install.py index 2fbde9b..541a7a6 100644 --- a/twitter-digest/scripts/install.py +++ b/twitter-digest/scripts/install.py @@ -7,7 +7,6 @@ import datetime as dt import json import os -import shutil as shutil_module import shutil import sys from pathlib import Path @@ -42,7 +41,6 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--skills-dir", default="", help="Override the target skills directory.") parser.add_argument("--copy", action="store_true", help="Copy files instead of creating a symlink. This is the default.") parser.add_argument("--symlink", action="store_true", help="Install as a symlink for local skill development.") - parser.add_argument("--skip-browser-check", action="store_true", help="Skip checking for a supported Chromium browser.") parser.add_argument( "--allow-claude-commands", action="store_true", @@ -67,45 +65,9 @@ def skill_root() -> Path: return Path(__file__).resolve().parents[1] -def browser_candidates() -> list[str]: - return [ - "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome", - "/Applications/Chromium.app/Contents/MacOS/Chromium", - "/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge", - "/Applications/Brave Browser.app/Contents/MacOS/Brave Browser", - "google-chrome", - "google-chrome-stable", - "chromium", - "chromium-browser", - "microsoft-edge", - "brave-browser", - ] - - -def find_supported_browser() -> Optional[str]: - for candidate in browser_candidates(): - path = Path(candidate).expanduser() - if path.is_absolute() and path.exists(): - return str(path) - resolved = shutil_module.which(candidate) - if resolved: - return resolved - return None - - -def check_runtime(skip_browser_check: bool) -> None: +def check_runtime() -> None: if sys.version_info < (3, 9): raise SystemExit("Python 3.9+ is required to run twitter-digest. Install or switch python3, then rerun this installer.") - if skip_browser_check: - print("Skipped browser check. Runtime still requires Chrome, Chromium, Edge, or Brave.", flush=True) - return - browser = find_supported_browser() - if not browser: - raise SystemExit( - "No supported Chromium browser found. Install one of: Google Chrome, Chromium, Microsoft Edge, or Brave. " - "Then rerun the installer. Use --skip-browser-check only if the browser will be installed later." - ) - print(f"Supported browser found: {display_path(Path(browser))}", flush=True) def backup_path(skills_dir: Path, name: str) -> Path: @@ -284,7 +246,7 @@ def write_claude_settings(target: Path, dry_run: bool, allow_commands: bool, all def main() -> None: args = parse_args() - check_runtime(args.skip_browser_check) + check_runtime() root = skill_root() copy = args.copy or not args.symlink client = detect_client() if args.client == "auto" else args.client diff --git a/twitter-digest/scripts/public_scraper.py b/twitter-digest/scripts/public_scraper.py deleted file mode 100644 index b2eb008..0000000 --- a/twitter-digest/scripts/public_scraper.py +++ /dev/null @@ -1,160 +0,0 @@ -"""Public X page scraping helpers. - -Module map: -- Readiness: wait until the public page has article nodes or meaningful text. -- Scrolling: load timeline/profile/search content through the 24-hour window. -- Extraction: run DOM scripts and normalize/dedupe public post dictionaries. -""" - -from __future__ import annotations - -import datetime as dt -import time -from typing import Any, Optional - -from cdp_client import cdp_call, cdp_eval, wait_for_cdp_page_ws -from dom_script_loader import load_dom_script - - -PUBLIC_READY_MIN_TEXT_LENGTH = 80 -PUBLIC_READY_POLL_SEC = 0.5 -PUBLIC_SCROLL_STAGNANT_LIMIT = 6 -PUBLIC_INITIAL_GROWTH_TIMEOUT_SEC = 6 -PUBLIC_LATER_GROWTH_TIMEOUT_SEC = 4 -PUBLIC_INITIAL_GROWTH_ROUNDS = 3 -PUBLIC_GROWTH_READY_SETTLE_SEC = 0.4 -PUBLIC_GROWTH_POLL_SEC = 0.5 -PUBLIC_GROWTH_STABLE_TICKS = 3 -PUBLIC_SCROLL_JS = "window.scrollBy(0, Math.max(900, window.innerHeight * 0.9));" -HANDLE_DETECT_TIMEOUT_SEC = 20 -HANDLE_DETECT_POLL_SEC = 1 - - -def wait_for_public_page_ready(ws_url: str, timeout_sec: int = 20) -> None: - deadline = time.time() + timeout_sec - while time.time() < deadline: - if extract_articles(ws_url): - return - if len(extract_main_text(ws_url)) > PUBLIC_READY_MIN_TEXT_LENGTH: - return - time.sleep(PUBLIC_READY_POLL_SEC) - -def collect_public_items(ws_url: str, max_scrolls: int, max_items: int, window_hours: int, min_scrolls: int = 5) -> dict[str, Any]: - posts: list[dict[str, Any]] = [] - scroll_limit = max(1, int(max_scrolls)) - item_limit = max(1, int(max_items)) - window = max(1, int(window_hours)) - minimum_scrolls = min(scroll_limit, max(0, int(min_scrolls))) - stagnant_rounds = 0 - previous_count = 0 - window_exceeded = False - scrolls_used = 0 - - for scroll_index in range(scroll_limit): - scrolls_used = scroll_index + 1 - posts = dedupe_items([*posts, *extract_articles(ws_url)]) - if len(posts) >= item_limit: - break - window_exceeded = public_posts_beyond_window(posts, window) - if window_exceeded and scrolls_used >= minimum_scrolls: - break - if len(posts) <= previous_count: - stagnant_rounds += 1 - else: - stagnant_rounds = 0 - if stagnant_rounds >= PUBLIC_SCROLL_STAGNANT_LIMIT and scrolls_used >= minimum_scrolls: - break - previous_count = len(posts) - cdp_eval(ws_url, PUBLIC_SCROLL_JS) - timeout_sec = PUBLIC_INITIAL_GROWTH_TIMEOUT_SEC if scroll_index < PUBLIC_INITIAL_GROWTH_ROUNDS else PUBLIC_LATER_GROWTH_TIMEOUT_SEC - wait_for_public_growth(ws_url, previous_count, timeout_sec=timeout_sec) - - posts = dedupe_items([*posts, *extract_articles(ws_url)]) - if len(posts) > item_limit: - posts = posts[:item_limit] - window_exceeded = public_posts_beyond_window(posts, window) - return { - "items": posts, - "public_scrolls_used": min(scroll_limit, scrolls_used), - "public_window_exceeded": window_exceeded, - "public_max_items": item_limit, - "public_window_hours": window, - "public_min_scrolls": minimum_scrolls, - "public_stagnant_rounds": stagnant_rounds, - } - -def wait_for_public_growth(ws_url: str, previous_count: int, timeout_sec: int = 4) -> None: - deadline = time.time() + max(1, timeout_sec) - last_count = 0 - stable_ticks = 0 - while time.time() < deadline: - current_count = len(extract_articles(ws_url)) - if current_count > previous_count: - time.sleep(PUBLIC_GROWTH_READY_SETTLE_SEC) - return - if current_count == last_count: - stable_ticks += 1 - else: - stable_ticks = 0 - if stable_ticks >= PUBLIC_GROWTH_STABLE_TICKS and current_count > 0: - return - last_count = current_count - time.sleep(PUBLIC_GROWTH_POLL_SEC) - -def public_posts_beyond_window(posts: list[dict[str, Any]], window_hours: int) -> bool: - now = dt.datetime.now(dt.timezone.utc) - oldest_age = 0.0 - for post in posts: - timestamp = str(post.get("time") or "") - if not timestamp: - continue - try: - parsed = dt.datetime.fromisoformat(timestamp.replace("Z", "+00:00")) - except ValueError: - continue - if parsed.tzinfo is None: - parsed = parsed.replace(tzinfo=dt.timezone.utc) - age = (now - parsed.astimezone(dt.timezone.utc)).total_seconds() / 3600 - oldest_age = max(oldest_age, age) - return oldest_age > max(1, int(window_hours)) - -def extract_articles(ws_url: str) -> list[dict[str, Any]]: - script = load_dom_script("extract_articles.js") - value = cdp_eval(ws_url, script) - return value if isinstance(value, list) else [] - -def extract_main_text(ws_url: str) -> str: - script = load_dom_script("extract_main_text.js") - value = cdp_eval(ws_url, script) - return str(value or "") - -def detect_handle(port: int) -> Optional[str]: - try: - ws_url = wait_for_cdp_page_ws(port) - cdp_call(ws_url, "Page.enable") - cdp_call(ws_url, "Runtime.enable") - script = load_dom_script("detect_handle.js") - for url in ("https://x.com/home", "https://x.com/notifications"): - cdp_call(ws_url, "Page.navigate", {"url": url}) - deadline = time.time() + HANDLE_DETECT_TIMEOUT_SEC - while time.time() < deadline: - value = cdp_eval(ws_url, script) - handle = str(value or "").lstrip("@").strip() - if handle: - return handle - time.sleep(HANDLE_DETECT_POLL_SEC) - return None - except Exception as exc: - print(f"Could not auto-detect X handle: {exc}") - return None - -def dedupe_items(items: list[dict[str, Any]]) -> list[dict[str, Any]]: - seen: set[str] = set() - out: list[dict[str, Any]] = [] - for item in items: - key = str(item.get("url") or item.get("text") or "")[:500] - if not key or key in seen: - continue - seen.add(key) - out.append(item) - return out diff --git a/twitter-digest/scripts/run_daily_digest.py b/twitter-digest/scripts/run_daily_digest.py index 5fcf58e..e214198 100644 --- a/twitter-digest/scripts/run_daily_digest.py +++ b/twitter-digest/scripts/run_daily_digest.py @@ -12,7 +12,7 @@ from typing import Optional from api_config_store import load_api_config, refresh_oauth_token_if_needed -from collector_commands import api_collector_command, browser_collector_command, summarize_collector_error +from collector_commands import api_collector_command, summarize_collector_error from digest_context import build_current_context_from_file from script_utils import display_path, open_script_in_terminal, rerun_from_installed_if_needed @@ -21,7 +21,7 @@ CONFIG_PATH = STATE_DIR / "config.json" DEFAULT_OUT_DIR = STATE_DIR / "run" DEFAULT_API_PUBLIC_ITEMS = 300 -DEFAULT_BROWSER_PUBLIC_ITEMS = 100 +UNSUPPORTED_OPTION_MESSAGE = "Source selection is no longer supported. twitter-digest uses API only." def parse_args() -> argparse.Namespace: @@ -30,35 +30,39 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--account-name") parser.add_argument("--save-default", action="store_true", help="Save --handle/--account-name as the default account for future chat runs.") parser.add_argument("--configure-only", action="store_true", help="Only save default account config; do not collect data.") - parser.add_argument("--keywords", default="", help="Optional comma-separated search queries. Default is empty; the daily digest focuses on timeline, mentions, and DMs.") + parser.add_argument("--keywords", default="", help="Optional comma-separated search queries. Default is empty; the daily digest focuses on timeline and mentions.") parser.add_argument("--out", default=str(DEFAULT_OUT_DIR)) - parser.add_argument("--source", choices=("auto", "browser", "api"), default="auto", help="Data collection source. Default auto uses API, configuring or reconfiguring it when needed. Use browser to force local browser collection.") - parser.add_argument("--configure-api", action="store_true", help="Open a secure prompt to save X API credentials, then exit.") + parser.add_argument("--configure-api", action="store_true", help="Open a secure prompt to save required X API credentials, then exit.") parser.add_argument("--configure-api-token", action="store_true", help="Open a secure prompt to paste an existing X user access token, then exit.") parser.add_argument("--api-base", default=os.environ.get("X_API_BASE_URL") or "") parser.add_argument("--user-id", default=os.environ.get("X_USER_ID") or os.environ.get("TWITTER_USER_ID") or "") parser.add_argument("--bearer-token", default=os.environ.get("X_BEARER_TOKEN") or os.environ.get("TWITTER_BEARER_TOKEN") or "") - parser.add_argument("--include-dms", action="store_true", help="Include visible DMs. This is already the default; kept for compatibility.") - parser.add_argument("--no-dms", action="store_true", help="Skip X Messages collection for this run.") - parser.add_argument("--dm-threads", type=int, default=5) - parser.add_argument("--dm-list-scrolls", type=int, default=20, help="Maximum downward scroll rounds used to scan today's DM conversation list.") - parser.add_argument("--dm-scrolls", type=int, default=200, help="Maximum upward scroll rounds per opened DM thread.") - parser.add_argument("--dm-max-messages", type=int, default=2000, help="Maximum message bubbles kept per opened DM thread.") + parser.add_argument("--include-dms", action="store_true", help=argparse.SUPPRESS) + parser.add_argument("--no-dms", action="store_true", help=argparse.SUPPRESS) + parser.add_argument("--dm-threads", type=int, default=5, help=argparse.SUPPRESS) + parser.add_argument("--dm-list-scrolls", type=int, default=20, help=argparse.SUPPRESS) + parser.add_argument("--dm-scrolls", type=int, default=200, help=argparse.SUPPRESS) + parser.add_argument("--dm-max-messages", type=int, default=2000, help=argparse.SUPPRESS) parser.add_argument("--dm-max-events", type=int, default=300, help="Maximum Direct Message API events kept per run.") - parser.add_argument("--dm-window-hours", type=int, default=0, help="Stop loading older DM history once messages beyond this window are detected. 0 means load full available thread history.") - parser.add_argument("--scrolls", type=int, default=40, help="Maximum scroll rounds per public page.") - parser.add_argument("--min-public-scrolls", type=int, default=5, help="Minimum public-page scroll rounds before early stop rules can end collection.") + parser.add_argument("--dm-window-hours", type=int, default=0, help=argparse.SUPPRESS) + parser.add_argument("--scrolls", type=int, default=40, help=argparse.SUPPRESS) + parser.add_argument("--min-public-scrolls", type=int, default=5, help=argparse.SUPPRESS) parser.add_argument( "--max-public-items", type=int, default=None, - help="Override maximum public post items for both collectors. Defaults: API 300, browser 100.", + help="Override maximum public post items for API collection. Default: API 300.", ) parser.add_argument("--public-window-hours", type=int, default=24, help="Stop loading older public timeline items once posts beyond this window are detected.") - parser.add_argument("--headless", action="store_true", help="Run browser collection headlessly. This is the default when login is already saved.") - parser.add_argument("--headed", action="store_true", help="Force a visible browser window for debugging or manual login.") - parser.add_argument("--non-interactive", action="store_true", help="Do not open a visible browser for DM passcode recovery; record a data gap instead.") - return parser.parse_args() + parser.add_argument("--headless", action="store_true", help=argparse.SUPPRESS) + parser.add_argument("--headed", action="store_true", help=argparse.SUPPRESS) + parser.add_argument("--non-interactive", action="store_true", help=argparse.SUPPRESS) + args, unknown = parser.parse_known_args() + if "--source" in unknown or any(arg.startswith("--source=") for arg in unknown): + raise SystemExit(UNSUPPORTED_OPTION_MESSAGE) + if unknown: + parser.error(f"unrecognized arguments: {' '.join(unknown)}") + return args def load_config() -> dict: @@ -88,7 +92,7 @@ def open_config_in_terminal(extra_args: list[str]) -> bool: args=extra_args, cwd=Path(__file__).resolve().parents[1], heading="X API 配置向导", - description="使用 OAuth2 PKCE，需要 X Developer App 的 Client ID，并通过浏览器授权账号。", + description="使用 OAuth2 PKCE，需要 X Developer App 的 Client ID，并完成 X OAuth2 授权。", ) if not opened: return False @@ -97,18 +101,30 @@ def open_config_in_terminal(extra_args: list[str]) -> bool: return True -def open_self_in_terminal_for_config(args: argparse.Namespace, reason: str) -> bool: +def open_required_config_in_terminal(reason: str) -> bool: opened = open_script_in_terminal( - script=Path(__file__).resolve(), - args=[arg for arg in sys.argv[1:] if arg not in {"--non-interactive"}], + script=Path(__file__).with_name("configure_api.py"), + args=["--oauth"], cwd=Path(__file__).resolve().parents[1], - heading="X 日报配置与采集", - description=f"{reason}。请在这个 Terminal 窗口完成 X API 配置，脚本随后会继续采集 X 数据。", + heading="X API 配置向导", + description=f"{reason}。X API 配置是生成日报的必需步骤；请在这个 Terminal 窗口完成配置。", ) if not opened: return False - print("已打开 Terminal 窗口用于配置 X API，并会在配置成功后继续采集 X 数据。", flush=True) + print("已打开 Terminal 窗口用于配置 X API。", flush=True) print(f"配置会保存到：{display_path(Path(__file__).resolve().parents[1] / '.state' / 'api_config.json')}", flush=True) + print( + json.dumps( + { + "status": "api_configuration_required", + "terminal_opened": True, + "next_step": "请先在刚打开的 Terminal 窗口完成 X API 配置。配置完成后，回到当前对话重新运行生成日报命令。不要在聊天里粘贴 Client Secret。", + }, + ensure_ascii=False, + indent=2, + ), + flush=True, + ) return True @@ -134,10 +150,6 @@ def api_public_item_limit(args: argparse.Namespace) -> int: return max(1, int(args.max_public_items if args.max_public_items is not None else DEFAULT_API_PUBLIC_ITEMS)) -def browser_public_item_limit(args: argparse.Namespace) -> int: - return max(1, int(args.max_public_items if args.max_public_items is not None else DEFAULT_BROWSER_PUBLIC_ITEMS)) - - def api_command(args: argparse.Namespace, out_dir: str, api_base: str, user_id: str, handle: str) -> list[str]: return api_collector_command( sys.executable, @@ -152,45 +164,15 @@ def api_command(args: argparse.Namespace, out_dir: str, api_base: str, user_id: handle=handle, ) - -def browser_command(args: argparse.Namespace, out_dir: str, handle: str, include_dms: bool, dm_only: bool = False) -> list[str]: - return browser_collector_command( - sys.executable, - Path(__file__).resolve().parent, - out_dir, - keywords=args.keywords, - max_public_items=browser_public_item_limit(args), - public_window_hours=args.public_window_hours, - min_public_scrolls=args.min_public_scrolls, - scrolls=args.scrolls, - dm_threads=args.dm_threads, - dm_list_scrolls=args.dm_list_scrolls, - dm_scrolls=args.dm_scrolls, - dm_max_messages=args.dm_max_messages, - dm_window_hours=args.dm_window_hours, - handle=handle, - include_dms=include_dms, - dm_only=dm_only, - headed=args.headed, - headless=args.headless, - non_interactive=args.non_interactive, - ) - - def run_api_command(cmd: list[str], env: dict[str, str]) -> None: completed = subprocess.run(cmd, check=True, env=env, capture_output=True, text=True) if completed.stdout: print(completed.stdout.strip(), flush=True) - -def run_browser_command(cmd: list[str]) -> None: - subprocess.run(cmd, check=True) - - def run_configure_api_flow(reason: str, args: argparse.Namespace) -> None: print(f"{reason} Starting X API configuration...", flush=True) if not sys.stdin.isatty(): - if open_self_in_terminal_for_config(args, reason): + if open_required_config_in_terminal(reason): raise SystemExit(0) raise SystemExit("当前没有可交互终端，且无法自动打开 Terminal。请在 Terminal 中运行本命令完成 X API 配置。") subprocess.run([sys.executable, str(Path(__file__).with_name("configure_api.py")), "--oauth"], check=True) @@ -244,59 +226,40 @@ def main() -> None: explicit_bearer_token = bool(args.bearer_token) api_config, refresh_error, bearer_token, api_base, user_id, handle = load_fresh_api_state(args, config) refresh_error = str(api_config.get("refresh_error") or "") - source = args.source - if source == "auto": - source = "api" if api_configuration_present(api_config, bearer_token) else "browser" - if source == "api" and not explicit_bearer_token and (refresh_error or not api_configured(bearer_token)): - reason = "X API 配置缺失或已失效" if not refresh_error else f"X API token refresh failed: {refresh_error}" + if not explicit_bearer_token and (refresh_error or not api_configured(bearer_token)): + reason = "X API 配置是必需项，但当前缺失或已失效" if not refresh_error else f"X API token refresh failed: {refresh_error}" run_configure_api_flow(reason, args) api_config, refresh_error, bearer_token, api_base, user_id, handle = load_fresh_api_state(args, config) if refresh_error or not api_configured(bearer_token): raise SystemExit("X API configuration did not produce a usable token. Re-run configuration and try again.") - if source == "api" and refresh_error and not explicit_bearer_token: - raise SystemExit("Saved X OAuth token refresh failed. Re-run --configure-api or pass X_BEARER_TOKEN to use API source. Browser collection will not be opened while API is configured.") - include_dms = not args.no_dms - if args.include_dms: - include_dms = True - if source == "api": - cmd = api_command(args, args.out, api_base, user_id, handle) - child_env = os.environ.copy() - if bearer_token: - child_env["X_BEARER_TOKEN"] = bearer_token - else: - cmd = browser_command(args, args.out, handle, include_dms) - child_env = None - if source == "api" and include_dms: - print("API source selected. Browser DM collection is not run for API source; use --source browser when DMs are required.", flush=True) - print(f"Collecting X digest data via {source} source.", flush=True) + if refresh_error and not explicit_bearer_token: + raise SystemExit("Saved X OAuth token refresh failed. Re-run --configure-api or pass X_BEARER_TOKEN to use API source.") + cmd = api_command(args, args.out, api_base, user_id, handle) + child_env = os.environ.copy() + if bearer_token: + child_env["X_BEARER_TOKEN"] = bearer_token + if args.include_dms and not args.no_dms: + print("DM collection is not supported in this API-only digest.", flush=True) + print("Collecting X digest data via API.", flush=True) retried_after_reconfigure = False while True: - if source == "api": - try: - run_api_command(cmd, child_env) - break - except subprocess.CalledProcessError as exc: - summary = summarize_child_error(exc) - if not explicit_bearer_token and not retried_after_reconfigure and api_auth_needs_reconfigure(summary): - retried_after_reconfigure = True - run_configure_api_flow(f"X API authentication failed: {summary}", args) - api_config, refresh_error, bearer_token, api_base, user_id, handle = load_fresh_api_state(args, config) - if refresh_error or not api_configured(bearer_token): - raise SystemExit("X API reconfiguration did not produce a usable token.") from exc - cmd = api_command(args, args.out, api_base, user_id, handle) - child_env = os.environ.copy() - child_env["X_BEARER_TOKEN"] = bearer_token - continue - print(f"API collection failed: {summary}", file=sys.stderr, flush=True) - raise SystemExit(exc.returncode) from exc - else: - try: - run_browser_command(cmd) - break - except subprocess.CalledProcessError as exc: - summary = summarize_child_error(exc) - print(f"Browser collection failed: {summary}", file=sys.stderr, flush=True) - raise SystemExit(exc.returncode) from exc + try: + run_api_command(cmd, child_env) + break + except subprocess.CalledProcessError as exc: + summary = summarize_child_error(exc) + if not explicit_bearer_token and not retried_after_reconfigure and api_auth_needs_reconfigure(summary): + retried_after_reconfigure = True + run_configure_api_flow(f"X API authentication failed: {summary}", args) + api_config, refresh_error, bearer_token, api_base, user_id, handle = load_fresh_api_state(args, config) + if refresh_error or not api_configured(bearer_token): + raise SystemExit("X API reconfiguration did not produce a usable token.") from exc + cmd = api_command(args, args.out, api_base, user_id, handle) + child_env = os.environ.copy() + child_env["X_BEARER_TOKEN"] = bearer_token + continue + print(f"API collection failed: {summary}", file=sys.stderr, flush=True) + raise SystemExit(exc.returncode) from exc out_dir = Path(args.out) build_current_context_from_file( input_path=out_dir / "digest-input.json", @@ -314,7 +277,7 @@ def main() -> None: "debug_raw_markdown": str(out_dir / "digest-input.md"), "debug_raw_json": str(out_dir / "digest-input.json"), "memory": "disabled", - "source": source, + "source": "api", } print(json.dumps(result, ensure_ascii=False, indent=2))