From c56edbd2262abc0d7e1e18984c923c2d69d22bc4 Mon Sep 17 00:00:00 2001 From: Sylvain Cau Date: Mon, 8 Jun 2026 15:28:27 -0700 Subject: [PATCH 01/10] feat(plugins): scaffold tsundoku release-source plugin Add release-tsundoku, an api-feed release source that will consume a Tsundoku instance's incremental series feed and announce new volume/chapter coverage for tracked series, matched by exact external IDs (no fuzzy matching). This change lays the foundation: package scaffolding, the manifest (api-feed capability declaring all supported Tsundoku external-ID providers, plus a config schema with a required baseUrl and optional defaultLanguage/pageLimit/requestTimeoutMs), config-driven initialization that captures the host RPC and KV-store clients, and single-source registration with a METHOD_NOT_FOUND retry. The poll handler is a no-candidate stub for now; the feed walk, cursor persistence, and matching land in follow-up changes. Tests cover base URL normalization and source registration. Wire the plugin into the build and CI matrices, docker-compose (backend and worker dist mounts, dev-watch volume, build and dev commands), and the official plugin gallery. No Makefile change is needed since it auto-discovers plugins via a glob. --- .github/workflows/build.yml | 2 + .github/workflows/ci.yml | 1 + docker-compose.yml | 9 +- plugins/release-tsundoku/package-lock.json | 2016 +++++++++++++++++ plugins/release-tsundoku/package.json | 51 + plugins/release-tsundoku/src/index.test.ts | 129 ++ plugins/release-tsundoku/src/index.ts | 220 ++ plugins/release-tsundoku/src/manifest.ts | 94 + plugins/release-tsundoku/tsconfig.json | 24 + plugins/release-tsundoku/vitest.config.ts | 7 + .../settings/plugins/OfficialPlugins.tsx | 16 + 11 files changed, 2567 insertions(+), 2 deletions(-) create mode 100644 plugins/release-tsundoku/package-lock.json create mode 100644 plugins/release-tsundoku/package.json create mode 100644 plugins/release-tsundoku/src/index.test.ts create mode 100644 plugins/release-tsundoku/src/index.ts create mode 100644 plugins/release-tsundoku/src/manifest.ts create mode 100644 plugins/release-tsundoku/tsconfig.json create mode 100644 plugins/release-tsundoku/vitest.config.ts diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9c10b073..adf42efe 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -156,6 +156,7 @@ jobs: - sync-anilist - release-mangaupdates - release-nyaa + - release-tsundoku steps: - uses: actions/checkout@v4 - name: Setup Node.js @@ -586,6 +587,7 @@ jobs: - sync-anilist - release-mangaupdates - release-nyaa + - release-tsundoku steps: - uses: actions/checkout@v4 with: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1a235725..cf1cdfd0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -138,6 +138,7 @@ jobs: - sync-anilist - release-mangaupdates - release-nyaa + - release-tsundoku steps: - uses: actions/checkout@v4 - name: Setup Node.js diff --git a/docker-compose.yml b/docker-compose.yml index 2961d40b..8142e234 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -104,6 +104,7 @@ services: - ./plugins/sync-anilist/dist:/opt/codex/plugins/sync-anilist/dist:ro - ./plugins/release-mangaupdates/dist:/opt/codex/plugins/release-mangaupdates/dist:ro - ./plugins/release-nyaa/dist:/opt/codex/plugins/release-nyaa/dist:ro + - ./plugins/release-tsundoku/dist:/opt/codex/plugins/release-tsundoku/dist:ro environment: RUST_BACKTRACE: 1 # Email configuration for Mailhog @@ -190,6 +191,7 @@ services: - ./plugins/sync-anilist/dist:/opt/codex/plugins/sync-anilist/dist:ro - ./plugins/release-mangaupdates/dist:/opt/codex/plugins/release-mangaupdates/dist:ro - ./plugins/release-nyaa/dist:/opt/codex/plugins/release-nyaa/dist:ro + - ./plugins/release-tsundoku/dist:/opt/codex/plugins/release-tsundoku/dist:ro command: [ "cargo", @@ -261,6 +263,7 @@ services: - /plugins/sync-anilist/node_modules - /plugins/release-mangaupdates/node_modules - /plugins/release-nyaa/node_modules + - /plugins/release-tsundoku/node_modules command: - sh - -c @@ -276,9 +279,10 @@ services: cd /plugins/sync-anilist && npm install && npm run build && cd /plugins/release-mangaupdates && npm install && npm run build && cd /plugins/release-nyaa && npm install && npm run build && + cd /plugins/release-tsundoku && npm install && npm run build && echo 'Initial build complete. Watching for changes...' && npm install -g concurrently && - concurrently --names 'sdk,echo,sync-echo,rec-echo,mangabaka,openlibrary,rec-anilist,sync-anilist,rel-mu,rel-nyaa' --prefix-colors 'blue,green,greenBright,cyanBright,yellow,magenta,cyan,red,gray,white' \ + concurrently --names 'sdk,echo,sync-echo,rec-echo,mangabaka,openlibrary,rec-anilist,sync-anilist,rel-mu,rel-nyaa,rel-tsundoku' --prefix-colors 'blue,green,greenBright,cyanBright,yellow,magenta,cyan,red,gray,white,blueBright' \ "cd /plugins/sdk-typescript && npm run dev" \ "cd /plugins/metadata-echo && npm run dev" \ "cd /plugins/sync-echo && npm run dev" \ @@ -288,7 +292,8 @@ services: "cd /plugins/recommendations-anilist && npm run dev" \ "cd /plugins/sync-anilist && npm run dev" \ "cd /plugins/release-mangaupdates && npm run dev" \ - "cd /plugins/release-nyaa && npm run dev" + "cd /plugins/release-nyaa && npm run dev" \ + "cd /plugins/release-tsundoku && npm run dev" networks: - codex-network profiles: diff --git a/plugins/release-tsundoku/package-lock.json b/plugins/release-tsundoku/package-lock.json new file mode 100644 index 00000000..6148abaa --- /dev/null +++ b/plugins/release-tsundoku/package-lock.json @@ -0,0 +1,2016 @@ +{ + "name": "@ashdev/codex-plugin-release-tsundoku", + "version": "1.36.1", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "@ashdev/codex-plugin-release-tsundoku", + "version": "1.36.1", + "license": "MIT", + "dependencies": { + "@ashdev/codex-plugin-sdk": "file:../sdk-typescript" + }, + "bin": { + "codex-plugin-release-tsundoku": "dist/index.js" + }, + "devDependencies": { + "@biomejs/biome": "^2.4.4", + "@types/node": "^22.0.0", + "esbuild": "^0.27.3", + "typescript": "^5.9.3", + "vitest": "^4.0.18" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "../sdk-typescript": { + "name": "@ashdev/codex-plugin-sdk", + "version": "1.36.1", + "license": "MIT", + "devDependencies": { + "@biomejs/biome": "^2.4.4", + "@types/node": "^22.0.0", + "typescript": "^5.9.3", + "vitest": "^4.0.18" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@ashdev/codex-plugin-sdk": { + "resolved": "../sdk-typescript", + "link": true + }, + "node_modules/@biomejs/biome": { + "version": "2.4.16", + "resolved": "https://registry.npmjs.org/@biomejs/biome/-/biome-2.4.16.tgz", + "integrity": "sha512-x9ajFh1zChVybCiM3TN6OD4phAqLgtPZjFrZF+aTMYCPjwBO+k529TX7PPsAqtGNLeV4UgzwQnowEgS7bGmzcA==", + "dev": true, + "license": "MIT OR Apache-2.0", + "bin": { + "biome": "bin/biome" + }, + "engines": { + "node": ">=14.21.3" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/biome" + }, + "optionalDependencies": { + "@biomejs/cli-darwin-arm64": "2.4.16", + "@biomejs/cli-darwin-x64": "2.4.16", + "@biomejs/cli-linux-arm64": "2.4.16", + "@biomejs/cli-linux-arm64-musl": "2.4.16", + "@biomejs/cli-linux-x64": "2.4.16", + "@biomejs/cli-linux-x64-musl": "2.4.16", + "@biomejs/cli-win32-arm64": "2.4.16", + "@biomejs/cli-win32-x64": "2.4.16" + } + }, + "node_modules/@biomejs/cli-darwin-arm64": { + "version": "2.4.16", + "resolved": "https://registry.npmjs.org/@biomejs/cli-darwin-arm64/-/cli-darwin-arm64-2.4.16.tgz", + "integrity": "sha512-wxPvu4XOA85YJk9ixSWUmq/QBHbid85BISbOAqqBM/5xQpPk9ayjk5375tOlSC0BeCwNSbPFafQBm+vBumXq0A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT OR Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=14.21.3" + } + }, + "node_modules/@biomejs/cli-darwin-x64": { + "version": "2.4.16", + "resolved": "https://registry.npmjs.org/@biomejs/cli-darwin-x64/-/cli-darwin-x64-2.4.16.tgz", + "integrity": "sha512-xFCqGPwYusQJp4N4NJLi1XJiZqjwFdjhT+KqtNy+Ug3qgfczqnTa6MSDvxJF6TkuDLoYJItMapz6tAf7kCekFw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT OR Apache-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=14.21.3" + } + }, + "node_modules/@biomejs/cli-linux-arm64": { + "version": "2.4.16", + "resolved": "https://registry.npmjs.org/@biomejs/cli-linux-arm64/-/cli-linux-arm64-2.4.16.tgz", + "integrity": "sha512-2kFb4//jxfZaP6D+Rj5VkHkxgyD9EoRAVBEQb8PKRv+s4NO2zYNJKXFaJmK1CmhufJOWEfpHKaRbOja7qjmdhQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT OR Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=14.21.3" + } + }, + "node_modules/@biomejs/cli-linux-arm64-musl": { + "version": "2.4.16", + "resolved": "https://registry.npmjs.org/@biomejs/cli-linux-arm64-musl/-/cli-linux-arm64-musl-2.4.16.tgz", + "integrity": "sha512-oYxnW0ARfJkr72ezzF2OR8N/rtkgLUQeYtF8cFhVswbknHxtTcmzSsanVJP8yQKnGpGpc2ck6c5zLvHahL6Cbg==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT OR Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=14.21.3" + } + }, + "node_modules/@biomejs/cli-linux-x64": { + "version": "2.4.16", + "resolved": "https://registry.npmjs.org/@biomejs/cli-linux-x64/-/cli-linux-x64-2.4.16.tgz", + "integrity": "sha512-NbcBbi/nJqn5baae6wqRXdS7Gadf2uRpehSh6vMSYpG8OhkXl/Xg8aorWrJ+9VWqAT5ml90alLvorkpMW0nBwQ==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT OR Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=14.21.3" + } + }, + "node_modules/@biomejs/cli-linux-x64-musl": { + "version": "2.4.16", + "resolved": "https://registry.npmjs.org/@biomejs/cli-linux-x64-musl/-/cli-linux-x64-musl-2.4.16.tgz", + "integrity": "sha512-iHDS+MCM65DPqWGu+ECC3uoALyj2H7F4nVUPxIPjz/PIl94EUu+EDfGZDzFP+NY1EOPVt9NQvwFqq7HdMmowdg==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT OR Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=14.21.3" + } + }, + "node_modules/@biomejs/cli-win32-arm64": { + "version": "2.4.16", + "resolved": "https://registry.npmjs.org/@biomejs/cli-win32-arm64/-/cli-win32-arm64-2.4.16.tgz", + "integrity": "sha512-0rgImMsNb5v/chhkIFe3wu7PEFClS6RBAYUijGL9UsYN3PanSaoK24HSSuSJb1pYbYYVjzAyZTl3gtjJ84BM8A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT OR Apache-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=14.21.3" + } + }, + "node_modules/@biomejs/cli-win32-x64": { + "version": "2.4.16", + "resolved": "https://registry.npmjs.org/@biomejs/cli-win32-x64/-/cli-win32-x64-2.4.16.tgz", + "integrity": "sha512-Kp85jgoBHa05gix6UIRjfCDiUV3w/8VIdZ247VyyO2gEjaw12WEVhdIjlxp/AMzXxqxQwbxNTDVZ3Mwd2RG5rw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT OR Apache-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=14.21.3" + } + }, + "node_modules/@emnapi/core": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.10.0.tgz", + "integrity": "sha512-yq6OkJ4p82CAfPl0u9mQebQHKPJkY7WrIuk205cTYnYe+k2Z8YBh11FrbRG/H6ihirqcacOgl2BIO8oyMQLeXw==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@emnapi/wasi-threads": "1.2.1", + "tslib": "^2.4.0" + } + }, + "node_modules/@emnapi/runtime": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.10.0.tgz", + "integrity": "sha512-ewvYlk86xUoGI0zQRNq/mC+16R1QeDlKQy21Ki3oSYXNgLb45GV1P6A0M+/s6nyCuNDqe5VpaY84BzXGwVbwFA==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@emnapi/wasi-threads": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.1.tgz", + "integrity": "sha512-uTII7OYF+/Mes/MrcIOYp5yOtSMLBWSIoLPpcgwipoiKbli6k322tcoFsxoIIxPDqW01SQGAgko4EzZi2BNv2w==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.27.7.tgz", + "integrity": "sha512-EKX3Qwmhz1eMdEJokhALr0YiD0lhQNwDqkPYyPhiSwKrh7/4KRjQc04sZ8db+5DVVnZ1LmbNDI1uAMPEUBnQPg==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.27.7.tgz", + "integrity": "sha512-jbPXvB4Yj2yBV7HUfE2KHe4GJX51QplCN1pGbYjvsyCZbQmies29EoJbkEc+vYuU5o45AfQn37vZlyXy4YJ8RQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.27.7.tgz", + "integrity": "sha512-62dPZHpIXzvChfvfLJow3q5dDtiNMkwiRzPylSCfriLvZeq0a1bWChrGx/BbUbPwOrsWKMn8idSllklzBy+dgQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.27.7.tgz", + "integrity": "sha512-x5VpMODneVDb70PYV2VQOmIUUiBtY3D3mPBG8NxVk5CogneYhkR7MmM3yR/uMdITLrC1ml/NV1rj4bMJuy9MCg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.27.7.tgz", + "integrity": "sha512-5lckdqeuBPlKUwvoCXIgI2D9/ABmPq3Rdp7IfL70393YgaASt7tbju3Ac+ePVi3KDH6N2RqePfHnXkaDtY9fkw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.27.7.tgz", + "integrity": "sha512-rYnXrKcXuT7Z+WL5K980jVFdvVKhCHhUwid+dDYQpH+qu+TefcomiMAJpIiC2EM3Rjtq0sO3StMV/+3w3MyyqQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.27.7.tgz", + "integrity": "sha512-B48PqeCsEgOtzME2GbNM2roU29AMTuOIN91dsMO30t+Ydis3z/3Ngoj5hhnsOSSwNzS+6JppqWsuhTp6E82l2w==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.27.7.tgz", + "integrity": "sha512-jOBDK5XEjA4m5IJK3bpAQF9/Lelu/Z9ZcdhTRLf4cajlB+8VEhFFRjWgfy3M1O4rO2GQ/b2dLwCUGpiF/eATNQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.27.7.tgz", + "integrity": "sha512-RkT/YXYBTSULo3+af8Ib0ykH8u2MBh57o7q/DAs3lTJlyVQkgQvlrPTnjIzzRPQyavxtPtfg0EopvDyIt0j1rA==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.27.7.tgz", + "integrity": "sha512-RZPHBoxXuNnPQO9rvjh5jdkRmVizktkT7TCDkDmQ0W2SwHInKCAV95GRuvdSvA7w4VMwfCjUiPwDi0ZO6Nfe9A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.27.7.tgz", + "integrity": "sha512-GA48aKNkyQDbd3KtkplYWT102C5sn/EZTY4XROkxONgruHPU72l+gW+FfF8tf2cFjeHaRbWpOYa/uRBz/Xq1Pg==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.27.7.tgz", + "integrity": "sha512-a4POruNM2oWsD4WKvBSEKGIiWQF8fZOAsycHOt6JBpZ+JN2n2JH9WAv56SOyu9X5IqAjqSIPTaJkqN8F7XOQ5Q==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.27.7.tgz", + "integrity": "sha512-KabT5I6StirGfIz0FMgl1I+R1H73Gp0ofL9A3nG3i/cYFJzKHhouBV5VWK1CSgKvVaG4q1RNpCTR2LuTVB3fIw==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.27.7.tgz", + "integrity": "sha512-gRsL4x6wsGHGRqhtI+ifpN/vpOFTQtnbsupUF5R5YTAg+y/lKelYR1hXbnBdzDjGbMYjVJLJTd2OFmMewAgwlQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.27.7.tgz", + "integrity": "sha512-hL25LbxO1QOngGzu2U5xeXtxXcW+/GvMN3ejANqXkxZ/opySAZMrc+9LY/WyjAan41unrR3YrmtTsUpwT66InQ==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.27.7.tgz", + "integrity": "sha512-2k8go8Ycu1Kb46vEelhu1vqEP+UeRVj2zY1pSuPdgvbd5ykAw82Lrro28vXUrRmzEsUV0NzCf54yARIK8r0fdw==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.27.7.tgz", + "integrity": "sha512-hzznmADPt+OmsYzw1EE33ccA+HPdIqiCRq7cQeL1Jlq2gb1+OyWBkMCrYGBJ+sxVzve2ZJEVeePbLM2iEIZSxA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.27.7.tgz", + "integrity": "sha512-b6pqtrQdigZBwZxAn1UpazEisvwaIDvdbMbmrly7cDTMFnw/+3lVxxCTGOrkPVnsYIosJJXAsILG9XcQS+Yu6w==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.27.7.tgz", + "integrity": "sha512-OfatkLojr6U+WN5EDYuoQhtM+1xco+/6FSzJJnuWiUw5eVcicbyK3dq5EeV/QHT1uy6GoDhGbFpprUiHUYggrw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.27.7.tgz", + "integrity": "sha512-AFuojMQTxAz75Fo8idVcqoQWEHIXFRbOc1TrVcFSgCZtQfSdc1RXgB3tjOn/krRHENUB4j00bfGjyl2mJrU37A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.27.7.tgz", + "integrity": "sha512-+A1NJmfM8WNDv5CLVQYJ5PshuRm/4cI6WMZRg1by1GwPIQPCTs1GLEUHwiiQGT5zDdyLiRM/l1G0Pv54gvtKIg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openharmony-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.27.7.tgz", + "integrity": "sha512-+KrvYb/C8zA9CU/g0sR6w2RBw7IGc5J2BPnc3dYc5VJxHCSF1yNMxTV5LQ7GuKteQXZtspjFbiuW5/dOj7H4Yw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.27.7.tgz", + "integrity": "sha512-ikktIhFBzQNt/QDyOL580ti9+5mL/YZeUPKU2ivGtGjdTYoqz6jObj6nOMfhASpS4GU4Q/Clh1QtxWAvcYKamA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.27.7.tgz", + "integrity": "sha512-7yRhbHvPqSpRUV7Q20VuDwbjW5kIMwTHpptuUzV+AA46kiPze5Z7qgt6CLCK3pWFrHeNfDd1VKgyP4O+ng17CA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.27.7.tgz", + "integrity": "sha512-SmwKXe6VHIyZYbBLJrhOoCJRB/Z1tckzmgTLfFYOfpMAx63BJEaL9ExI8x7v0oAO3Zh6D/Oi1gVxEYr5oUCFhw==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.27.7.tgz", + "integrity": "sha512-56hiAJPhwQ1R4i+21FVF7V8kSD5zZTdHcVuRFMW0hn753vVfQN8xlx4uOPT4xoGH0Z/oVATuR82AiqSTDIpaHg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.5.5", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.5.tgz", + "integrity": "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og==", + "dev": true, + "license": "MIT" + }, + "node_modules/@napi-rs/wasm-runtime": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.1.4.tgz", + "integrity": "sha512-3NQNNgA1YSlJb/kMH1ildASP9HW7/7kYnRI2szWJaofaS1hWmbGI4H+d3+22aGzXXN9IJ+n+GiFVcGipJP18ow==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@tybys/wasm-util": "^0.10.1" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/Brooooooklyn" + }, + "peerDependencies": { + "@emnapi/core": "^1.7.1", + "@emnapi/runtime": "^1.7.1" + } + }, + "node_modules/@oxc-project/types": { + "version": "0.133.0", + "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.133.0.tgz", + "integrity": "sha512-KzkdCd6Uxqnf6l3HOw1xfatAlUURA0g14cvBYFyJ5SaNOQbOUvBr9PKArcPcrNIeRsBdgcUzOGrhKveVpvOIGA==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/Boshen" + } + }, + "node_modules/@rolldown/binding-android-arm64": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.0.3.tgz", + "integrity": "sha512-454rs7jHngixp/NMxd5srYD57OnzSlZ/eFTETjORQHLwJG1lRtmNOJcBerZlfu4GjKqeq8aCCIQrMdHyhI51Hw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-darwin-arm64": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-arm64/-/binding-darwin-arm64-1.0.3.tgz", + "integrity": "sha512-PcAhP+ynjURNyy8SKGl5DQP94aGuB/7JrXJb/t7P+hanXvQVMWzUvRRhBAcg/lNRadBhoUPqSoP4xw5tR/KBEA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-darwin-x64": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-x64/-/binding-darwin-x64-1.0.3.tgz", + "integrity": "sha512-9YpfeUvSE2RS7wysJ81uOZkXJz7f7Q55H2Gvp3VEw/EsahqDtrphrZ0EwDLK5vvKOzaCrBsjF8JmnMLcUt78Gg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-freebsd-x64": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-freebsd-x64/-/binding-freebsd-x64-1.0.3.tgz", + "integrity": "sha512-yB1IlAsSNHncV6SCTL27/MVGR5htvQsoGxIv5KMGXALp+Ll1wYsn+x98M9MW7qa+NdSbvrrY7ANI4wLJ0n1e6g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm-gnueabihf": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.0.3.tgz", + "integrity": "sha512-Yi30IVAAfLUCy2MseFjbB1jAMDl1VMCAas5StnYp8da9+CKvMd2H2cbEjWcw5NPaPqzvYkVIaF1nNUG+b7u/sw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm64-gnu": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.0.3.tgz", + "integrity": "sha512-jsO7R8To+AdlYgUmN5sHSCZbfhtMBkO0WUx8iORQnPcMMdgr7qM2DQmMwgabs3GhNztdmoKkMKQFHD6DTMCIQw==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-arm64-musl": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.0.3.tgz", + "integrity": "sha512-VWkUHwWriDciit80wleYwKILoR/KMvxh/IdwS/paX+ZgpuRpCrKLUdadJbc0NpBEiyhpYawsJ73j9aCvOH+f7Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-ppc64-gnu": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.0.3.tgz", + "integrity": "sha512-5f1laC0SlIR0yDbFCd8acUhvJIag6N3zC5P7oUPN6wX0aOma+uKJ0wBDH5aq7I1PVI2ttTlhJwzwRIBnLiSGEg==", + "cpu": [ + "ppc64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-s390x-gnu": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.0.3.tgz", + "integrity": "sha512-Iq4ko0r4XsgbrF/LunNgHtAGLRRVE2kXonAXQ/MV0mC6jQpMOhW1SvtZja2EhC/kd05++bP78dsqBeIQyYJ6Yg==", + "cpu": [ + "s390x" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-x64-gnu": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.0.3.tgz", + "integrity": "sha512-B8m6tD5+/N5FeNQFbKlLA/2yVq9ycQP1SeedyEYYKWBNR3ZQbkvIUcNnDNM03lO1l5F2roiiFJGgvoLLyZXtSg==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-linux-x64-musl": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-musl/-/binding-linux-x64-musl-1.0.3.tgz", + "integrity": "sha512-pSdpdUJHkuCxun9LE7jvgUB9qsRgaiyNNCX7m/AvHTcq67AiT/Yhoxvw5zPfhrM8k/BfP8ce/hMOpthKDpEUow==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-openharmony-arm64": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-openharmony-arm64/-/binding-openharmony-arm64-1.0.3.tgz", + "integrity": "sha512-OXXS3RKJgX2uLwM+gYyuH5omcH8fL1LJs96pZGgtetVCahON57+d4SJHzTgZiOjxgGkSnpXpOsWuPDGAKAigEg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-wasm32-wasi": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-wasm32-wasi/-/binding-wasm32-wasi-1.0.3.tgz", + "integrity": "sha512-JTtb8BWFynicNSoPrehsCzBtOKjZ6jhMiPFEmOiuXg1Fl8dn2KHQob+GuPSGR0dryQa1PQJbzjF3dqO/whhjLg==", + "cpu": [ + "wasm32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "@emnapi/core": "1.10.0", + "@emnapi/runtime": "1.10.0", + "@napi-rs/wasm-runtime": "^1.1.4" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-win32-arm64-msvc": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.0.3.tgz", + "integrity": "sha512-gEdFFEN70A/jxb2svrWsN3aDL7OUtmvlOy+6fa2jxG8K0wQ1ZbdeLGnidov6Yu5/733dI5ySfzFlQ/cb0bSz1g==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/binding-win32-x64-msvc": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.0.3.tgz", + "integrity": "sha512-eXB7CHuaQdqmJcc3koCNtNPmT/bj2gc999kUFgBxG8Ac0NdgXc4rkCHhqrgrhN3zddvvvrgzj1e90SuSfmyIXA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^20.19.0 || >=22.12.0" + } + }, + "node_modules/@rolldown/pluginutils": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.1.tgz", + "integrity": "sha512-2j9bGt5Jh8hj+vPtgzPtl72j0yRxHAyumoo6TNfAjsLB04UtpSvPbPcDcBMxz7n+9CYB0c1GxQFxYRg2jimqGw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@standard-schema/spec": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz", + "integrity": "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==", + "dev": true, + "license": "MIT" + }, + "node_modules/@tybys/wasm-util": { + "version": "0.10.2", + "resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.2.tgz", + "integrity": "sha512-RoBvJ2X0wuKlWFIjrwffGw1IqZHKQqzIchKaadZZfnNpsAYp2mM0h36JtPCjNDAHGgYez/15uMBpfGwchhiMgg==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, + "node_modules/@types/chai": { + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/@types/chai/-/chai-5.2.3.tgz", + "integrity": "sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/deep-eql": "*", + "assertion-error": "^2.0.1" + } + }, + "node_modules/@types/deep-eql": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@types/deep-eql/-/deep-eql-4.0.2.tgz", + "integrity": "sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/estree": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.9.tgz", + "integrity": "sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/node": { + "version": "22.19.20", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.19.20.tgz", + "integrity": "sha512-6tELRwSDYWW9EdZhbeZmYGZ1/7Djkt+Ah3/ScEYT9cDord7UJzasR/4D3VONg9tQI5CDp+/CZC1AXj2pCFOvpw==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/@vitest/expect": { + "version": "4.1.8", + "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.1.8.tgz", + "integrity": "sha512-h3nDO677RDLEGlBxyQ5CW8RlMThSKSRLUePLOx09gNIWRL40edgA1GCZSZgf1W55MFAG6/Sw14KeaAnqv0NKdQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@standard-schema/spec": "^1.1.0", + "@types/chai": "^5.2.2", + "@vitest/spy": "4.1.8", + "@vitest/utils": "4.1.8", + "chai": "^6.2.2", + "tinyrainbow": "^3.1.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/mocker": { + "version": "4.1.8", + "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-4.1.8.tgz", + "integrity": "sha512-LEiN/xe4OSIbKe9HQIp5OC24agGD9J5CnmMgsLohVVoOPWL9a2sBoR6VBx43jQZb7Kr1l4RCuyCJzcAa0+dojw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/spy": "4.1.8", + "estree-walker": "^3.0.3", + "magic-string": "^0.30.21" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "msw": "^2.4.9", + "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" + }, + "peerDependenciesMeta": { + "msw": { + "optional": true + }, + "vite": { + "optional": true + } + } + }, + "node_modules/@vitest/pretty-format": { + "version": "4.1.8", + "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-4.1.8.tgz", + "integrity": "sha512-9GasEBxpZ1VYIpqHf/0+YGg121uSNwCKOJqIrTwWP/TB7DmFCiaBpNl3aPZzoLWfWkuqhbH8vJIVobZkvdo2cA==", + "dev": true, + "license": "MIT", + "dependencies": { + "tinyrainbow": "^3.1.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/runner": { + "version": "4.1.8", + "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-4.1.8.tgz", + "integrity": "sha512-EmVxeBAfMJvycdjd6Hm+RbFBbA9fKvo0Kx37hNpBYoYeavH3RNsBXWDooR1mgD52dCrxIIuP7UotpfiwOikvcg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/utils": "4.1.8", + "pathe": "^2.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/snapshot": { + "version": "4.1.8", + "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-4.1.8.tgz", + "integrity": "sha512-acfZboRmAIf05DEKcBQy33VXojFJjtUdLyo7oOmV9kebb2xdU01UknNiPuPZoJZQyO7DF0gZdTGTpeAzET9QPQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/pretty-format": "4.1.8", + "@vitest/utils": "4.1.8", + "magic-string": "^0.30.21", + "pathe": "^2.0.3" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/spy": { + "version": "4.1.8", + "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-4.1.8.tgz", + "integrity": "sha512-6EevtBp6OZOPF7bmz36HrGMeP3txgVSrgebWxHOafDXGkhIzfXK14f8KF6MuFfgXXUeHxmpD3BQxkV00/3s5mA==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/@vitest/utils": { + "version": "4.1.8", + "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-4.1.8.tgz", + "integrity": "sha512-uOJamYALNhfJ6iolExyQM40yIQwDqYnkKtQ5VCiSe17E33H0aQ/u+1GlRuz4LZBk6Mm3sg90G9hEbmEt37C1Zg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/pretty-format": "4.1.8", + "convert-source-map": "^2.0.0", + "tinyrainbow": "^3.1.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + } + }, + "node_modules/assertion-error": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-2.0.1.tgz", + "integrity": "sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + } + }, + "node_modules/chai": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/chai/-/chai-6.2.2.tgz", + "integrity": "sha512-NUPRluOfOiTKBKvWPtSD4PhFvWCqOi0BGStNWs57X9js7XGTprSmFoz5F0tWhR4WPjNeR9jXqdC7/UpSJTnlRg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/convert-source-map": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", + "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==", + "dev": true, + "license": "MIT" + }, + "node_modules/detect-libc": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=8" + } + }, + "node_modules/es-module-lexer": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-2.1.0.tgz", + "integrity": "sha512-n27zTYMjYu1aj4MjCWzSP7G9r75utsaoc8m61weK+W8JMBGGQybd43GstCXZ3WNmSFtGT9wi59qQTW6mhTR5LQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/esbuild": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.27.7.tgz", + "integrity": "sha512-IxpibTjyVnmrIQo5aqNpCgoACA/dTKLTlhMHihVHhdkxKyPO1uBBthumT0rdHmcsk9uMonIWS0m4FljWzILh3w==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.27.7", + "@esbuild/android-arm": "0.27.7", + "@esbuild/android-arm64": "0.27.7", + "@esbuild/android-x64": "0.27.7", + "@esbuild/darwin-arm64": "0.27.7", + "@esbuild/darwin-x64": "0.27.7", + "@esbuild/freebsd-arm64": "0.27.7", + "@esbuild/freebsd-x64": "0.27.7", + "@esbuild/linux-arm": "0.27.7", + "@esbuild/linux-arm64": "0.27.7", + "@esbuild/linux-ia32": "0.27.7", + "@esbuild/linux-loong64": "0.27.7", + "@esbuild/linux-mips64el": "0.27.7", + "@esbuild/linux-ppc64": "0.27.7", + "@esbuild/linux-riscv64": "0.27.7", + "@esbuild/linux-s390x": "0.27.7", + "@esbuild/linux-x64": "0.27.7", + "@esbuild/netbsd-arm64": "0.27.7", + "@esbuild/netbsd-x64": "0.27.7", + "@esbuild/openbsd-arm64": "0.27.7", + "@esbuild/openbsd-x64": "0.27.7", + "@esbuild/openharmony-arm64": "0.27.7", + "@esbuild/sunos-x64": "0.27.7", + "@esbuild/win32-arm64": "0.27.7", + "@esbuild/win32-ia32": "0.27.7", + "@esbuild/win32-x64": "0.27.7" + } + }, + "node_modules/estree-walker": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-3.0.3.tgz", + "integrity": "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "^1.0.0" + } + }, + "node_modules/expect-type": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/expect-type/-/expect-type-1.3.0.tgz", + "integrity": "sha512-knvyeauYhqjOYvQ66MznSMs83wmHrCycNEN6Ao+2AeYEfxUIkuiVxdEa1qlGEPK+We3n0THiDciYSsCcgW/DoA==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/fdir": { + "version": "6.5.0", + "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", + "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12.0.0" + }, + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/lightningcss": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss/-/lightningcss-1.32.0.tgz", + "integrity": "sha512-NXYBzinNrblfraPGyrbPoD19C1h9lfI/1mzgWYvXUTe414Gz/X1FD2XBZSZM7rRTrMA8JL3OtAaGifrIKhQ5yQ==", + "dev": true, + "license": "MPL-2.0", + "dependencies": { + "detect-libc": "^2.0.3" + }, + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + }, + "optionalDependencies": { + "lightningcss-android-arm64": "1.32.0", + "lightningcss-darwin-arm64": "1.32.0", + "lightningcss-darwin-x64": "1.32.0", + "lightningcss-freebsd-x64": "1.32.0", + "lightningcss-linux-arm-gnueabihf": "1.32.0", + "lightningcss-linux-arm64-gnu": "1.32.0", + "lightningcss-linux-arm64-musl": "1.32.0", + "lightningcss-linux-x64-gnu": "1.32.0", + "lightningcss-linux-x64-musl": "1.32.0", + "lightningcss-win32-arm64-msvc": "1.32.0", + "lightningcss-win32-x64-msvc": "1.32.0" + } + }, + "node_modules/lightningcss-android-arm64": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-android-arm64/-/lightningcss-android-arm64-1.32.0.tgz", + "integrity": "sha512-YK7/ClTt4kAK0vo6w3X+Pnm0D2cf2vPHbhOXdoNti1Ga0al1P4TBZhwjATvjNwLEBCnKvjJc2jQgHXH0NEwlAg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-darwin-arm64": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-darwin-arm64/-/lightningcss-darwin-arm64-1.32.0.tgz", + "integrity": "sha512-RzeG9Ju5bag2Bv1/lwlVJvBE3q6TtXskdZLLCyfg5pt+HLz9BqlICO7LZM7VHNTTn/5PRhHFBSjk5lc4cmscPQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-darwin-x64": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-darwin-x64/-/lightningcss-darwin-x64-1.32.0.tgz", + "integrity": "sha512-U+QsBp2m/s2wqpUYT/6wnlagdZbtZdndSmut/NJqlCcMLTWp5muCrID+K5UJ6jqD2BFshejCYXniPDbNh73V8w==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-freebsd-x64": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-freebsd-x64/-/lightningcss-freebsd-x64-1.32.0.tgz", + "integrity": "sha512-JCTigedEksZk3tHTTthnMdVfGf61Fky8Ji2E4YjUTEQX14xiy/lTzXnu1vwiZe3bYe0q+SpsSH/CTeDXK6WHig==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm-gnueabihf": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm-gnueabihf/-/lightningcss-linux-arm-gnueabihf-1.32.0.tgz", + "integrity": "sha512-x6rnnpRa2GL0zQOkt6rts3YDPzduLpWvwAF6EMhXFVZXD4tPrBkEFqzGowzCsIWsPjqSK+tyNEODUBXeeVHSkw==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm64-gnu": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-gnu/-/lightningcss-linux-arm64-gnu-1.32.0.tgz", + "integrity": "sha512-0nnMyoyOLRJXfbMOilaSRcLH3Jw5z9HDNGfT/gwCPgaDjnx0i8w7vBzFLFR1f6CMLKF8gVbebmkUN3fa/kQJpQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-arm64-musl": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-musl/-/lightningcss-linux-arm64-musl-1.32.0.tgz", + "integrity": "sha512-UpQkoenr4UJEzgVIYpI80lDFvRmPVg6oqboNHfoH4CQIfNA+HOrZ7Mo7KZP02dC6LjghPQJeBsvXhJod/wnIBg==", + "cpu": [ + "arm64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-x64-gnu": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-gnu/-/lightningcss-linux-x64-gnu-1.32.0.tgz", + "integrity": "sha512-V7Qr52IhZmdKPVr+Vtw8o+WLsQJYCTd8loIfpDaMRWGUZfBOYEJeyJIkqGIDMZPwPx24pUMfwSxxI8phr/MbOA==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "glibc" + ], + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-linux-x64-musl": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-musl/-/lightningcss-linux-x64-musl-1.32.0.tgz", + "integrity": "sha512-bYcLp+Vb0awsiXg/80uCRezCYHNg1/l3mt0gzHnWV9XP1W5sKa5/TCdGWaR/zBM2PeF/HbsQv/j2URNOiVuxWg==", + "cpu": [ + "x64" + ], + "dev": true, + "libc": [ + "musl" + ], + "license": "MPL-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-win32-arm64-msvc": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-win32-arm64-msvc/-/lightningcss-win32-arm64-msvc-1.32.0.tgz", + "integrity": "sha512-8SbC8BR40pS6baCM8sbtYDSwEVQd4JlFTOlaD3gWGHfThTcABnNDBda6eTZeqbofalIJhFx0qKzgHJmcPTnGdw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/lightningcss-win32-x64-msvc": { + "version": "1.32.0", + "resolved": "https://registry.npmjs.org/lightningcss-win32-x64-msvc/-/lightningcss-win32-x64-msvc-1.32.0.tgz", + "integrity": "sha512-Amq9B/SoZYdDi1kFrojnoqPLxYhQ4Wo5XiL8EVJrVsB8ARoC1PWW6VGtT0WKCemjy8aC+louJnjS7U18x3b06Q==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MPL-2.0", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 12.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/parcel" + } + }, + "node_modules/magic-string": { + "version": "0.30.21", + "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.21.tgz", + "integrity": "sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.5" + } + }, + "node_modules/nanoid": { + "version": "3.3.12", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.12.tgz", + "integrity": "sha512-ZB9RH/39qpq5Vu6Y+NmUaFhQR6pp+M2Xt76XBnEwDaGcVAqhlvxrl3B2bKS5D3NH3QR76v3aSrKaF/Kiy7lEtQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/obug": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/obug/-/obug-2.1.2.tgz", + "integrity": "sha512-AWGB9WFcRXOQs48Z/udjI5ZcZMHXwX8XPByNpOydgcGsDLIzjGizhoMWJyKAWze7AVW/2W1i+/gPX4YtKe5cyg==", + "dev": true, + "funding": [ + "https://github.com/sponsors/sxzz", + "https://opencollective.com/debug" + ], + "license": "MIT", + "engines": { + "node": ">=12.20.0" + } + }, + "node_modules/pathe": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/pathe/-/pathe-2.0.3.tgz", + "integrity": "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==", + "dev": true, + "license": "MIT" + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "dev": true, + "license": "ISC" + }, + "node_modules/picomatch": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", + "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/postcss": { + "version": "8.5.15", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.15.tgz", + "integrity": "sha512-FfR8sjd4em2T6fb3I2MwAJU7HWVMr9zba+enmQeeWFfCbm+UOC/0X4DS8XtpUTMwWMGbjKYP7xjfNekzyGmB3A==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "nanoid": "^3.3.12", + "picocolors": "^1.1.1", + "source-map-js": "^1.2.1" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/rolldown": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.0.3.tgz", + "integrity": "sha512-i00lAJ2ks1BYr7rjNjKC7BcqAS7nVfiT3QX1SI5aY+AFHblCmaUf9OE9dbdzDvW6dJxbi2ZCZiy9v3CcwOiX3g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@oxc-project/types": "=0.133.0", + "@rolldown/pluginutils": "^1.0.0" + }, + "bin": { + "rolldown": "bin/cli.mjs" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "optionalDependencies": { + "@rolldown/binding-android-arm64": "1.0.3", + "@rolldown/binding-darwin-arm64": "1.0.3", + "@rolldown/binding-darwin-x64": "1.0.3", + "@rolldown/binding-freebsd-x64": "1.0.3", + "@rolldown/binding-linux-arm-gnueabihf": "1.0.3", + "@rolldown/binding-linux-arm64-gnu": "1.0.3", + "@rolldown/binding-linux-arm64-musl": "1.0.3", + "@rolldown/binding-linux-ppc64-gnu": "1.0.3", + "@rolldown/binding-linux-s390x-gnu": "1.0.3", + "@rolldown/binding-linux-x64-gnu": "1.0.3", + "@rolldown/binding-linux-x64-musl": "1.0.3", + "@rolldown/binding-openharmony-arm64": "1.0.3", + "@rolldown/binding-wasm32-wasi": "1.0.3", + "@rolldown/binding-win32-arm64-msvc": "1.0.3", + "@rolldown/binding-win32-x64-msvc": "1.0.3" + } + }, + "node_modules/siginfo": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/siginfo/-/siginfo-2.0.0.tgz", + "integrity": "sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==", + "dev": true, + "license": "ISC" + }, + "node_modules/source-map-js": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", + "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/stackback": { + "version": "0.0.2", + "resolved": "https://registry.npmjs.org/stackback/-/stackback-0.0.2.tgz", + "integrity": "sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==", + "dev": true, + "license": "MIT" + }, + "node_modules/std-env": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/std-env/-/std-env-4.1.0.tgz", + "integrity": "sha512-Rq7ybcX2RuC55r9oaPVEW7/xu3tj8u4GeBYHBWCychFtzMIr86A7e3PPEBPT37sHStKX3+TiX/Fr/ACmJLVlLQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/tinybench": { + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/tinybench/-/tinybench-2.9.0.tgz", + "integrity": "sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==", + "dev": true, + "license": "MIT" + }, + "node_modules/tinyexec": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-1.2.4.tgz", + "integrity": "sha512-SHf/r48b7vOrjve9PxJo3MN5v5yuyjHvdUcrQffT3WXMUfnGmHDVbC4k3sHJaJTgZCwpUplIaAo5ANtMyp3YHg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/tinyglobby": { + "version": "0.2.17", + "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.17.tgz", + "integrity": "sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g==", + "dev": true, + "license": "MIT", + "dependencies": { + "fdir": "^6.5.0", + "picomatch": "^4.0.4" + }, + "engines": { + "node": ">=12.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/SuperchupuDev" + } + }, + "node_modules/tinyrainbow": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/tinyrainbow/-/tinyrainbow-3.1.0.tgz", + "integrity": "sha512-Bf+ILmBgretUrdJxzXM0SgXLZ3XfiaUuOj/IKQHuTXip+05Xn+uyEYdVg0kYDipTBcLrCVyUzAPz7QmArb0mmw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "dev": true, + "license": "0BSD", + "optional": true + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/vite": { + "version": "8.0.16", + "resolved": "https://registry.npmjs.org/vite/-/vite-8.0.16.tgz", + "integrity": "sha512-h9bXPmJichP5fLmVQo3PyaGSDE2n3aPuomeAlVRm0JLmt4rY6zmPKd59HYI4LNW8oTK7tlTsuC7l/m7awx9Jcw==", + "dev": true, + "license": "MIT", + "dependencies": { + "lightningcss": "^1.32.0", + "picomatch": "^4.0.4", + "postcss": "^8.5.15", + "rolldown": "1.0.3", + "tinyglobby": "^0.2.17" + }, + "bin": { + "vite": "bin/vite.js" + }, + "engines": { + "node": "^20.19.0 || >=22.12.0" + }, + "funding": { + "url": "https://github.com/vitejs/vite?sponsor=1" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + }, + "peerDependencies": { + "@types/node": "^20.19.0 || >=22.12.0", + "@vitejs/devtools": "^0.1.18", + "esbuild": "^0.27.0 || ^0.28.0", + "jiti": ">=1.21.0", + "less": "^4.0.0", + "sass": "^1.70.0", + "sass-embedded": "^1.70.0", + "stylus": ">=0.54.8", + "sugarss": "^5.0.0", + "terser": "^5.16.0", + "tsx": "^4.8.1", + "yaml": "^2.4.2" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "@vitejs/devtools": { + "optional": true + }, + "esbuild": { + "optional": true + }, + "jiti": { + "optional": true + }, + "less": { + "optional": true + }, + "sass": { + "optional": true + }, + "sass-embedded": { + "optional": true + }, + "stylus": { + "optional": true + }, + "sugarss": { + "optional": true + }, + "terser": { + "optional": true + }, + "tsx": { + "optional": true + }, + "yaml": { + "optional": true + } + } + }, + "node_modules/vitest": { + "version": "4.1.8", + "resolved": "https://registry.npmjs.org/vitest/-/vitest-4.1.8.tgz", + "integrity": "sha512-flY6ScbCIt9HThs+C5HS7jvGOB560DJtk/Z15IQROTA6zEy49Nh8T/dofWTQL+n3vswqn87sbJNiuqw1SDp5Ig==", + "dev": true, + "license": "MIT", + "dependencies": { + "@vitest/expect": "4.1.8", + "@vitest/mocker": "4.1.8", + "@vitest/pretty-format": "4.1.8", + "@vitest/runner": "4.1.8", + "@vitest/snapshot": "4.1.8", + "@vitest/spy": "4.1.8", + "@vitest/utils": "4.1.8", + "es-module-lexer": "^2.0.0", + "expect-type": "^1.3.0", + "magic-string": "^0.30.21", + "obug": "^2.1.1", + "pathe": "^2.0.3", + "picomatch": "^4.0.3", + "std-env": "^4.0.0-rc.1", + "tinybench": "^2.9.0", + "tinyexec": "^1.0.2", + "tinyglobby": "^0.2.15", + "tinyrainbow": "^3.1.0", + "vite": "^6.0.0 || ^7.0.0 || ^8.0.0", + "why-is-node-running": "^2.3.0" + }, + "bin": { + "vitest": "vitest.mjs" + }, + "engines": { + "node": "^20.0.0 || ^22.0.0 || >=24.0.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "@edge-runtime/vm": "*", + "@opentelemetry/api": "^1.9.0", + "@types/node": "^20.0.0 || ^22.0.0 || >=24.0.0", + "@vitest/browser-playwright": "4.1.8", + "@vitest/browser-preview": "4.1.8", + "@vitest/browser-webdriverio": "4.1.8", + "@vitest/coverage-istanbul": "4.1.8", + "@vitest/coverage-v8": "4.1.8", + "@vitest/ui": "4.1.8", + "happy-dom": "*", + "jsdom": "*", + "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" + }, + "peerDependenciesMeta": { + "@edge-runtime/vm": { + "optional": true + }, + "@opentelemetry/api": { + "optional": true + }, + "@types/node": { + "optional": true + }, + "@vitest/browser-playwright": { + "optional": true + }, + "@vitest/browser-preview": { + "optional": true + }, + "@vitest/browser-webdriverio": { + "optional": true + }, + "@vitest/coverage-istanbul": { + "optional": true + }, + "@vitest/coverage-v8": { + "optional": true + }, + "@vitest/ui": { + "optional": true + }, + "happy-dom": { + "optional": true + }, + "jsdom": { + "optional": true + }, + "vite": { + "optional": false + } + } + }, + "node_modules/why-is-node-running": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/why-is-node-running/-/why-is-node-running-2.3.0.tgz", + "integrity": "sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w==", + "dev": true, + "license": "MIT", + "dependencies": { + "siginfo": "^2.0.0", + "stackback": "0.0.2" + }, + "bin": { + "why-is-node-running": "cli.js" + }, + "engines": { + "node": ">=8" + } + } + } +} diff --git a/plugins/release-tsundoku/package.json b/plugins/release-tsundoku/package.json new file mode 100644 index 00000000..7d780da2 --- /dev/null +++ b/plugins/release-tsundoku/package.json @@ -0,0 +1,51 @@ +{ + "name": "@ashdev/codex-plugin-release-tsundoku", + "version": "1.36.1", + "description": "Tsundoku release-source plugin for Codex - announces new volume/chapter coverage for tracked series via the Tsundoku incremental series feed, matched by exact external IDs", + "main": "dist/index.js", + "bin": "dist/index.js", + "type": "module", + "files": [ + "dist", + "README.md" + ], + "repository": { + "type": "git", + "url": "https://github.com/AshDevFr/codex.git", + "directory": "plugins/release-tsundoku" + }, + "scripts": { + "build": "esbuild src/index.ts --bundle --platform=node --target=node22 --format=esm --outfile=dist/index.js --sourcemap --banner:js='#!/usr/bin/env node'", + "dev": "npm run build -- --watch", + "clean": "rm -rf dist", + "start": "node dist/index.js", + "lint": "biome check .", + "lint:fix": "biome check --write .", + "typecheck": "tsc --noEmit", + "test": "vitest run --passWithNoTests", + "test:watch": "vitest", + "prepublishOnly": "npm run lint && npm run build" + }, + "keywords": [ + "codex", + "plugin", + "tsundoku", + "release-source", + "manga" + ], + "author": "Codex", + "license": "MIT", + "engines": { + "node": ">=22.0.0" + }, + "dependencies": { + "@ashdev/codex-plugin-sdk": "file:../sdk-typescript" + }, + "devDependencies": { + "@biomejs/biome": "^2.4.4", + "@types/node": "^22.0.0", + "esbuild": "^0.27.3", + "typescript": "^5.9.3", + "vitest": "^4.0.18" + } +} diff --git a/plugins/release-tsundoku/src/index.test.ts b/plugins/release-tsundoku/src/index.test.ts new file mode 100644 index 00000000..8b4befcd --- /dev/null +++ b/plugins/release-tsundoku/src/index.test.ts @@ -0,0 +1,129 @@ +import { HostRpcClient } from "@ashdev/codex-plugin-sdk"; +import { describe, expect, it } from "vitest"; +import { normalizeBaseUrl, registerSources } from "./index.js"; + +// ----------------------------------------------------------------------------- +// Mock host RPC +// ----------------------------------------------------------------------------- + +interface CapturedCall { + method: string; + params: unknown; +} + +type Responder = ( + method: string, + params: unknown, + attempt: number, +) => unknown | { __error: { code: number; message: string } }; + +/** + * Build a `HostRpcClient` whose calls are intercepted in-memory. The custom + * `writeFn` captures each request and synthesizes a JSON-RPC response (result + * or error) via the real id-correlation path. `respond` may return a normal + * result, or `{ __error: { code, message } }` to drive an error response with + * a specific code (e.g. -32601 for METHOD_NOT_FOUND). + */ +function makeMockRpc(respond: Responder): { + rpc: HostRpcClient; + calls: CapturedCall[]; +} { + const calls: CapturedCall[] = []; + let attemptByMethod: Record = {}; + // `rpc` is referenced inside writeFn (a closure) before its assignment runs, + // so it must be declared with `let` and initialized after writeFn is built. + let rpc: HostRpcClient; + const writeFn = (line: string) => { + const req = JSON.parse(line.trim()) as { + id: number; + method: string; + params: unknown; + }; + calls.push({ method: req.method, params: req.params }); + attemptByMethod[req.method] = (attemptByMethod[req.method] ?? 0) + 1; + const outcome = respond(req.method, req.params, attemptByMethod[req.method]); + setImmediate(() => { + const isError = + outcome !== null && + typeof outcome === "object" && + "__error" in (outcome as Record); + const payload = isError + ? { + jsonrpc: "2.0", + id: req.id, + error: (outcome as { __error: { code: number; message: string } }).__error, + } + : { jsonrpc: "2.0", id: req.id, result: outcome }; + rpc.handleResponse(JSON.stringify(payload)); + }); + }; + rpc = new HostRpcClient(writeFn); + attemptByMethod = {}; + return { rpc, calls }; +} + +// ----------------------------------------------------------------------------- +// normalizeBaseUrl +// ----------------------------------------------------------------------------- + +describe("normalizeBaseUrl", () => { + it("strips trailing slashes and trims whitespace", () => { + expect(normalizeBaseUrl("https://t.example.com/")).toBe("https://t.example.com"); + expect(normalizeBaseUrl(" https://t.example.com/// ")).toBe("https://t.example.com"); + expect(normalizeBaseUrl("https://t.example.com")).toBe("https://t.example.com"); + }); +}); + +// ----------------------------------------------------------------------------- +// registerSources +// ----------------------------------------------------------------------------- + +describe("registerSources", () => { + it("registers exactly one api-feed source keyed 'default'", async () => { + const { rpc, calls } = makeMockRpc(() => ({ registered: 1, pruned: 0 })); + const result = await registerSources(rpc); + + expect(result).toEqual({ registered: 1, pruned: 0 }); + expect(calls).toHaveLength(1); + expect(calls[0].method).toBe("releases/register_sources"); + const params = calls[0].params as { sources: Array> }; + expect(params.sources).toHaveLength(1); + expect(params.sources[0]).toMatchObject({ + sourceKey: "default", + displayName: "Tsundoku Releases", + kind: "api-feed", + }); + }); + + it("retries on METHOD_NOT_FOUND then succeeds", async () => { + const { rpc, calls } = makeMockRpc((_m, _p, attempt) => + attempt < 3 + ? { __error: { code: -32601, message: "method not found" } } + : { registered: 1, pruned: 0 }, + ); + const result = await registerSources(rpc); + + expect(result).toEqual({ registered: 1, pruned: 0 }); + expect(calls.length).toBe(3); + }); + + it("returns null after exhausting retries on METHOD_NOT_FOUND", async () => { + const { rpc, calls } = makeMockRpc(() => ({ + __error: { code: -32601, message: "method not found" }, + })); + const result = await registerSources(rpc); + + expect(result).toBeNull(); + expect(calls.length).toBe(5); + }); + + it("does not retry on a non-METHOD_NOT_FOUND error", async () => { + const { rpc, calls } = makeMockRpc(() => ({ + __error: { code: -32000, message: "db error" }, + })); + const result = await registerSources(rpc); + + expect(result).toBeNull(); + expect(calls.length).toBe(1); + }); +}); diff --git a/plugins/release-tsundoku/src/index.ts b/plugins/release-tsundoku/src/index.ts new file mode 100644 index 00000000..e4c1ae69 --- /dev/null +++ b/plugins/release-tsundoku/src/index.ts @@ -0,0 +1,220 @@ +/** + * Tsundoku API-feed release-source plugin for Codex. + * + * Tsundoku exposes a single, catalog-wide incremental feed + * (`GET /api/v1/series/feed`) ordered by `(updatedAt, id)` and walked with an + * opaque keyset cursor. Each item carries the provider external IDs Codex + * matches on plus the merged volume/chapter coverage for the series. This + * plugin polls that feed, matches each item to a tracked Codex series by + * *exact* external ID (no fuzzy matching), and records release candidates. + * + * Unlike the per-series RSS plugins (MangaUpdates, Nyaa), the feed is not + * scoped to the user's tracked series — it's the whole Tsundoku catalog's + * recent activity. So each poll: + * 1. Loads the stored cursor from the plugin KV store. + * 2. Builds a reverse index `"provider:id" -> codexSeriesId` from the + * host's `releases/list_tracked` rows (scoped by `requiresExternalIds`). + * 3. Walks the feed from the cursor, matching each item against the index + * and streaming matches via `releases/record`. + * 4. Persists the advancing cursor back to the KV store. + * + * The feed walk and matching land in dedicated modules (`fetcher`, + * `matcher`, `candidate`); this entry point owns plugin lifecycle, config, + * source registration, and the poll orchestration that ties them together. + */ + +import { + createLogger, + createReleaseSourcePlugin, + type HostRpcClient, + HostRpcError, + type InitializeParams, + type PluginStorage, + RELEASES_METHODS, + type ReleasePollRequest, + type ReleasePollResponse, +} from "@ashdev/codex-plugin-sdk"; +import { manifest } from "./manifest.js"; + +const logger = createLogger({ name: manifest.name, level: "info" }); + +/** KV-store key under which the feed cursor bookmark is persisted. */ +export const CURSOR_STORAGE_KEY = "feed_cursor"; + +/** Default feed page size when config omits / mis-types `pageLimit`. */ +const DEFAULT_PAGE_LIMIT = 100; +/** Tsundoku caps the feed page size at 500. */ +const MAX_PAGE_LIMIT = 500; +/** Default per-request timeout when config omits / mis-types `requestTimeoutMs`. */ +const DEFAULT_TIMEOUT_MS = 10_000; +const MIN_TIMEOUT_MS = 1_000; +const MAX_TIMEOUT_MS = 60_000; +const DEFAULT_LANGUAGE = "en"; + +// ============================================================================= +// Plugin-level state (set during initialize) +// ============================================================================= + +interface PluginState { + hostRpc: HostRpcClient | null; + storage: PluginStorage | null; + /** Tsundoku instance base URL (no trailing slash), e.g. `https://t.example.com`. */ + baseUrl: string; + /** ISO 639-1 tag stamped on every candidate (the feed carries none). */ + defaultLanguage: string; + /** Feed page size (1..=MAX_PAGE_LIMIT). */ + pageLimit: number; + /** Hard timeout for a single feed-page fetch. */ + requestTimeoutMs: number; +} + +const state: PluginState = { + hostRpc: null, + storage: null, + baseUrl: "", + defaultLanguage: DEFAULT_LANGUAGE, + pageLimit: DEFAULT_PAGE_LIMIT, + requestTimeoutMs: DEFAULT_TIMEOUT_MS, +}; + +/** Reset state. Exported for tests; not part of the plugin contract. */ +export function _resetState(): void { + state.hostRpc = null; + state.storage = null; + state.baseUrl = ""; + state.defaultLanguage = DEFAULT_LANGUAGE; + state.pageLimit = DEFAULT_PAGE_LIMIT; + state.requestTimeoutMs = DEFAULT_TIMEOUT_MS; +} + +/** Strip a single trailing slash so URL building stays predictable. */ +export function normalizeBaseUrl(raw: string): string { + return raw.trim().replace(/\/+$/, ""); +} + +// ============================================================================= +// Source registration +// ============================================================================= + +/** + * Register the single static source row representing the Tsundoku feed. The + * whole catalog is polled under one logical source keyed `default`. Retries + * on `METHOD_NOT_FOUND` to absorb the brief race where the host has not yet + * installed the releases reverse-RPC handler at startup. + */ +export async function registerSources( + rpc: HostRpcClient, +): Promise<{ registered: number; pruned: number } | null> { + const sources = [ + { + sourceKey: "default", + displayName: "Tsundoku Releases", + kind: "api-feed" as const, + config: null, + }, + ]; + const maxAttempts = 5; + for (let attempt = 1; attempt <= maxAttempts; attempt++) { + try { + return await rpc.call<{ registered: number; pruned: number }>( + RELEASES_METHODS.REGISTER_SOURCES, + { sources }, + ); + } catch (err) { + const isMethodNotFound = err instanceof HostRpcError && err.code === -32601; + if (isMethodNotFound && attempt < maxAttempts) { + await new Promise((r) => setTimeout(r, 50 * attempt)); + continue; + } + const reason = err instanceof Error ? err.message : String(err); + logger.error(`register_sources failed: ${reason}`); + return null; + } + } + return null; +} + +// ============================================================================= +// Poll +// ============================================================================= + +/** + * Top-level poll handler. The feed walk + matching are added in dedicated + * modules; until they land, the source registers and polls cleanly while + * recording no candidates. Exported for tests. + */ +export async function poll( + _params: ReleasePollRequest, + _rpc: HostRpcClient, +): Promise { + return { + notModified: false, + upstreamStatus: 200, + parsed: 0, + matched: 0, + recorded: 0, + deduped: 0, + }; +} + +// ============================================================================= +// Plugin Initialization +// ============================================================================= + +createReleaseSourcePlugin({ + manifest, + provider: { + async poll(params: ReleasePollRequest): Promise { + if (!state.hostRpc) { + throw new Error("Plugin not initialized: hostRpc client missing"); + } + if (!state.baseUrl) { + throw new Error("Plugin not configured: baseUrl is required"); + } + return poll(params, state.hostRpc); + }, + }, + logLevel: "info", + async onInitialize(params: InitializeParams) { + state.hostRpc = params.hostRpc; + state.storage = params.storage; + + const ac = params.adminConfig ?? {}; + if (typeof ac.baseUrl === "string") { + state.baseUrl = normalizeBaseUrl(ac.baseUrl); + } + if (typeof ac.defaultLanguage === "string" && ac.defaultLanguage.trim().length > 0) { + state.defaultLanguage = ac.defaultLanguage.trim().toLowerCase(); + } + if (typeof ac.pageLimit === "number" && Number.isFinite(ac.pageLimit)) { + state.pageLimit = Math.max(1, Math.min(Math.trunc(ac.pageLimit), MAX_PAGE_LIMIT)); + } + if (typeof ac.requestTimeoutMs === "number" && Number.isFinite(ac.requestTimeoutMs)) { + state.requestTimeoutMs = Math.max( + MIN_TIMEOUT_MS, + Math.min(ac.requestTimeoutMs, MAX_TIMEOUT_MS), + ); + } + + if (!state.baseUrl) { + logger.warn( + "initialized without a baseUrl — set it in the plugin config; polls will error until then", + ); + } + logger.info( + `initialized: baseUrl=${state.baseUrl || "(unset)"} defaultLanguage=${state.defaultLanguage} pageLimit=${state.pageLimit} timeoutMs=${state.requestTimeoutMs}`, + ); + + // Materialize the single static source row. Deferred to a microtask so we + // run *after* the host installs the releases reverse-RPC handler. + queueMicrotask(() => { + void registerSources(params.hostRpc).then((result) => { + if (result) { + logger.info(`register_sources: registered=${result.registered} pruned=${result.pruned}`); + } + }); + }); + }, +}); + +logger.info("Tsundoku release-source plugin started"); diff --git a/plugins/release-tsundoku/src/manifest.ts b/plugins/release-tsundoku/src/manifest.ts new file mode 100644 index 00000000..0f3ba88c --- /dev/null +++ b/plugins/release-tsundoku/src/manifest.ts @@ -0,0 +1,94 @@ +import type { PluginManifest } from "@ashdev/codex-plugin-sdk"; +import packageJson from "../package.json" with { type: "json" }; + +/** + * External-ID source names Tsundoku exposes on each feed item, in match + * priority order (most canonical first). These are the *bare* service names + * (no `api:` / `plugin:` prefix): the host strips the stored prefix before + * filtering, so a manifest `requiresExternalIds` entry of `"mangabaka"` + * matches a series whose ID was stored as `api:mangabaka` or + * `plugin:mangabaka`. + * + * The order matters only for the candidate's `reason` string — the first + * provider that resolves a tracked series wins. MangaBaka is the dominant + * cross-reference hub in Codex, so it leads. + */ +export const TSUNDOKU_EXTERNAL_ID_SOURCES = [ + "mangabaka", + "anilist", + "mal", + "mangaupdates", + "kitsu", + "shikimori", + "anime_planet", + "anime_news_network", +] as const; + +export const manifest = { + name: "release-tsundoku", + displayName: "Tsundoku Releases", + version: packageJson.version, + description: + "Announces new volume/chapter coverage for tracked series via a Tsundoku instance's incremental series feed. Matches series by exact external IDs (no fuzzy matching) and walks the feed by cursor, persisting its position between polls.", + author: "Codex", + homepage: "https://github.com/AshDevFr/codex", + protocolVersion: "1.1", + capabilities: { + releaseSource: { + kinds: ["api-feed"], + requiresAliases: false, + requiresExternalIds: [...TSUNDOKU_EXTERNAL_ID_SOURCES], + canAnnounceChapters: true, + canAnnounceVolumes: true, + }, + }, + configSchema: { + description: + "Tsundoku plugin configuration. Point `baseUrl` at your Tsundoku instance; the plugin polls its public `/api/v1/series/feed` endpoint and matches results to your tracked series by external ID.", + fields: [ + { + key: "baseUrl", + label: "Tsundoku Base URL", + description: + "Base URL of the Tsundoku instance, e.g. `https://tsundoku.example.com`. The plugin appends `/api/v1/series/feed`. No trailing slash required.", + type: "string" as const, + required: true, + example: "https://tsundoku.example.com", + }, + { + key: "defaultLanguage", + label: "Default Language", + description: + "ISO 639-1 language tag stamped on every announcement. The Tsundoku feed tracks official release coverage and carries no language of its own, so a default is required. Per-series language preferences on each series' tracking config still gate the high-water mark host-side.", + type: "string" as const, + required: false, + default: "en", + example: "en", + }, + { + key: "pageLimit", + label: "Feed Page Size", + description: + "Items requested per feed page (1–500). Larger pages mean fewer round-trips when walking a long backlog. Defaults to 100.", + type: "number" as const, + required: false, + default: 100, + }, + { + key: "requestTimeoutMs", + label: "Request Timeout (ms)", + description: + "How long to wait for a single feed page before giving up. Defaults to 10000 (10 seconds).", + type: "number" as const, + required: false, + default: 10_000, + }, + ], + }, + userDescription: + "Announces new volumes and chapters for series you've tracked, using a Tsundoku instance as the source. Matches your series by external ID (MangaBaka, AniList, MAL, and more). Notification-only — Codex does not download anything.", + adminSetupInstructions: + "1. Set `baseUrl` to your Tsundoku instance URL (e.g. `https://tsundoku.example.com`) and save. The plugin auto-registers a single source row (`Tsundoku Releases`) in **Settings → Release tracking**, where you can disable it, change the poll interval, or hit *Poll now*. 2. To get announcements for a series, make sure it has at least one external ID Tsundoku also knows (MangaBaka, AniList, MAL, MangaUpdates, Kitsu, Shikimori, Anime-Planet, or Anime News Network) — populate these via a metadata refresh or by pasting them in the series tracking panel. 3. Optional: adjust `defaultLanguage` (default `en`), `pageLimit`, and `requestTimeoutMs`. The Tsundoku feed endpoint is public; no credentials are needed. Note: the feed is incremental, so newly tracked series only announce on their *next* Tsundoku coverage change.", +} as const satisfies PluginManifest & { + capabilities: { releaseSource: { kinds: ["api-feed"] } }; +}; diff --git a/plugins/release-tsundoku/tsconfig.json b/plugins/release-tsundoku/tsconfig.json new file mode 100644 index 00000000..ef1ca5f9 --- /dev/null +++ b/plugins/release-tsundoku/tsconfig.json @@ -0,0 +1,24 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "lib": ["ES2022"], + "outDir": "./dist", + "rootDir": "./src", + "declaration": true, + "sourceMap": true, + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "resolveJsonModule": true, + "isolatedModules": true, + "noUnusedLocals": true, + "noUnusedParameters": true, + "noImplicitReturns": true, + "noFallthroughCasesInSwitch": true + }, + "include": ["src/**/*"], + "exclude": ["node_modules", "dist", "**/*.test.ts"] +} diff --git a/plugins/release-tsundoku/vitest.config.ts b/plugins/release-tsundoku/vitest.config.ts new file mode 100644 index 00000000..ae847ff6 --- /dev/null +++ b/plugins/release-tsundoku/vitest.config.ts @@ -0,0 +1,7 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + include: ["src/**/*.test.ts"], + }, +}); diff --git a/web/src/pages/settings/plugins/OfficialPlugins.tsx b/web/src/pages/settings/plugins/OfficialPlugins.tsx index 3b1b33bd..99b7d87a 100644 --- a/web/src/pages/settings/plugins/OfficialPlugins.tsx +++ b/web/src/pages/settings/plugins/OfficialPlugins.tsx @@ -156,6 +156,22 @@ export const OFFICIAL_PLUGINS: OfficialPlugin[] = [ credentialDelivery: "env", }, }, + { + name: "release-tsundoku", + displayName: "Tsundoku Releases", + description: + "Announces new volume and chapter coverage for tracked series via a Tsundoku instance's incremental series feed. Matches series by exact external IDs (MangaBaka, AniList, MAL, and more) — no fuzzy matching. Notify-only — Codex does not download anything.", + type: "Releases", + packageName: "@ashdev/codex-plugin-release-tsundoku", + authInfo: "Tsundoku instance URL required for setup", + author: "Codex Team", + scope: "system", + formDefaults: { + command: "npx", + args: "-y\n@ashdev/codex-plugin-release-tsundoku", + credentialDelivery: "env", + }, + }, ]; // --------------------------------------------------------------------------- From 1071f2c81742168a88ac83b9a025121f0aa5f79f Mon Sep 17 00:00:00 2001 From: Sylvain Cau Date: Mon, 8 Jun 2026 15:37:52 -0700 Subject: [PATCH 02/10] feat(plugins): add tsundoku feed fetcher and cursor persistence Add the building blocks for walking the Tsundoku series feed: wire types for the feed response, a feedUrl builder, and fetchFeedPage, which wraps fetch with a hard timeout and JSON parsing and returns a discriminated ok/error result (passing upstream status through for host-side backoff, and rejecting malformed bodies). Add loadCursor/saveCursor over the plugin KV store so the feed position survives between polls. Both are tolerant of a missing key and storage errors: a failed read restarts the walk from the beginning and a failed write is logged without aborting the poll, which is safe given keyset pagination plus host-side dedup. Includes tests for the fetcher and cursor helpers. --- plugins/release-tsundoku/src/fetcher.test.ts | 151 +++++++++++++++++ plugins/release-tsundoku/src/fetcher.ts | 165 +++++++++++++++++++ plugins/release-tsundoku/src/index.test.ts | 74 ++++++++- plugins/release-tsundoku/src/index.ts | 36 ++++ 4 files changed, 423 insertions(+), 3 deletions(-) create mode 100644 plugins/release-tsundoku/src/fetcher.test.ts create mode 100644 plugins/release-tsundoku/src/fetcher.ts diff --git a/plugins/release-tsundoku/src/fetcher.test.ts b/plugins/release-tsundoku/src/fetcher.test.ts new file mode 100644 index 00000000..e7b0d730 --- /dev/null +++ b/plugins/release-tsundoku/src/fetcher.test.ts @@ -0,0 +1,151 @@ +import { describe, expect, it, vi } from "vitest"; +import { type FeedResponse, feedUrl, fetchFeedPage } from "./fetcher.js"; + +// ----------------------------------------------------------------------------- +// feedUrl +// ----------------------------------------------------------------------------- + +describe("feedUrl", () => { + it("appends the feed path and limit", () => { + const url = feedUrl("https://t.example.com", null, 100); + expect(url).toBe("https://t.example.com/api/v1/series/feed?limit=100"); + }); + + it("includes the cursor when provided", () => { + const url = feedUrl("https://t.example.com", "abc123", 50); + const parsed = new URL(url); + expect(parsed.pathname).toBe("/api/v1/series/feed"); + expect(parsed.searchParams.get("limit")).toBe("50"); + expect(parsed.searchParams.get("cursor")).toBe("abc123"); + }); + + it("omits the cursor param when null or empty", () => { + expect(feedUrl("https://t.example.com", "", 10)).not.toContain("cursor="); + expect(feedUrl("https://t.example.com", null, 10)).not.toContain("cursor="); + }); + + it("strips trailing slashes from the base URL", () => { + expect(feedUrl("https://t.example.com///", null, 10)).toBe( + "https://t.example.com/api/v1/series/feed?limit=10", + ); + }); + + it("url-encodes an opaque cursor", () => { + const url = feedUrl("https://t.example.com", "a b/c+d", 10); + expect(new URL(url).searchParams.get("cursor")).toBe("a b/c+d"); + }); +}); + +// ----------------------------------------------------------------------------- +// fetchFeedPage +// ----------------------------------------------------------------------------- + +const samplePage: FeedResponse = { + items: [ + { + seriesId: 87, + canonicalTitle: "Example Series", + externalIds: [{ provider: "mangabaka", externalId: "9741", fetchedAt: 1_780_943_416 }], + volumeCoverage: [{ start: 1, end: 16 }], + chapterCoverage: [], + highestVolume: 16, + highestChapter: null, + updatedAt: 1_780_943_416, + }, + ], + hasMore: true, + nextCursor: "next-cursor-token", +}; + +function jsonResponse(body: unknown, status = 200): Response { + return new Response(JSON.stringify(body), { + status, + headers: { "content-type": "application/json" }, + }); +} + +describe("fetchFeedPage", () => { + it("returns ok with the parsed page on 200", async () => { + const fetchImpl = vi + .fn() + .mockResolvedValue(jsonResponse(samplePage)) as unknown as typeof fetch; + const result = await fetchFeedPage("https://t.example.com", null, 100, { fetchImpl }); + + expect(result.kind).toBe("ok"); + if (result.kind !== "ok") throw new Error("expected ok"); + expect(result.data.hasMore).toBe(true); + expect(result.data.nextCursor).toBe("next-cursor-token"); + expect(result.data.items).toHaveLength(1); + expect(result.data.items[0].seriesId).toBe(87); + }); + + it("sends the cursor and limit in the request URL", async () => { + const fetchImpl = vi + .fn() + .mockResolvedValue(jsonResponse(samplePage)) as unknown as typeof fetch; + await fetchFeedPage("https://t.example.com", "cur-1", 250, { fetchImpl }); + + const calledUrl = (fetchImpl as unknown as ReturnType).mock.calls[0][0] as string; + const parsed = new URL(calledUrl); + expect(parsed.searchParams.get("cursor")).toBe("cur-1"); + expect(parsed.searchParams.get("limit")).toBe("250"); + }); + + it("requests JSON via the Accept header", async () => { + const fetchImpl = vi + .fn() + .mockResolvedValue(jsonResponse(samplePage)) as unknown as typeof fetch; + await fetchFeedPage("https://t.example.com", null, 100, { fetchImpl }); + + const init = (fetchImpl as unknown as ReturnType).mock.calls[0][1] as RequestInit; + expect((init.headers as Record).Accept).toBe("application/json"); + }); + + it("maps a non-200 status to an error result", async () => { + const fetchImpl = vi + .fn() + .mockResolvedValue(new Response("nope", { status: 503 })) as unknown as typeof fetch; + const result = await fetchFeedPage("https://t.example.com", null, 100, { fetchImpl }); + + expect(result.kind).toBe("error"); + if (result.kind !== "error") throw new Error("expected error"); + expect(result.status).toBe(503); + }); + + it("maps a network throw to status 0", async () => { + const fetchImpl = vi + .fn() + .mockRejectedValue(new Error("ECONNREFUSED")) as unknown as typeof fetch; + const result = await fetchFeedPage("https://t.example.com", null, 100, { fetchImpl }); + + expect(result.kind).toBe("error"); + if (result.kind !== "error") throw new Error("expected error"); + expect(result.status).toBe(0); + expect(result.message).toContain("ECONNREFUSED"); + }); + + it("errors on a 200 with invalid JSON", async () => { + const fetchImpl = vi + .fn() + .mockResolvedValue( + new Response("not json", { status: 200, headers: { "content-type": "application/json" } }), + ) as unknown as typeof fetch; + const result = await fetchFeedPage("https://t.example.com", null, 100, { fetchImpl }); + + expect(result.kind).toBe("error"); + if (result.kind !== "error") throw new Error("expected error"); + expect(result.status).toBe(200); + expect(result.message).toContain("parse"); + }); + + it("errors on a 200 whose body is missing items[]", async () => { + const fetchImpl = vi + .fn() + .mockResolvedValue(jsonResponse({ hasMore: false })) as unknown as typeof fetch; + const result = await fetchFeedPage("https://t.example.com", null, 100, { fetchImpl }); + + expect(result.kind).toBe("error"); + if (result.kind !== "error") throw new Error("expected error"); + expect(result.message).toContain("malformed"); + }); +}); diff --git a/plugins/release-tsundoku/src/fetcher.ts b/plugins/release-tsundoku/src/fetcher.ts new file mode 100644 index 00000000..54a0ac65 --- /dev/null +++ b/plugins/release-tsundoku/src/fetcher.ts @@ -0,0 +1,165 @@ +/** + * Tsundoku series-feed fetcher. + * + * Wraps `fetch` against `GET {baseUrl}/api/v1/series/feed` with a hard + * timeout and JSON parsing, returning a discriminated result so the caller + * can act on a parsed page (`ok`) or surface the upstream status back to the + * host's per-host backoff layer (`error`). + * + * The feed is keyset-paginated: pass the previous response's `nextCursor` + * back as `cursor` and walk while `hasMore` is true. Network and parsing are + * the only side effects; nothing here touches storage, the host, or process + * state, which keeps it trivially testable with a mocked `fetch`. + */ + +// ============================================================================= +// Wire types (mirror Tsundoku's SeriesFeedResponse / SeriesFeedItem) +// ============================================================================= + +/** One provider mapping on a feed item (e.g. `{ provider: "mangabaka", ... }`). */ +export interface FeedExternalId { + provider: string; + externalId: string; + /** Epoch seconds the mapping was last fetched upstream. */ + fetchedAt: number; +} + +/** One inclusive `[start, end]` coverage range (single values are `start === end`). */ +export interface FeedCoverageSpan { + start: number; + end: number; +} + +/** One series in the incremental release feed. */ +export interface FeedItem { + seriesId: number; + canonicalTitle: string; + /** Provider mappings the consumer matches on. */ + externalIds: FeedExternalId[]; + /** Merged available volume ranges (sorted, gaps preserved). */ + volumeCoverage: FeedCoverageSpan[]; + /** Merged available chapter ranges (sorted, gaps preserved). */ + chapterCoverage: FeedCoverageSpan[]; + /** Max end of `volumeCoverage`, or null when there is none. */ + highestVolume: number | null; + /** Max end of `chapterCoverage`, or null when there is none. */ + highestChapter: number | null; + /** Epoch seconds this series' coverage last changed (the cursor key). */ + updatedAt: number; +} + +/** One page of the feed. */ +export interface FeedResponse { + items: FeedItem[]; + /** `true` when more series remain after this page (fetch again now). */ + hasMore: boolean; + /** Opaque cursor at the last item, or null/absent when the page is empty. */ + nextCursor?: string | null; +} + +// ============================================================================= +// Fetch result + options +// ============================================================================= + +/** Discriminated fetch result. */ +export type FeedFetchResult = + | { kind: "ok"; data: FeedResponse; status: 200 } + | { kind: "error"; status: number; message: string }; + +export interface FeedFetcherOptions { + /** Custom `fetch` impl (for testing). Defaults to global `fetch`. */ + fetchImpl?: typeof fetch; + /** Per-request timeout. Defaults to 10s. */ + timeoutMs?: number; +} + +/** Feed endpoint path appended to the configured base URL. */ +export const FEED_PATH = "/api/v1/series/feed"; + +const DEFAULT_TIMEOUT_MS = 10_000; + +/** + * Build the feed URL for a page. Defensively strips trailing slashes off + * `baseUrl` so callers don't have to. `limit` is always sent; `cursor` is + * sent only when non-empty (its absence starts the walk from the beginning). + */ +export function feedUrl(baseUrl: string, cursor: string | null, limit: number): string { + const base = baseUrl.replace(/\/+$/, ""); + const params = new URLSearchParams(); + params.set("limit", String(limit)); + if (cursor) { + params.set("cursor", cursor); + } + return `${base}${FEED_PATH}?${params.toString()}`; +} + +/** + * Fetch one page of the Tsundoku series feed. + * + * @param baseUrl - Tsundoku instance base URL (trailing slash tolerated). + * @param cursor - Cursor from the previous page, or null to start over. + * @param limit - Page size (the caller is responsible for clamping to 1..=500). + * @param opts - Fetcher options (custom fetch, timeout). + */ +export async function fetchFeedPage( + baseUrl: string, + cursor: string | null, + limit: number, + opts: FeedFetcherOptions = {}, +): Promise { + const fetchImpl = opts.fetchImpl ?? globalThis.fetch; + const timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS; + + const url = feedUrl(baseUrl, cursor, limit); + const headers: Record = { + Accept: "application/json", + "User-Agent": "Codex-ReleaseTracker/1.0 (+https://github.com/AshDevFr/codex)", + }; + + // AbortSignal.timeout is the cleanest path; we already require Node 22+. + const signal = AbortSignal.timeout(timeoutMs); + + let resp: Response; + try { + resp = await fetchImpl(url, { method: "GET", headers, signal }); + } catch (err) { + const msg = err instanceof Error ? err.message : "Unknown fetch error"; + // Aborts and transport-level failures map to 0/unavailable so the host's + // per-host backoff can react without us inventing a fake HTTP status. + return { kind: "error", status: 0, message: msg }; + } + + if (resp.status !== 200) { + // Pass through 429 / 5xx so the host's backoff layer sees the real status. + return { + kind: "error", + status: resp.status, + message: `upstream returned ${resp.status} ${resp.statusText}`.trim(), + }; + } + + let parsed: unknown; + try { + parsed = await resp.json(); + } catch (err) { + const msg = err instanceof Error ? err.message : "invalid JSON"; + return { kind: "error", status: 200, message: `failed to parse feed JSON: ${msg}` }; + } + + if (!isFeedResponse(parsed)) { + return { kind: "error", status: 200, message: "malformed feed response: missing items[]" }; + } + + return { kind: "ok", data: parsed, status: 200 }; +} + +/** + * Minimal structural guard: a valid page must carry an `items` array and a + * boolean `hasMore`. We don't deep-validate each item — the matcher tolerates + * missing fields per-item rather than failing the whole page. + */ +function isFeedResponse(value: unknown): value is FeedResponse { + if (value === null || typeof value !== "object") return false; + const obj = value as Record; + return Array.isArray(obj.items) && typeof obj.hasMore === "boolean"; +} diff --git a/plugins/release-tsundoku/src/index.test.ts b/plugins/release-tsundoku/src/index.test.ts index 8b4befcd..0ff2819b 100644 --- a/plugins/release-tsundoku/src/index.test.ts +++ b/plugins/release-tsundoku/src/index.test.ts @@ -1,6 +1,12 @@ -import { HostRpcClient } from "@ashdev/codex-plugin-sdk"; -import { describe, expect, it } from "vitest"; -import { normalizeBaseUrl, registerSources } from "./index.js"; +import { HostRpcClient, type PluginStorage } from "@ashdev/codex-plugin-sdk"; +import { describe, expect, it, vi } from "vitest"; +import { + CURSOR_STORAGE_KEY, + loadCursor, + normalizeBaseUrl, + registerSources, + saveCursor, +} from "./index.js"; // ----------------------------------------------------------------------------- // Mock host RPC @@ -74,6 +80,68 @@ describe("normalizeBaseUrl", () => { }); }); +// ----------------------------------------------------------------------------- +// Cursor persistence +// ----------------------------------------------------------------------------- + +/** Minimal in-memory `PluginStorage` double exposing only get/set. */ +function makeFakeStorage(initial?: unknown): { + storage: PluginStorage; + get: ReturnType; + set: ReturnType; +} { + const get = vi.fn(async () => ({ data: initial ?? null })); + const set = vi.fn(async () => ({ success: true })); + const storage = { get, set } as unknown as PluginStorage; + return { storage, get, set }; +} + +describe("loadCursor", () => { + it("returns the stored cursor string", async () => { + const { storage, get } = makeFakeStorage("cursor-42"); + expect(await loadCursor(storage)).toBe("cursor-42"); + expect(get).toHaveBeenCalledWith(CURSOR_STORAGE_KEY); + }); + + it("returns null when no cursor is stored", async () => { + const { storage } = makeFakeStorage(null); + expect(await loadCursor(storage)).toBeNull(); + }); + + it("returns null for a non-string / empty stored value", async () => { + expect(await loadCursor(makeFakeStorage("").storage)).toBeNull(); + expect(await loadCursor(makeFakeStorage(123).storage)).toBeNull(); + }); + + it("returns null and does not throw when the read fails", async () => { + const storage = { + get: vi.fn(async () => { + throw new Error("kv down"); + }), + set: vi.fn(), + } as unknown as PluginStorage; + expect(await loadCursor(storage)).toBeNull(); + }); +}); + +describe("saveCursor", () => { + it("writes the cursor under the feed-cursor key", async () => { + const { storage, set } = makeFakeStorage(); + await saveCursor(storage, "cursor-99"); + expect(set).toHaveBeenCalledWith(CURSOR_STORAGE_KEY, "cursor-99"); + }); + + it("swallows a write failure without throwing", async () => { + const storage = { + get: vi.fn(), + set: vi.fn(async () => { + throw new Error("kv full"); + }), + } as unknown as PluginStorage; + await expect(saveCursor(storage, "cursor-99")).resolves.toBeUndefined(); + }); +}); + // ----------------------------------------------------------------------------- // registerSources // ----------------------------------------------------------------------------- diff --git a/plugins/release-tsundoku/src/index.ts b/plugins/release-tsundoku/src/index.ts index e4c1ae69..935e2542 100644 --- a/plugins/release-tsundoku/src/index.ts +++ b/plugins/release-tsundoku/src/index.ts @@ -92,6 +92,42 @@ export function normalizeBaseUrl(raw: string): string { return raw.trim().replace(/\/+$/, ""); } +// ============================================================================= +// Cursor persistence (plugin KV store) +// ============================================================================= + +/** + * Load the feed cursor bookmark from the KV store. Returns `null` when no + * cursor has been stored yet (first run) or when the read fails — a missing + * cursor simply restarts the walk from the beginning, which is safe given + * at-least-once delivery + host-side dedup. + */ +export async function loadCursor(storage: PluginStorage): Promise { + try { + const res = await storage.get(CURSOR_STORAGE_KEY); + const data = res?.data; + return typeof data === "string" && data.length > 0 ? data : null; + } catch (err) { + const reason = err instanceof Error ? err.message : String(err); + logger.warn(`failed to load cursor; restarting from the beginning: ${reason}`); + return null; + } +} + +/** + * Persist the feed cursor bookmark. Best-effort: a failed write is logged but + * never aborts a poll — the worst case is re-walking already-seen pages on + * the next poll, which dedups host-side. + */ +export async function saveCursor(storage: PluginStorage, cursor: string): Promise { + try { + await storage.set(CURSOR_STORAGE_KEY, cursor); + } catch (err) { + const reason = err instanceof Error ? err.message : String(err); + logger.warn(`failed to persist cursor "${cursor}": ${reason}`); + } +} + // ============================================================================= // Source registration // ============================================================================= From 2498c3733b671f1d7bf9a2d344a8d644d96d22d6 Mon Sep 17 00:00:00 2001 From: Sylvain Cau Date: Mon, 8 Jun 2026 15:45:30 -0700 Subject: [PATCH 03/10] feat(plugins): add tsundoku external-id matching and candidate mapping Add the matcher that resolves Tsundoku feed items to tracked Codex series by exact external ID. buildIndex turns the host's tracked-series rows into a "provider:id" -> series reverse index, and matchItem sweeps an item's provider IDs in priority order (mangabaka first) for a stable, fuzzy-free match. Add the candidate mapping that turns a matched feed item into a release candidate: coverage spans pass through onto the volume/chapter axes, the external release id is keyed on the coverage high-water mark so a new row fires only when the frontier advances, and confidence is fixed at 1.0 since the match is exact. Includes tests for both modules. --- .../release-tsundoku/src/candidate.test.ts | 150 ++++++++++++++++++ plugins/release-tsundoku/src/candidate.ts | 86 ++++++++++ plugins/release-tsundoku/src/matcher.test.ts | 101 ++++++++++++ plugins/release-tsundoku/src/matcher.ts | 82 ++++++++++ 4 files changed, 419 insertions(+) create mode 100644 plugins/release-tsundoku/src/candidate.test.ts create mode 100644 plugins/release-tsundoku/src/candidate.ts create mode 100644 plugins/release-tsundoku/src/matcher.test.ts create mode 100644 plugins/release-tsundoku/src/matcher.ts diff --git a/plugins/release-tsundoku/src/candidate.test.ts b/plugins/release-tsundoku/src/candidate.test.ts new file mode 100644 index 00000000..b16b247b --- /dev/null +++ b/plugins/release-tsundoku/src/candidate.test.ts @@ -0,0 +1,150 @@ +import { describe, expect, it } from "vitest"; +import { externalReleaseId, feedItemToCandidate, toSpans } from "./candidate.js"; +import type { FeedItem } from "./fetcher.js"; +import type { MatchResult } from "./matcher.js"; + +// ----------------------------------------------------------------------------- +// Helpers +// ----------------------------------------------------------------------------- + +function feedItem(overrides: Partial = {}): FeedItem { + return { + seriesId: 87, + canonicalTitle: "Example Series", + externalIds: [{ provider: "mangabaka", externalId: "9741", fetchedAt: 1_780_943_416 }], + volumeCoverage: [{ start: 1, end: 16 }], + chapterCoverage: [], + highestVolume: 16, + highestChapter: null, + updatedAt: 1_780_943_416, + ...overrides, + }; +} + +const match: MatchResult = { + codexSeriesId: "uuid-a", + provider: "mangabaka", + externalId: "9741", +}; + +const opts = { + baseUrl: "https://t.example.com", + language: "en", + observedAt: "2026-06-08T00:00:00.000Z", +}; + +// ----------------------------------------------------------------------------- +// toSpans +// ----------------------------------------------------------------------------- + +describe("toSpans", () => { + it("maps coverage spans verbatim", () => { + expect( + toSpans([ + { start: 1, end: 16 }, + { start: 18, end: 20 }, + ]), + ).toEqual([ + { start: 1, end: 16 }, + { start: 18, end: 20 }, + ]); + }); + + it("returns null for an empty coverage list", () => { + expect(toSpans([])).toBeNull(); + }); + + it("preserves decimal chapter spans", () => { + expect(toSpans([{ start: 1, end: 45.5 }])).toEqual([{ start: 1, end: 45.5 }]); + }); +}); + +// ----------------------------------------------------------------------------- +// externalReleaseId +// ----------------------------------------------------------------------------- + +describe("externalReleaseId", () => { + it("keys on series id and both high-water marks", () => { + expect(externalReleaseId(feedItem({ highestVolume: 16, highestChapter: 45 }))).toBe( + "tsundoku:87:v16:c45", + ); + }); + + it("renders null high-water values as a dash", () => { + expect(externalReleaseId(feedItem({ highestVolume: null, highestChapter: null }))).toBe( + "tsundoku:87:v-:c-", + ); + expect(externalReleaseId(feedItem({ highestVolume: 16, highestChapter: null }))).toBe( + "tsundoku:87:v16:c-", + ); + }); + + it("is stable across re-delivery of the same coverage", () => { + expect(externalReleaseId(feedItem())).toBe(externalReleaseId(feedItem())); + }); + + it("changes when the frontier advances", () => { + expect(externalReleaseId(feedItem({ highestVolume: 16 }))).not.toBe( + externalReleaseId(feedItem({ highestVolume: 17 })), + ); + }); +}); + +// ----------------------------------------------------------------------------- +// feedItemToCandidate +// ----------------------------------------------------------------------------- + +describe("feedItemToCandidate", () => { + it("builds an exact-match candidate (confidence 1.0)", () => { + const c = feedItemToCandidate(feedItem(), match, opts); + expect(c.seriesMatch).toEqual({ + codexSeriesId: "uuid-a", + confidence: 1.0, + reason: "tsundoku:mangabaka:9741", + }); + expect(c.externalReleaseId).toBe("tsundoku:87:v16:c-"); + expect(c.language).toBe("en"); + expect(c.groupOrUploader).toBeNull(); + }); + + it("maps coverage onto volume/chapter axes (empty -> null)", () => { + const c = feedItemToCandidate( + feedItem({ + volumeCoverage: [{ start: 1, end: 4 }], + chapterCoverage: [{ start: 1, end: 21 }], + }), + match, + opts, + ); + expect(c.volumes).toEqual([{ start: 1, end: 4 }]); + expect(c.chapters).toEqual([{ start: 1, end: 21 }]); + + const volumeOnly = feedItemToCandidate(feedItem({ chapterCoverage: [] }), match, opts); + expect(volumeOnly.chapters).toBeNull(); + expect(volumeOnly.volumes).toEqual([{ start: 1, end: 16 }]); + }); + + it("builds the series landing URL and tolerates a trailing slash on baseUrl", () => { + const c = feedItemToCandidate(feedItem(), match, { + ...opts, + baseUrl: "https://t.example.com/", + }); + expect(c.payloadUrl).toBe("https://t.example.com/series/87"); + }); + + it("derives releasedAt from updatedAt (epoch seconds) and uses observedAt", () => { + const c = feedItemToCandidate(feedItem({ updatedAt: 1_780_943_416 }), match, opts); + expect(c.releasedAt).toBe(new Date(1_780_943_416 * 1000).toISOString()); + expect(c.observedAt).toBe("2026-06-08T00:00:00.000Z"); + }); + + it("carries Tsundoku context in metadata", () => { + const c = feedItemToCandidate(feedItem(), match, opts); + expect(c.metadata).toEqual({ + tsundokuSeriesId: 87, + canonicalTitle: "Example Series", + highestVolume: 16, + highestChapter: null, + }); + }); +}); diff --git a/plugins/release-tsundoku/src/candidate.ts b/plugins/release-tsundoku/src/candidate.ts new file mode 100644 index 00000000..c2399647 --- /dev/null +++ b/plugins/release-tsundoku/src/candidate.ts @@ -0,0 +1,86 @@ +/** + * Map a matched Tsundoku feed item to a Codex `ReleaseCandidate`. + * + * The feed already carries merged, gap-preserving coverage spans that line up + * with Codex's `NumericSpan` model, so the volume/chapter axes pass through + * verbatim. The candidate's `externalReleaseId` is keyed on the coverage + * high-water mark, so a new ledger row (and announcement) fires only when the + * frontier advances — re-delivery of the same coverage dedups host-side, and + * the host's auto-ignore + `latest_known_*` gate handle "already owned". + */ + +import type { ReleaseCandidate } from "@ashdev/codex-plugin-sdk"; +import type { FeedCoverageSpan, FeedItem } from "./fetcher.js"; +import type { MatchResult } from "./matcher.js"; + +// `FeedCoverageSpan` is structurally identical to the SDK's `NumericSpan` +// (`{ start, end }`), so a span list assigns directly to a candidate's +// `volumes` / `chapters` without a separate type or an SDK barrel export. + +/** Inputs the candidate mapping needs beyond the feed item + match. */ +export interface CandidateOptions { + /** Tsundoku base URL (trailing slash tolerated) for building the landing link. */ + baseUrl: string; + /** ISO 639-1 language stamped on the candidate (the feed carries none). */ + language: string; + /** Detection timestamp (ISO-8601). Defaults to now; injectable for tests. */ + observedAt?: string; +} + +/** + * Convert a feed coverage list to a `NumericSpan[]`, or `null` when empty. + * Coverage is already merged + sorted upstream, so this is a structural copy. + */ +export function toSpans(coverage: FeedCoverageSpan[]): FeedCoverageSpan[] | null { + if (coverage.length === 0) return null; + return coverage.map((s) => ({ start: s.start, end: s.end })); +} + +/** Format a high-water value for the dedup key (`null` -> `-`). */ +function fmtHighwater(value: number | null): string { + return value === null ? "-" : String(value); +} + +/** + * Stable per-source dedup key. Keyed on the coverage high-water mark so the + * same frontier re-delivers to the same `(sourceId, externalReleaseId)` ledger + * row (a no-op dedup), while a genuine advance produces a new row. + */ +export function externalReleaseId(item: FeedItem): string { + return `tsundoku:${item.seriesId}:v${fmtHighwater(item.highestVolume)}:c${fmtHighwater(item.highestChapter)}`; +} + +/** + * Build a `ReleaseCandidate` for a matched feed item. Confidence is 1.0 — the + * match is an exact external-ID hit, never fuzzy. + */ +export function feedItemToCandidate( + item: FeedItem, + match: MatchResult, + opts: CandidateOptions, +): ReleaseCandidate { + const base = opts.baseUrl.replace(/\/+$/, ""); + return { + seriesMatch: { + codexSeriesId: match.codexSeriesId, + confidence: 1.0, + reason: `tsundoku:${match.provider}:${match.externalId}`, + }, + externalReleaseId: externalReleaseId(item), + volumes: toSpans(item.volumeCoverage), + chapters: toSpans(item.chapterCoverage), + language: opts.language, + groupOrUploader: null, + payloadUrl: `${base}/series/${item.seriesId}`, + observedAt: opts.observedAt ?? new Date().toISOString(), + // Tsundoku's `updatedAt` is epoch seconds; a coverage change is the closest + // thing the feed has to a publish date. Not skew-checked host-side. + releasedAt: new Date(item.updatedAt * 1000).toISOString(), + metadata: { + tsundokuSeriesId: item.seriesId, + canonicalTitle: item.canonicalTitle, + highestVolume: item.highestVolume, + highestChapter: item.highestChapter, + }, + }; +} diff --git a/plugins/release-tsundoku/src/matcher.test.ts b/plugins/release-tsundoku/src/matcher.test.ts new file mode 100644 index 00000000..7af2277f --- /dev/null +++ b/plugins/release-tsundoku/src/matcher.test.ts @@ -0,0 +1,101 @@ +import type { TrackedSeriesEntry } from "@ashdev/codex-plugin-sdk"; +import { describe, expect, it } from "vitest"; +import type { FeedExternalId, FeedItem } from "./fetcher.js"; +import { buildIndex, matchItem } from "./matcher.js"; + +// ----------------------------------------------------------------------------- +// Helpers +// ----------------------------------------------------------------------------- + +function tracked(seriesId: string, externalIds?: Record): TrackedSeriesEntry { + return externalIds ? { seriesId, externalIds } : { seriesId }; +} + +function feedItem(externalIds: FeedExternalId[], seriesId = 1): FeedItem { + return { + seriesId, + canonicalTitle: "T", + externalIds, + volumeCoverage: [], + chapterCoverage: [], + highestVolume: null, + highestChapter: null, + updatedAt: 1_700_000_000, + }; +} + +function ext(provider: string, externalId: string): FeedExternalId { + return { provider, externalId, fetchedAt: 1_700_000_000 }; +} + +// ----------------------------------------------------------------------------- +// buildIndex +// ----------------------------------------------------------------------------- + +describe("buildIndex", () => { + it("indexes each provider/id pair to its series", () => { + const index = buildIndex([ + tracked("uuid-a", { mangabaka: "9741", anilist: "122180" }), + tracked("uuid-b", { mal: "128555" }), + ]); + expect(index.get("mangabaka:9741")).toBe("uuid-a"); + expect(index.get("anilist:122180")).toBe("uuid-a"); + expect(index.get("mal:128555")).toBe("uuid-b"); + expect(index.size).toBe(3); + }); + + it("skips entries without external IDs", () => { + const index = buildIndex([tracked("uuid-a"), tracked("uuid-b", {})]); + expect(index.size).toBe(0); + }); + + it("ignores empty external-id values", () => { + const index = buildIndex([tracked("uuid-a", { mangabaka: "" })]); + expect(index.size).toBe(0); + }); +}); + +// ----------------------------------------------------------------------------- +// matchItem +// ----------------------------------------------------------------------------- + +describe("matchItem", () => { + it("matches on a single provider id", () => { + const index = buildIndex([tracked("uuid-a", { mangabaka: "9741" })]); + const result = matchItem(feedItem([ext("mangabaka", "9741")]), index); + expect(result).toEqual({ codexSeriesId: "uuid-a", provider: "mangabaka", externalId: "9741" }); + }); + + it("returns null when no provider id is tracked", () => { + const index = buildIndex([tracked("uuid-a", { mangabaka: "9741" })]); + expect(matchItem(feedItem([ext("mangabaka", "0000")]), index)).toBeNull(); + expect(matchItem(feedItem([ext("anilist", "9741")]), index)).toBeNull(); + }); + + it("prefers the highest-priority provider when several would match", () => { + // The same series is tracked under both mangabaka and mal; mangabaka leads + // the priority order, so it should win regardless of array order on the item. + const index = buildIndex([tracked("uuid-a", { mangabaka: "9741", mal: "128555" })]); + const result = matchItem(feedItem([ext("mal", "128555"), ext("mangabaka", "9741")]), index); + expect(result?.provider).toBe("mangabaka"); + }); + + it("falls through to a lower-priority provider when the leader misses", () => { + const index = buildIndex([tracked("uuid-a", { mal: "128555" })]); + const result = matchItem( + feedItem([ext("mangabaka", "not-tracked"), ext("mal", "128555")]), + index, + ); + expect(result).toEqual({ codexSeriesId: "uuid-a", provider: "mal", externalId: "128555" }); + }); + + it("ignores providers outside the supported set", () => { + const index = new Map([["someunknownprovider:1", "uuid-a"]]); + expect(matchItem(feedItem([ext("someunknownprovider", "1")]), index)).toBeNull(); + }); + + it("returns null for an item with no external ids", () => { + const index = buildIndex([tracked("uuid-a", { mangabaka: "9741" })]); + expect(matchItem(feedItem([]), index)).toBeNull(); + }); +}); diff --git a/plugins/release-tsundoku/src/matcher.ts b/plugins/release-tsundoku/src/matcher.ts new file mode 100644 index 00000000..91e7242c --- /dev/null +++ b/plugins/release-tsundoku/src/matcher.ts @@ -0,0 +1,82 @@ +/** + * Exact external-ID matching between the Tsundoku feed and Codex's tracked + * series. + * + * Codex's `releases/record` path keys on a `codexSeriesId` (UUID), not on + * external IDs — so matching happens here, plugin-side, with zero fuzzy + * logic. The host returns each tracked series' provider IDs via + * `releases/list_tracked` (scoped by the manifest's `requiresExternalIds`, + * prefix-stripped to bare provider names). We index those into + * `"provider:id" -> codexSeriesId`, then resolve each feed item by looking + * up its own provider IDs in priority order. The first provider that hits + * wins; the match is exact, so the candidate's confidence is always 1.0. + */ + +import type { TrackedSeriesEntry } from "@ashdev/codex-plugin-sdk"; +import type { FeedItem } from "./fetcher.js"; +import { TSUNDOKU_EXTERNAL_ID_SOURCES } from "./manifest.js"; + +/** Result of resolving a feed item to a tracked Codex series. */ +export interface MatchResult { + /** The Codex series UUID the candidate should be recorded against. */ + codexSeriesId: string; + /** The provider whose ID produced the match (for the candidate `reason`). */ + provider: string; + /** The external ID value that matched. */ + externalId: string; +} + +/** Compose the index key for a `(provider, externalId)` pair. */ +function indexKey(provider: string, externalId: string): string { + return `${provider}:${externalId}`; +} + +/** + * Build a reverse index `"provider:id" -> codexSeriesId` from the host's + * tracked-series rows. Entries without external IDs contribute nothing. + * + * If two tracked series somehow share the same `(provider, id)` (shouldn't + * happen — provider IDs are unique per series), the later entry wins. That's + * an arbitrary-but-deterministic tie-break for a degenerate input. + */ +export function buildIndex(entries: TrackedSeriesEntry[]): Map { + const index = new Map(); + for (const entry of entries) { + const ids = entry.externalIds; + if (!ids) continue; + for (const [provider, externalId] of Object.entries(ids)) { + if (!externalId) continue; + index.set(indexKey(provider, externalId), entry.seriesId); + } + } + return index; +} + +/** + * Resolve a feed item against the reverse index. Providers are tried in the + * manifest's declared priority order (`TSUNDOKU_EXTERNAL_ID_SOURCES`, most + * canonical first) so the `reason` is stable when an item carries several + * matchable IDs. Returns `null` when no provider ID hits the index — the + * common case, since the feed spans the whole Tsundoku catalog and the user + * only tracks a slice of it. + */ +export function matchItem(item: FeedItem, index: Map): MatchResult | null { + // Collapse the item's external-ID array into a provider -> id lookup so the + // priority sweep below is O(providers) rather than O(providers * ids). + const byProvider = new Map(); + for (const ext of item.externalIds) { + if (ext.externalId) { + byProvider.set(ext.provider, ext.externalId); + } + } + + for (const provider of TSUNDOKU_EXTERNAL_ID_SOURCES) { + const externalId = byProvider.get(provider); + if (externalId === undefined) continue; + const codexSeriesId = index.get(indexKey(provider, externalId)); + if (codexSeriesId !== undefined) { + return { codexSeriesId, provider, externalId }; + } + } + return null; +} From 9c2f964430bed9f56eac6c98c2afc400e26bcb8b Mon Sep 17 00:00:00 2001 From: Sylvain Cau Date: Mon, 8 Jun 2026 16:06:55 -0700 Subject: [PATCH 04/10] feat(plugins): implement tsundoku release poll loop Wire the Tsundoku feed walk end to end. Each poll builds an exact-match index from the host's tracked series, loads the saved feed cursor, then walks the feed page by page: matching items by external ID, recording hits as release candidates, and persisting the cursor after every page so an interrupted walk resumes cleanly. Per-candidate record failures are logged and skipped, the walk stops on a fetch error with the cursor preserved, and the poll returns parsed/matched/recorded/deduped counters plus the worst upstream status for host backoff. Add a README covering setup, configuration, the exact-match model, cursor behavior, and known limitations (default language, incremental backfill gap, high-water dedup). Includes a poll test suite covering multi-page walks, cursor persistence, host dedup, no-match skips, record-error tolerance, and fetch-error handling. --- plugins/release-tsundoku/README.md | 125 ++++++++++++ plugins/release-tsundoku/src/index.test.ts | 197 +++++++++++++++++++ plugins/release-tsundoku/src/index.ts | 217 +++++++++++++++++++-- 3 files changed, 526 insertions(+), 13 deletions(-) create mode 100644 plugins/release-tsundoku/README.md diff --git a/plugins/release-tsundoku/README.md b/plugins/release-tsundoku/README.md new file mode 100644 index 00000000..73adc073 --- /dev/null +++ b/plugins/release-tsundoku/README.md @@ -0,0 +1,125 @@ +# @ashdev/codex-plugin-release-tsundoku + +A Codex release-source plugin that announces new volume and chapter coverage +for your tracked series using a [Tsundoku](https://github.com/AshDevFr) instance's +incremental series feed. **Notification-only** — Codex does not download anything. + +## Features + +- **Exact external-ID matching, no fuzzy logic.** Series are matched to your + Codex catalog by provider IDs (MangaBaka, AniList, MAL, MangaUpdates, Kitsu, + Shikimori, Anime-Planet, Anime News Network), so announcements always land on + the right series with full confidence. +- **Incremental, cursor-based.** Walks Tsundoku's keyset-paginated + `/api/v1/series/feed`, persisting its position in the plugin KV store so each + poll only processes activity since the last run. +- **Volume- and chapter-aware.** The feed's merged, gap-preserving coverage + spans map directly onto Codex's release model. + +## Authentication + +None. The Tsundoku feed endpoint is public; you only need to point the plugin at +your instance with `baseUrl`. + +## Admin Setup + +### Adding the Plugin to Codex + +Add the plugin from **Settings → Plugins** (it appears in the official plugin +gallery as "Tsundoku Releases"), or configure it manually: + +- **Command:** `npx` +- **Args:** + ``` + -y + @ashdev/codex-plugin-release-tsundoku + ``` + +Set `baseUrl` to your Tsundoku instance (e.g. `https://tsundoku.example.com`) +and save. The plugin auto-registers a single source row ("Tsundoku Releases") in +**Settings → Release tracking**, where you can disable it, change the poll +interval, or trigger an immediate poll. + +### Linking Series to Tsundoku + +A series is matched whenever it carries at least one external ID that Tsundoku +also knows. Populate these by running a metadata refresh (e.g. the MangaBaka +metadata plugin) or by pasting an ID into the series' tracking panel. Supported +providers, in match-priority order: + +`mangabaka`, `anilist`, `mal`, `mangaupdates`, `kitsu`, `shikimori`, +`anime_planet`, `anime_news_network`. + +## Configuration + +| Field | Required | Default | Description | +| ------------------ | -------- | ------- | ------------------------------------------------------------------------ | +| `baseUrl` | yes | — | Tsundoku instance base URL. The plugin appends `/api/v1/series/feed`. | +| `defaultLanguage` | no | `en` | ISO 639-1 tag stamped on every announcement (the feed carries none). | +| `pageLimit` | no | `100` | Items per feed page (1–500). | +| `requestTimeoutMs` | no | `10000` | Per-page fetch timeout in milliseconds. | + +## How It Works + +On each poll the plugin: + +1. Loads its stored feed cursor from the plugin KV store. +2. Builds a reverse index (`provider:id → Codex series`) from your tracked + series via the host's `releases/list_tracked`. +3. Walks the feed from the cursor. Each item is matched against the index by + external ID; on a hit it records a release candidate (confidence `1.0`) whose + `volumes`/`chapters` mirror the item's coverage. The cursor is persisted after + each processed page, so an interrupted walk resumes cleanly. +4. Reports counters back to the host; the host applies its own threshold, + auto-ignore (for coverage you already own), and dedup. + +The candidate's dedup key is the coverage high-water mark +(`tsundoku:{seriesId}:v{highestVolume}:c{highestChapter}`), so a new +announcement fires only when the frontier advances; re-delivery of the same +coverage dedups host-side. + +### Limitations + +- **Default language.** Tsundoku tracks official release coverage and carries no + language, so every candidate uses `defaultLanguage` (`en` unless overridden). + Per-series language preferences still gate the high-water mark host-side. +- **Incremental backfill gap.** Because the walk is cursor-based, a series you + start tracking *after* its last Tsundoku coverage change won't get a catch-up + announcement until it changes again. This is correct for "new releases going + forward"; a full backfill would require resetting the cursor. +- **High-water dedup.** A filled interior gap that doesn't move the highest + volume/chapter won't re-announce. + +## Development + +```bash +# Install dependencies +npm install + +# Build the plugin +npm run build + +# Type check +npm run typecheck + +# Run tests +npm run test + +# Lint +npm run lint +``` + +## Project Structure + +``` +src/ +├── index.ts # Plugin lifecycle, config, source registration, poll loop +├── manifest.ts # Capability + config schema + supported providers +├── fetcher.ts # Feed wire types + paginated fetchFeedPage +├── matcher.ts # Reverse index + exact external-ID matching +└── candidate.ts # Feed item → ReleaseCandidate mapping +``` + +## License + +MIT diff --git a/plugins/release-tsundoku/src/index.test.ts b/plugins/release-tsundoku/src/index.test.ts index 0ff2819b..57241dd3 100644 --- a/plugins/release-tsundoku/src/index.test.ts +++ b/plugins/release-tsundoku/src/index.test.ts @@ -1,9 +1,11 @@ import { HostRpcClient, type PluginStorage } from "@ashdev/codex-plugin-sdk"; import { describe, expect, it, vi } from "vitest"; +import type { FeedItem, FeedResponse } from "./fetcher.js"; import { CURSOR_STORAGE_KEY, loadCursor, normalizeBaseUrl, + poll, registerSources, saveCursor, } from "./index.js"; @@ -195,3 +197,198 @@ describe("registerSources", () => { expect(calls.length).toBe(1); }); }); + +// ----------------------------------------------------------------------------- +// poll +// ----------------------------------------------------------------------------- + +/** Stateful in-memory storage double recording every `set`. */ +function makePollStorage(initialCursor: string | null = null): { + storage: PluginStorage; + sets: string[]; + current: () => string | null; +} { + let value = initialCursor; + const sets: string[] = []; + const storage = { + get: vi.fn(async () => ({ data: value })), + set: vi.fn(async (_key: string, data: unknown) => { + value = data as string; + sets.push(value); + return { success: true }; + }), + } as unknown as PluginStorage; + return { storage, sets, current: () => value }; +} + +/** A `fetch` impl that returns the given pages in order, then empty pages. */ +function makeFetchSequence(pages: FeedResponse[]): { + fetchImpl: typeof fetch; + urls: string[]; +} { + const urls: string[] = []; + let i = 0; + const fetchImpl = vi.fn(async (url: string) => { + urls.push(url); + const page = pages[i++] ?? { items: [], hasMore: false, nextCursor: null }; + return new Response(JSON.stringify(page), { + status: 200, + headers: { "content-type": "application/json" }, + }); + }) as unknown as typeof fetch; + return { fetchImpl, urls }; +} + +function item( + seriesId: number, + provider: string, + externalId: string, + highestVolume: number | null = 1, +): FeedItem { + return { + seriesId, + canonicalTitle: `Series ${seriesId}`, + externalIds: [{ provider, externalId, fetchedAt: 1_700_000_000 }], + volumeCoverage: highestVolume === null ? [] : [{ start: 1, end: highestVolume }], + chapterCoverage: [], + highestVolume, + highestChapter: null, + updatedAt: 1_700_000_000, + }; +} + +/** + * Mock host RPC for poll tests. `listTracked` supplies one page of tracked + * series (no `nextOffset`, so the sweep stops); `record` returns a result + * computed by `onRecord` (default: a fresh insert). + */ +function makePollRpc(opts: { + tracked: Array<{ seriesId: string; externalIds?: Record }>; + onRecord?: (n: number) => { ledgerId: string; deduped: boolean } | { __error: object }; +}): { rpc: HostRpcClient; calls: CapturedCall[] } { + let recordCount = 0; + return makeMockRpc((method) => { + if (method === "releases/list_tracked") { + return { tracked: opts.tracked }; + } + if (method === "releases/record") { + recordCount++; + return opts.onRecord + ? opts.onRecord(recordCount) + : { ledgerId: `l${recordCount}`, deduped: false }; + } + if (method === "releases/report_progress") { + return { emitted: true }; + } + return {}; + }); +} + +const pollDeps = (storage: PluginStorage, fetchImpl: typeof fetch) => ({ + storage, + baseUrl: "https://t.example.com", + language: "en", + pageLimit: 100, + timeoutMs: 5_000, + fetchImpl, +}); + +describe("poll", () => { + it("walks pages, matches by external id, records, and persists the cursor per page", async () => { + const { rpc } = makePollRpc({ + tracked: [{ seriesId: "uuid-a", externalIds: { mangabaka: "9741" } }], + }); + const { storage, sets } = makePollStorage(); + const { fetchImpl } = makeFetchSequence([ + { + items: [item(87, "mangabaka", "9741", 16), item(99, "anilist", "999")], + hasMore: true, + nextCursor: "c1", + }, + { items: [item(87, "mangabaka", "9741", 17)], hasMore: false, nextCursor: "c2" }, + ]); + + const res = await poll({ sourceId: "src-1" }, rpc, pollDeps(storage, fetchImpl)); + + expect(res).toMatchObject({ + parsed: 3, + matched: 2, + recorded: 2, + deduped: 0, + upstreamStatus: 200, + }); + expect(sets).toEqual(["c1", "c2"]); + }); + + it("counts host dedup separately from inserts", async () => { + const { rpc } = makePollRpc({ + tracked: [{ seriesId: "uuid-a", externalIds: { mangabaka: "9741" } }], + onRecord: (n) => ({ ledgerId: `l${n}`, deduped: n > 1 }), + }); + const { storage } = makePollStorage(); + const { fetchImpl } = makeFetchSequence([ + { + items: [item(87, "mangabaka", "9741", 16), item(87, "mangabaka", "9741", 17)], + hasMore: false, + nextCursor: "c1", + }, + ]); + + const res = await poll({ sourceId: "src-1" }, rpc, pollDeps(storage, fetchImpl)); + expect(res).toMatchObject({ matched: 2, recorded: 1, deduped: 1 }); + }); + + it("skips items with no tracked match (no record calls)", async () => { + const { rpc, calls } = makePollRpc({ + tracked: [{ seriesId: "uuid-a", externalIds: { mangabaka: "9741" } }], + }); + const { storage } = makePollStorage(); + const { fetchImpl } = makeFetchSequence([ + { items: [item(99, "anilist", "999")], hasMore: false, nextCursor: "c1" }, + ]); + + const res = await poll({ sourceId: "src-1" }, rpc, pollDeps(storage, fetchImpl)); + expect(res).toMatchObject({ parsed: 1, matched: 0, recorded: 0 }); + expect(calls.some((c) => c.method === "releases/record")).toBe(false); + }); + + it("tolerates a record failure without aborting the walk", async () => { + const { rpc } = makePollRpc({ + tracked: [{ seriesId: "uuid-a", externalIds: { mangabaka: "9741" } }], + onRecord: () => ({ __error: { code: -32000, message: "ledger down" } }), + }); + const { storage, sets } = makePollStorage(); + const { fetchImpl } = makeFetchSequence([ + { items: [item(87, "mangabaka", "9741", 16)], hasMore: false, nextCursor: "c1" }, + ]); + + const res = await poll({ sourceId: "src-1" }, rpc, pollDeps(storage, fetchImpl)); + expect(res).toMatchObject({ parsed: 1, matched: 1, recorded: 0, deduped: 0 }); + expect(sets).toEqual(["c1"]); // walk still completed and advanced the cursor + }); + + it("passes the stored cursor to the first fetch", async () => { + const { rpc } = makePollRpc({ tracked: [] }); + const { storage } = makePollStorage("resume-here"); + const { fetchImpl, urls } = makeFetchSequence([ + { items: [], hasMore: false, nextCursor: null }, + ]); + + await poll({ sourceId: "src-1" }, rpc, pollDeps(storage, fetchImpl)); + expect(new URL(urls[0]).searchParams.get("cursor")).toBe("resume-here"); + }); + + it("stops and preserves the cursor on a fetch error", async () => { + const { rpc } = makePollRpc({ + tracked: [{ seriesId: "uuid-a", externalIds: { mangabaka: "9741" } }], + }); + const { storage, sets } = makePollStorage("kept"); + const fetchImpl = vi + .fn() + .mockResolvedValue(new Response("boom", { status: 503 })) as unknown as typeof fetch; + + const res = await poll({ sourceId: "src-1" }, rpc, pollDeps(storage, fetchImpl)); + expect(res).toMatchObject({ parsed: 0, upstreamStatus: 503 }); + expect(sets).toEqual([]); // never advanced past the failing page + }); +}); diff --git a/plugins/release-tsundoku/src/index.ts b/plugins/release-tsundoku/src/index.ts index 935e2542..0fba8c2a 100644 --- a/plugins/release-tsundoku/src/index.ts +++ b/plugins/release-tsundoku/src/index.ts @@ -31,10 +31,15 @@ import { type InitializeParams, type PluginStorage, RELEASES_METHODS, + type ReleaseCandidate, type ReleasePollRequest, type ReleasePollResponse, + type TrackedSeriesEntry, } from "@ashdev/codex-plugin-sdk"; +import { feedItemToCandidate } from "./candidate.js"; +import { fetchFeedPage } from "./fetcher.js"; import { manifest } from "./manifest.js"; +import { buildIndex, matchItem } from "./matcher.js"; const logger = createLogger({ name: manifest.name, level: "info" }); @@ -170,26 +175,206 @@ export async function registerSources( return null; } +// ============================================================================= +// Reverse-RPC wrappers +// ============================================================================= + +interface ListTrackedResponse { + tracked: TrackedSeriesEntry[]; + nextOffset?: number; +} + +interface RecordResponse { + ledgerId: string; + deduped: boolean; +} + +/** Page size for the tracked-series sweep that builds the match index. */ +const TRACKED_PAGE_SIZE = 200; + +/** + * Lazily walk all tracked-series pages from the host. Yields one entry at a + * time so the caller can build the reverse index without materializing every + * page at once. + */ +async function* iterateTrackedSeries( + rpc: HostRpcClient, + sourceId: string, +): AsyncGenerator { + let offset = 0; + while (true) { + const page = await rpc.call(RELEASES_METHODS.LIST_TRACKED, { + sourceId, + offset, + limit: TRACKED_PAGE_SIZE, + }); + for (const entry of page.tracked) { + yield entry; + } + if (page.nextOffset === undefined || page.tracked.length === 0) return; + offset = page.nextOffset; + } +} + +/** + * Submit one candidate to the host ledger. Per-candidate failures (threshold + * rejection, validation, transient host error) are logged and swallowed so a + * single bad item never aborts the walk; the next poll retries it. + */ +async function recordCandidate( + rpc: HostRpcClient, + sourceId: string, + candidate: ReleaseCandidate, +): Promise { + try { + return await rpc.call(RELEASES_METHODS.RECORD, { sourceId, candidate }); + } catch (err) { + const reason = err instanceof Error ? err.message : String(err); + const code = err instanceof HostRpcError ? ` (code ${err.code})` : ""; + logger.warn(`record failed for ${candidate.externalReleaseId}: ${reason}${code}`); + return null; + } +} + +/** + * Best-effort progress emit. Failures (including older hosts without the + * method) are swallowed — progress is a UX nicety, never a reason to abort. + */ +async function reportProgress( + rpc: HostRpcClient, + current: number, + total: number, + message: string, +): Promise { + try { + await rpc.call(RELEASES_METHODS.REPORT_PROGRESS, { current, total, message }); + } catch (err) { + if (err instanceof HostRpcError && err.code === -32601) return; + const reason = err instanceof Error ? err.message : String(err); + logger.debug(`report_progress dropped: ${reason}`); + } +} + // ============================================================================= // Poll // ============================================================================= +/** Dependencies a poll needs, defaulted from plugin state at the call site. */ +export interface PollDeps { + storage: PluginStorage; + /** Tsundoku base URL (no trailing slash). */ + baseUrl: string; + /** Language stamped on every candidate. */ + language: string; + /** Feed page size. */ + pageLimit: number; + /** Per-page fetch timeout. */ + timeoutMs: number; + /** Custom `fetch` impl (tests). */ + fetchImpl?: typeof fetch; +} + /** - * Top-level poll handler. The feed walk + matching are added in dedicated - * modules; until they land, the source registers and polls cleanly while - * recording no candidates. Exported for tests. + * Top-level poll handler. + * + * Builds the exact-match index from the host's tracked series, then walks the + * Tsundoku feed from the stored cursor: each item is matched by external ID + * and, on a hit, recorded as a candidate. The cursor is persisted after every + * processed page so an interrupted walk resumes from the last completed page + * (keyset pagination is gap-free, and host-side dedup makes re-processing + * safe). Exported for tests. */ export async function poll( - _params: ReleasePollRequest, - _rpc: HostRpcClient, + params: ReleasePollRequest, + rpc: HostRpcClient, + deps: PollDeps, ): Promise { + const sourceId = params.sourceId; + + // 1. Build the reverse index from the user's tracked series. The feed spans + // the whole Tsundoku catalog, so this is what scopes it to the user. + const trackedEntries: TrackedSeriesEntry[] = []; + for await (const entry of iterateTrackedSeries(rpc, sourceId)) { + trackedEntries.push(entry); + } + const index = buildIndex(trackedEntries); + if (index.size === 0) { + logger.info( + `poll: no tracked series carry a Tsundoku-known external ID (source=${sourceId}); nothing to match`, + ); + } + + // 2. Walk the feed from the stored cursor. + let cursor = await loadCursor(deps.storage); + let parsed = 0; + let matched = 0; + let recorded = 0; + let deduped = 0; + let worstStatus = 200; + + while (true) { + const result = await fetchFeedPage(deps.baseUrl, cursor, deps.pageLimit, { + timeoutMs: deps.timeoutMs, + fetchImpl: deps.fetchImpl, + }); + + if (result.kind === "error") { + worstStatus = Math.max(worstStatus, result.status); + logger.warn( + `feed fetch failed (status ${result.status}): ${result.message}; stopping walk, cursor preserved`, + ); + break; + } + + const page = result.data; + for (const item of page.items) { + parsed++; + const match = matchItem(item, index); + if (!match) continue; + matched++; + const candidate = feedItemToCandidate(item, match, { + baseUrl: deps.baseUrl, + language: deps.language, + }); + const outcome = await recordCandidate(rpc, sourceId, candidate); + if (!outcome) continue; + if (outcome.deduped) { + deduped++; + } else { + recorded++; + } + } + + // Advance + persist the cursor before deciding whether to continue, so an + // error or crash on the next page resumes from this point. + const next = page.nextCursor ?? null; + if (next) { + cursor = next; + await saveCursor(deps.storage, next); + } + + await reportProgress(rpc, parsed, parsed, `Processed ${parsed} feed items`); + + if (!page.hasMore) break; + if (!next) { + // hasMore with no advancing cursor would loop forever; stop defensively. + logger.warn("feed reported hasMore but no nextCursor; stopping walk"); + break; + } + if (page.items.length === 0) break; + } + + logger.info( + `poll complete: source=${sourceId} tracked=${trackedEntries.length} parsed=${parsed} matched=${matched} recorded=${recorded} deduped=${deduped} worst_status=${worstStatus}`, + ); + return { notModified: false, - upstreamStatus: 200, - parsed: 0, - matched: 0, - recorded: 0, - deduped: 0, + upstreamStatus: worstStatus, + parsed, + matched, + recorded, + deduped, }; } @@ -201,13 +386,19 @@ createReleaseSourcePlugin({ manifest, provider: { async poll(params: ReleasePollRequest): Promise { - if (!state.hostRpc) { - throw new Error("Plugin not initialized: hostRpc client missing"); + if (!state.hostRpc || !state.storage) { + throw new Error("Plugin not initialized: host RPC / storage client missing"); } if (!state.baseUrl) { throw new Error("Plugin not configured: baseUrl is required"); } - return poll(params, state.hostRpc); + return poll(params, state.hostRpc, { + storage: state.storage, + baseUrl: state.baseUrl, + language: state.defaultLanguage, + pageLimit: state.pageLimit, + timeoutMs: state.requestTimeoutMs, + }); }, }, logLevel: "info", From e530a2895916146dbb2415730f67e19fee8f2a6c Mon Sep 17 00:00:00 2001 From: Sylvain Cau Date: Mon, 8 Jun 2026 16:49:23 -0700 Subject: [PATCH 05/10] fix(plugins): make release-source init race-free and fix system-plugin cursor storage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two related fixes for release-source plugins, which run as system plugins with no per-user context. Host: add a readiness barrier to reverse-RPC dispatch. A plugin can call a reverse method (e.g. releases/register_sources) from onInitialize the moment it returns its manifest — before the host finishes installing the post-initialize context — and the dispatcher bounced it with METHOD_NOT_FOUND, leaving registration to flaky plugin-side retries. The context now carries a readiness notification that set_capabilities fires; an early reverse-RPC parks until capabilities are installed (bounded by a timeout) and then dispatches normally, falling back to the previous METHOD_NOT_FOUND only if initialization never completes. This fixes source registration for every release-source plugin, not just one. Plugin (tsundoku): persist the feed cursor in the source's etag slot via releases/source_state instead of the plugin KV store. The KV store is scoped per user, but a release source has no user context, so storage/* is rejected with "Storage is not available". The host hands the saved cursor back as the poll request's etag and persists the etag returned on the poll response, so the per-source state slot is the correct single-row bookmark. Also surface a hard error when the first feed page can't be fetched so the source reports last_error rather than a silent empty poll. Includes host tests for the readiness barrier (early call succeeds once capabilities land) and updates the plugin tests for the new cursor source. --- crates/codex-services/src/plugin/rpc.rs | 161 ++++++++++++++++--- plugins/release-tsundoku/README.md | 14 +- plugins/release-tsundoku/package-lock.json | 42 ----- plugins/release-tsundoku/src/index.test.ts | 172 +++++++++------------ plugins/release-tsundoku/src/index.ts | 75 +++++---- 5 files changed, 260 insertions(+), 204 deletions(-) diff --git a/crates/codex-services/src/plugin/rpc.rs b/crates/codex-services/src/plugin/rpc.rs index 530b74bd..40d908c0 100644 --- a/crates/codex-services/src/plugin/rpc.rs +++ b/crates/codex-services/src/plugin/rpc.rs @@ -10,7 +10,7 @@ use std::time::Duration; use serde::Serialize; use serde::de::DeserializeOwned; use serde_json::Value; -use tokio::sync::{Mutex, RwLock, mpsc}; +use tokio::sync::{Mutex, Notify, RwLock, mpsc}; use tokio::time::timeout; use tracing::{Instrument, debug, error, warn}; @@ -35,6 +35,11 @@ pub struct ReverseRpcContext { releases_handler: Option, /// `None` until the plugin has been initialized. capabilities: Option, + /// Notified once `capabilities` is populated. Lets the dispatcher park an + /// early reverse-RPC call (one that raced ahead of the host installing the + /// post-`initialize` context) until the plugin is ready, instead of + /// bouncing it with `METHOD_NOT_FOUND` and relying on plugin-side retries. + ready: Arc, } impl ReverseRpcContext { @@ -43,6 +48,7 @@ impl ReverseRpcContext { storage_handler: None, releases_handler: None, capabilities: None, + ready: Arc::new(Notify::new()), } } @@ -51,13 +57,16 @@ impl ReverseRpcContext { storage_handler: Some(storage_handler), releases_handler: None, capabilities: None, + ready: Arc::new(Notify::new()), } } /// Replace the plugin's capability snapshot, used by [`super::handle::PluginHandle`] - /// once `initialize` returns. + /// once `initialize` returns. Wakes any reverse-RPC calls parked in the + /// readiness barrier (see [`await_capabilities`]). pub fn set_capabilities(&mut self, caps: PluginCapabilities) { self.capabilities = Some(caps); + self.ready.notify_waiters(); } /// Install the releases handler. Called after capabilities are known @@ -385,6 +394,7 @@ impl RpcClient { &reverse_method, &reverse_request, &self.reverse_ctx, + REVERSE_RPC_READINESS_TIMEOUT, ) .await; // Write the response back to the plugin. Best-effort: @@ -511,6 +521,50 @@ impl Drop for RpcClient { } } +/// How long a reverse-RPC call parks waiting for the plugin's capabilities to +/// be installed before giving up with `METHOD_NOT_FOUND`. Real initialization +/// completes in milliseconds; this is a generous upper bound for a process +/// under load. Only calls that race `initialize` ever wait at all. +const REVERSE_RPC_READINESS_TIMEOUT: Duration = Duration::from_secs(5); + +/// Park until the plugin's capabilities are installed (`set_capabilities`), or +/// `timeout` elapses. Returns immediately when capabilities are already set +/// (the common case). This closes the startup race where a plugin fires a +/// reverse-RPC from `onInitialize` before the host installs the post-`initialize` +/// context, without requiring plugins to retry on `METHOD_NOT_FOUND`. +async fn await_capabilities(reverse_ctx: &Arc>, timeout: Duration) { + // Fast path + grab the notify handle without holding the read lock across + // the await below. + let ready = { + let guard = reverse_ctx.read().await; + if guard.capabilities.is_some() { + return; + } + guard.ready.clone() + }; + + let sleep = tokio::time::sleep(timeout); + tokio::pin!(sleep); + + loop { + // Register interest *before* re-checking the condition so a notify that + // fires between the check and the await isn't lost (the canonical + // `tokio::sync::Notify` pattern, via `enable()`). + let notified = ready.notified(); + tokio::pin!(notified); + notified.as_mut().enable(); + + if reverse_ctx.read().await.capabilities.is_some() { + return; + } + + tokio::select! { + _ = &mut sleep => return, + _ = &mut notified => { /* woken — re-check on the next iteration */ } + } + } +} + /// Dispatch a single reverse-RPC request to the appropriate handler after /// running the permission check. /// @@ -523,28 +577,35 @@ async fn dispatch_reverse_rpc( method: &str, request: &JsonRpcRequest, reverse_ctx: &Arc>, + readiness_timeout: Duration, ) -> JsonRpcResponse { let request_id = request.id.clone(); + // Readiness barrier. A plugin can fire a reverse-RPC (e.g. a release + // source's `register_sources`) from its `onInitialize` the instant after + // it returns the manifest — before the host has finished installing the + // post-`initialize` context (capabilities + handlers). Rather than bounce + // that call and depend on plugin-side retries, park here until the + // capabilities are installed or the timeout elapses. + await_capabilities(reverse_ctx, readiness_timeout).await; + // Take a read snapshot of the context. We keep it as long as we're // dispatching so the handlers don't get swapped mid-call. let ctx_guard = reverse_ctx.read().await; - // 1. Permission check. If capabilities haven't been set yet (i.e. the - // plugin tried to make a reverse-RPC call before the host installed - // the per-plugin reverse-RPC handlers), we return METHOD_NOT_FOUND - // rather than AUTH_FAILED. From the plugin's perspective the method - // isn't dispatchable *yet* — distinguishing this from a real - // permission denial lets the plugin SDK retry with backoff to ride - // out the brief initialization race (see e.g. release-nyaa's - // `registerSources` retry on -32601). AUTH_FAILED stays reserved - // for actual capability-declined-method denials. + // 1. Permission check. If capabilities are *still* unset after the + // readiness wait (initialization stalled or failed), we return + // METHOD_NOT_FOUND rather than AUTH_FAILED. From the plugin's + // perspective the method isn't dispatchable *yet* — distinguishing this + // from a real permission denial lets a plugin SDK retry with backoff as + // a last resort. AUTH_FAILED stays reserved for actual + // capability-declined-method denials. let caps = match ctx_guard.capabilities.as_ref() { Some(c) => c, None => { warn!( method = %method, - "Reverse-RPC call before plugin initialized; deferring (METHOD_NOT_FOUND)" + "Reverse-RPC call still uninitialized after readiness wait; deferring (METHOD_NOT_FOUND)" ); return JsonRpcResponse::error( Some(request_id), @@ -870,7 +931,13 @@ async fn dispatch_and_write( process: &Arc>, ) { let request_id = request.id.clone(); - let response = dispatch_reverse_rpc(&method, &request, reverse_ctx).await; + let response = dispatch_reverse_rpc( + &method, + &request, + reverse_ctx, + REVERSE_RPC_READINESS_TIMEOUT, + ) + .await; let response_json = match serde_json::to_string(&response) { Ok(j) => j, Err(e) => { @@ -1009,12 +1076,11 @@ mod tests { } } - /// Reverse-RPC dispatch should reject calls before the plugin has been - /// initialized — at that point the host doesn't yet know the plugin's - /// capabilities. Returned as `METHOD_NOT_FOUND` (rather than - /// `AUTH_FAILED`) so plugin SDKs can retry with backoff to ride out the - /// brief init race; an `AUTH_FAILED` response would tell the SDK to - /// give up. See the doc comment on `dispatch_reverse_rpc`. + /// Reverse-RPC dispatch parks on the readiness barrier when capabilities + /// aren't installed, then — if they never arrive within the timeout — + /// rejects with `METHOD_NOT_FOUND` (rather than `AUTH_FAILED`). The short + /// timeout here exercises the give-up path; `test_dispatch_waits_for_late_capabilities` + /// covers the success path where caps land mid-wait. #[tokio::test] async fn test_dispatch_rejects_before_init() { let ctx = Arc::new(RwLock::new(ReverseRpcContext::new())); @@ -1023,11 +1089,57 @@ mod tests { super::super::protocol::methods::STORAGE_GET, Some(json!({"key": "x"})), ); - let resp = dispatch_reverse_rpc(&request.method, &request, &ctx).await; + let resp = + dispatch_reverse_rpc(&request.method, &request, &ctx, Duration::from_millis(50)).await; assert!(resp.is_error()); assert_eq!(resp.error.unwrap().code, error_codes::METHOD_NOT_FOUND); } + /// The readiness barrier: a reverse-RPC that arrives before the host has + /// installed capabilities must not be bounced. It parks until + /// `set_capabilities` runs, then dispatches normally — here a storage call + /// with no handler resolves to the "no storage handler" error (proving it + /// got *past* the capabilities gate) rather than the pre-init rejection. + #[tokio::test] + async fn test_dispatch_waits_for_late_capabilities() { + use super::super::protocol::PluginCapabilities; + + let ctx = Arc::new(RwLock::new(ReverseRpcContext::new())); + let request = JsonRpcRequest::new( + 1i64, + super::super::protocol::methods::STORAGE_GET, + Some(json!({"key": "x"})), + ); + + // Install capabilities shortly after dispatch begins, simulating the + // host finishing `update_reverse_ctx` while an early call is parked. + let ctx_writer = Arc::clone(&ctx); + let setter = tokio::spawn(async move { + tokio::time::sleep(Duration::from_millis(20)).await; + ctx_writer + .write() + .await + .set_capabilities(PluginCapabilities::default()); + }); + + // Generous barrier timeout so the late caps win the race. + let resp = + dispatch_reverse_rpc(&request.method, &request, &ctx, Duration::from_secs(2)).await; + setter.await.unwrap(); + + assert!(resp.is_error()); + // Past the capabilities gate: storage is permitted but no handler is + // installed, so we get the handler-missing error, NOT a pre-init bounce. + // (Both are METHOD_NOT_FOUND, so assert on the message to disambiguate.) + let err = resp.error.unwrap(); + assert_eq!(err.code, error_codes::METHOD_NOT_FOUND); + assert!( + err.message.contains("Storage is not available"), + "expected to pass the readiness gate and reach the storage handler check, got: {}", + err.message + ); + } + /// A plugin without `release_source` calling `releases/record` should be /// rejected with AUTH_FAILED, regardless of payload. #[tokio::test] @@ -1046,7 +1158,8 @@ mod tests { super::super::protocol::methods::RELEASES_RECORD, Some(json!({})), ); - let resp = dispatch_reverse_rpc(&request.method, &request, &ctx).await; + let resp = + dispatch_reverse_rpc(&request.method, &request, &ctx, Duration::from_millis(50)).await; assert!(resp.is_error()); assert_eq!(resp.error.unwrap().code, error_codes::AUTH_FAILED); } @@ -1062,7 +1175,8 @@ mod tests { let ctx = Arc::new(RwLock::new(ctx_inner)); let request = JsonRpcRequest::new(1i64, "frobnicate/zap", Some(json!({}))); - let resp = dispatch_reverse_rpc(&request.method, &request, &ctx).await; + let resp = + dispatch_reverse_rpc(&request.method, &request, &ctx, Duration::from_millis(50)).await; assert!(resp.is_error()); assert_eq!(resp.error.unwrap().code, error_codes::METHOD_NOT_FOUND); } @@ -1083,7 +1197,8 @@ mod tests { super::super::protocol::methods::STORAGE_GET, Some(json!({"key": "x"})), ); - let resp = dispatch_reverse_rpc(&request.method, &request, &ctx).await; + let resp = + dispatch_reverse_rpc(&request.method, &request, &ctx, Duration::from_millis(50)).await; assert!(resp.is_error()); assert_eq!(resp.error.unwrap().code, error_codes::METHOD_NOT_FOUND); } diff --git a/plugins/release-tsundoku/README.md b/plugins/release-tsundoku/README.md index 73adc073..12b34e74 100644 --- a/plugins/release-tsundoku/README.md +++ b/plugins/release-tsundoku/README.md @@ -11,8 +11,8 @@ incremental series feed. **Notification-only** — Codex does not download anyth Shikimori, Anime-Planet, Anime News Network), so announcements always land on the right series with full confidence. - **Incremental, cursor-based.** Walks Tsundoku's keyset-paginated - `/api/v1/series/feed`, persisting its position in the plugin KV store so each - poll only processes activity since the last run. + `/api/v1/series/feed`, persisting its position in the source's state (etag + slot) so each poll only processes activity since the last run. - **Volume- and chapter-aware.** The feed's merged, gap-preserving coverage spans map directly onto Codex's release model. @@ -63,7 +63,8 @@ providers, in match-priority order: On each poll the plugin: -1. Loads its stored feed cursor from the plugin KV store. +1. Loads its stored feed cursor (the host hands it back as the source's + persisted `etag`). 2. Builds a reverse index (`provider:id → Codex series`) from your tracked series via the host's `releases/list_tracked`. 3. Walks the feed from the cursor. Each item is matched against the index by @@ -78,6 +79,13 @@ The candidate's dedup key is the coverage high-water mark announcement fires only when the frontier advances; re-delivery of the same coverage dedups host-side. +If the very first feed page can't be fetched (e.g. `baseUrl` is wrong or the +instance is unreachable), the poll fails and the source shows `last_error` in +**Settings → Release tracking** rather than silently reporting "0 items". In +Docker, remember the plugin runs inside the worker container: use a URL the +container can resolve (e.g. `http://host.docker.internal:`), not +`http://localhost:`. + ### Limitations - **Default language.** Tsundoku tracks official release coverage and carries no diff --git a/plugins/release-tsundoku/package-lock.json b/plugins/release-tsundoku/package-lock.json index 6148abaa..fd52cfe2 100644 --- a/plugins/release-tsundoku/package-lock.json +++ b/plugins/release-tsundoku/package-lock.json @@ -112,9 +112,6 @@ "arm64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT OR Apache-2.0", "optional": true, "os": [ @@ -132,9 +129,6 @@ "arm64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT OR Apache-2.0", "optional": true, "os": [ @@ -152,9 +146,6 @@ "x64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT OR Apache-2.0", "optional": true, "os": [ @@ -172,9 +163,6 @@ "x64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT OR Apache-2.0", "optional": true, "os": [ @@ -823,9 +811,6 @@ "arm64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -843,9 +828,6 @@ "arm64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -863,9 +845,6 @@ "ppc64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -883,9 +862,6 @@ "s390x" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -903,9 +879,6 @@ "x64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MIT", "optional": true, "os": [ @@ -923,9 +896,6 @@ "x64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MIT", "optional": true, "os": [ @@ -1460,9 +1430,6 @@ "arm64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MPL-2.0", "optional": true, "os": [ @@ -1484,9 +1451,6 @@ "arm64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MPL-2.0", "optional": true, "os": [ @@ -1508,9 +1472,6 @@ "x64" ], "dev": true, - "libc": [ - "glibc" - ], "license": "MPL-2.0", "optional": true, "os": [ @@ -1532,9 +1493,6 @@ "x64" ], "dev": true, - "libc": [ - "musl" - ], "license": "MPL-2.0", "optional": true, "os": [ diff --git a/plugins/release-tsundoku/src/index.test.ts b/plugins/release-tsundoku/src/index.test.ts index 57241dd3..34042e5c 100644 --- a/plugins/release-tsundoku/src/index.test.ts +++ b/plugins/release-tsundoku/src/index.test.ts @@ -1,14 +1,7 @@ -import { HostRpcClient, type PluginStorage } from "@ashdev/codex-plugin-sdk"; +import { HostRpcClient } from "@ashdev/codex-plugin-sdk"; import { describe, expect, it, vi } from "vitest"; import type { FeedItem, FeedResponse } from "./fetcher.js"; -import { - CURSOR_STORAGE_KEY, - loadCursor, - normalizeBaseUrl, - poll, - registerSources, - saveCursor, -} from "./index.js"; +import { loadCursor, normalizeBaseUrl, poll, registerSources, saveCursor } from "./index.js"; // ----------------------------------------------------------------------------- // Mock host RPC @@ -83,64 +76,32 @@ describe("normalizeBaseUrl", () => { }); // ----------------------------------------------------------------------------- -// Cursor persistence +// Cursor persistence (per-source state / etag slot) // ----------------------------------------------------------------------------- -/** Minimal in-memory `PluginStorage` double exposing only get/set. */ -function makeFakeStorage(initial?: unknown): { - storage: PluginStorage; - get: ReturnType; - set: ReturnType; -} { - const get = vi.fn(async () => ({ data: initial ?? null })); - const set = vi.fn(async () => ({ success: true })); - const storage = { get, set } as unknown as PluginStorage; - return { storage, get, set }; -} - describe("loadCursor", () => { - it("returns the stored cursor string", async () => { - const { storage, get } = makeFakeStorage("cursor-42"); - expect(await loadCursor(storage)).toBe("cursor-42"); - expect(get).toHaveBeenCalledWith(CURSOR_STORAGE_KEY); - }); - - it("returns null when no cursor is stored", async () => { - const { storage } = makeFakeStorage(null); - expect(await loadCursor(storage)).toBeNull(); + it("returns the etag the host passed back from the last poll", () => { + expect(loadCursor({ sourceId: "s", etag: "cursor-42" })).toBe("cursor-42"); }); - it("returns null for a non-string / empty stored value", async () => { - expect(await loadCursor(makeFakeStorage("").storage)).toBeNull(); - expect(await loadCursor(makeFakeStorage(123).storage)).toBeNull(); - }); - - it("returns null and does not throw when the read fails", async () => { - const storage = { - get: vi.fn(async () => { - throw new Error("kv down"); - }), - set: vi.fn(), - } as unknown as PluginStorage; - expect(await loadCursor(storage)).toBeNull(); + it("returns null when no etag was supplied", () => { + expect(loadCursor({ sourceId: "s" })).toBeNull(); + expect(loadCursor({ sourceId: "s", etag: "" })).toBeNull(); }); }); describe("saveCursor", () => { - it("writes the cursor under the feed-cursor key", async () => { - const { storage, set } = makeFakeStorage(); - await saveCursor(storage, "cursor-99"); - expect(set).toHaveBeenCalledWith(CURSOR_STORAGE_KEY, "cursor-99"); + it("persists the cursor into the source-state etag slot", async () => { + const { rpc, calls } = makeMockRpc(() => ({ success: true })); + await saveCursor(rpc, "src-1", "cursor-99"); + expect(calls).toHaveLength(1); + expect(calls[0].method).toBe("releases/source_state/set"); + expect(calls[0].params).toEqual({ sourceId: "src-1", etag: "cursor-99" }); }); it("swallows a write failure without throwing", async () => { - const storage = { - get: vi.fn(), - set: vi.fn(async () => { - throw new Error("kv full"); - }), - } as unknown as PluginStorage; - await expect(saveCursor(storage, "cursor-99")).resolves.toBeUndefined(); + const { rpc } = makeMockRpc(() => ({ __error: { code: -32000, message: "db error" } })); + await expect(saveCursor(rpc, "src-1", "cursor-99")).resolves.toBeUndefined(); }); }); @@ -202,27 +163,10 @@ describe("registerSources", () => { // poll // ----------------------------------------------------------------------------- -/** Stateful in-memory storage double recording every `set`. */ -function makePollStorage(initialCursor: string | null = null): { - storage: PluginStorage; - sets: string[]; - current: () => string | null; -} { - let value = initialCursor; - const sets: string[] = []; - const storage = { - get: vi.fn(async () => ({ data: value })), - set: vi.fn(async (_key: string, data: unknown) => { - value = data as string; - sets.push(value); - return { success: true }; - }), - } as unknown as PluginStorage; - return { storage, sets, current: () => value }; -} +type PageOrError = FeedResponse | { errorStatus: number }; -/** A `fetch` impl that returns the given pages in order, then empty pages. */ -function makeFetchSequence(pages: FeedResponse[]): { +/** A `fetch` impl that returns the given pages/errors in order, then empties. */ +function makeFetchSequence(pages: PageOrError[]): { fetchImpl: typeof fetch; urls: string[]; } { @@ -231,6 +175,9 @@ function makeFetchSequence(pages: FeedResponse[]): { const fetchImpl = vi.fn(async (url: string) => { urls.push(url); const page = pages[i++] ?? { items: [], hasMore: false, nextCursor: null }; + if ("errorStatus" in page) { + return new Response("err", { status: page.errorStatus }); + } return new Response(JSON.stringify(page), { status: 200, headers: { "content-type": "application/json" }, @@ -239,6 +186,13 @@ function makeFetchSequence(pages: FeedResponse[]): { return { fetchImpl, urls }; } +/** Cursors persisted via `releases/source_state/set`, in order. */ +function cursorsPersisted(calls: CapturedCall[]): string[] { + return calls + .filter((c) => c.method === "releases/source_state/set") + .map((c) => (c.params as { etag: string }).etag); +} + function item( seriesId: number, provider: string, @@ -280,12 +234,14 @@ function makePollRpc(opts: { if (method === "releases/report_progress") { return { emitted: true }; } + if (method === "releases/source_state/set") { + return { success: true }; + } return {}; }); } -const pollDeps = (storage: PluginStorage, fetchImpl: typeof fetch) => ({ - storage, +const pollDeps = (fetchImpl: typeof fetch) => ({ baseUrl: "https://t.example.com", language: "en", pageLimit: 100, @@ -295,10 +251,9 @@ const pollDeps = (storage: PluginStorage, fetchImpl: typeof fetch) => ({ describe("poll", () => { it("walks pages, matches by external id, records, and persists the cursor per page", async () => { - const { rpc } = makePollRpc({ + const { rpc, calls } = makePollRpc({ tracked: [{ seriesId: "uuid-a", externalIds: { mangabaka: "9741" } }], }); - const { storage, sets } = makePollStorage(); const { fetchImpl } = makeFetchSequence([ { items: [item(87, "mangabaka", "9741", 16), item(99, "anilist", "999")], @@ -308,7 +263,7 @@ describe("poll", () => { { items: [item(87, "mangabaka", "9741", 17)], hasMore: false, nextCursor: "c2" }, ]); - const res = await poll({ sourceId: "src-1" }, rpc, pollDeps(storage, fetchImpl)); + const res = await poll({ sourceId: "src-1" }, rpc, pollDeps(fetchImpl)); expect(res).toMatchObject({ parsed: 3, @@ -316,8 +271,9 @@ describe("poll", () => { recorded: 2, deduped: 0, upstreamStatus: 200, + etag: "c2", }); - expect(sets).toEqual(["c1", "c2"]); + expect(cursorsPersisted(calls)).toEqual(["c1", "c2"]); }); it("counts host dedup separately from inserts", async () => { @@ -325,7 +281,6 @@ describe("poll", () => { tracked: [{ seriesId: "uuid-a", externalIds: { mangabaka: "9741" } }], onRecord: (n) => ({ ledgerId: `l${n}`, deduped: n > 1 }), }); - const { storage } = makePollStorage(); const { fetchImpl } = makeFetchSequence([ { items: [item(87, "mangabaka", "9741", 16), item(87, "mangabaka", "9741", 17)], @@ -334,7 +289,7 @@ describe("poll", () => { }, ]); - const res = await poll({ sourceId: "src-1" }, rpc, pollDeps(storage, fetchImpl)); + const res = await poll({ sourceId: "src-1" }, rpc, pollDeps(fetchImpl)); expect(res).toMatchObject({ matched: 2, recorded: 1, deduped: 1 }); }); @@ -342,53 +297,66 @@ describe("poll", () => { const { rpc, calls } = makePollRpc({ tracked: [{ seriesId: "uuid-a", externalIds: { mangabaka: "9741" } }], }); - const { storage } = makePollStorage(); const { fetchImpl } = makeFetchSequence([ { items: [item(99, "anilist", "999")], hasMore: false, nextCursor: "c1" }, ]); - const res = await poll({ sourceId: "src-1" }, rpc, pollDeps(storage, fetchImpl)); + const res = await poll({ sourceId: "src-1" }, rpc, pollDeps(fetchImpl)); expect(res).toMatchObject({ parsed: 1, matched: 0, recorded: 0 }); expect(calls.some((c) => c.method === "releases/record")).toBe(false); }); it("tolerates a record failure without aborting the walk", async () => { - const { rpc } = makePollRpc({ + const { rpc, calls } = makePollRpc({ tracked: [{ seriesId: "uuid-a", externalIds: { mangabaka: "9741" } }], onRecord: () => ({ __error: { code: -32000, message: "ledger down" } }), }); - const { storage, sets } = makePollStorage(); const { fetchImpl } = makeFetchSequence([ { items: [item(87, "mangabaka", "9741", 16)], hasMore: false, nextCursor: "c1" }, ]); - const res = await poll({ sourceId: "src-1" }, rpc, pollDeps(storage, fetchImpl)); + const res = await poll({ sourceId: "src-1" }, rpc, pollDeps(fetchImpl)); expect(res).toMatchObject({ parsed: 1, matched: 1, recorded: 0, deduped: 0 }); - expect(sets).toEqual(["c1"]); // walk still completed and advanced the cursor + expect(cursorsPersisted(calls)).toEqual(["c1"]); // walk still completed and advanced the cursor }); - it("passes the stored cursor to the first fetch", async () => { + it("resumes from the etag the host passed in", async () => { const { rpc } = makePollRpc({ tracked: [] }); - const { storage } = makePollStorage("resume-here"); const { fetchImpl, urls } = makeFetchSequence([ { items: [], hasMore: false, nextCursor: null }, ]); - await poll({ sourceId: "src-1" }, rpc, pollDeps(storage, fetchImpl)); + await poll({ sourceId: "src-1", etag: "resume-here" }, rpc, pollDeps(fetchImpl)); expect(new URL(urls[0]).searchParams.get("cursor")).toBe("resume-here"); }); - it("stops and preserves the cursor on a fetch error", async () => { - const { rpc } = makePollRpc({ + it("throws when even the first page can't be fetched (so the source shows last_error)", async () => { + const { rpc, calls } = makePollRpc({ + tracked: [{ seriesId: "uuid-a", externalIds: { mangabaka: "9741" } }], + }); + const { fetchImpl } = makeFetchSequence([{ errorStatus: 503 }]); + + await expect(poll({ sourceId: "src-1" }, rpc, pollDeps(fetchImpl))).rejects.toThrow(/503/); + expect(cursorsPersisted(calls)).toEqual([]); // nothing persisted on a hard failure + }); + + it("stops without throwing on a mid-walk fetch error, keeping prior progress", async () => { + const { rpc, calls } = makePollRpc({ tracked: [{ seriesId: "uuid-a", externalIds: { mangabaka: "9741" } }], }); - const { storage, sets } = makePollStorage("kept"); - const fetchImpl = vi - .fn() - .mockResolvedValue(new Response("boom", { status: 503 })) as unknown as typeof fetch; - - const res = await poll({ sourceId: "src-1" }, rpc, pollDeps(storage, fetchImpl)); - expect(res).toMatchObject({ parsed: 0, upstreamStatus: 503 }); - expect(sets).toEqual([]); // never advanced past the failing page + const { fetchImpl } = makeFetchSequence([ + { items: [item(87, "mangabaka", "9741", 16)], hasMore: true, nextCursor: "c1" }, + { errorStatus: 503 }, + ]); + + const res = await poll({ sourceId: "src-1" }, rpc, pollDeps(fetchImpl)); + expect(res).toMatchObject({ + parsed: 1, + matched: 1, + recorded: 1, + upstreamStatus: 503, + etag: "c1", + }); + expect(cursorsPersisted(calls)).toEqual(["c1"]); }); }); diff --git a/plugins/release-tsundoku/src/index.ts b/plugins/release-tsundoku/src/index.ts index 0fba8c2a..53e181cd 100644 --- a/plugins/release-tsundoku/src/index.ts +++ b/plugins/release-tsundoku/src/index.ts @@ -29,7 +29,6 @@ import { type HostRpcClient, HostRpcError, type InitializeParams, - type PluginStorage, RELEASES_METHODS, type ReleaseCandidate, type ReleasePollRequest, @@ -43,9 +42,6 @@ import { buildIndex, matchItem } from "./matcher.js"; const logger = createLogger({ name: manifest.name, level: "info" }); -/** KV-store key under which the feed cursor bookmark is persisted. */ -export const CURSOR_STORAGE_KEY = "feed_cursor"; - /** Default feed page size when config omits / mis-types `pageLimit`. */ const DEFAULT_PAGE_LIMIT = 100; /** Tsundoku caps the feed page size at 500. */ @@ -62,7 +58,6 @@ const DEFAULT_LANGUAGE = "en"; interface PluginState { hostRpc: HostRpcClient | null; - storage: PluginStorage | null; /** Tsundoku instance base URL (no trailing slash), e.g. `https://t.example.com`. */ baseUrl: string; /** ISO 639-1 tag stamped on every candidate (the feed carries none). */ @@ -75,7 +70,6 @@ interface PluginState { const state: PluginState = { hostRpc: null, - storage: null, baseUrl: "", defaultLanguage: DEFAULT_LANGUAGE, pageLimit: DEFAULT_PAGE_LIMIT, @@ -85,7 +79,6 @@ const state: PluginState = { /** Reset state. Exported for tests; not part of the plugin contract. */ export function _resetState(): void { state.hostRpc = null; - state.storage = null; state.baseUrl = ""; state.defaultLanguage = DEFAULT_LANGUAGE; state.pageLimit = DEFAULT_PAGE_LIMIT; @@ -98,35 +91,40 @@ export function normalizeBaseUrl(raw: string): string { } // ============================================================================= -// Cursor persistence (plugin KV store) +// Cursor persistence (per-source state) // ============================================================================= +// +// The feed cursor is persisted in the source's `etag` slot via +// `releases/source_state`, NOT the plugin KV store: the KV store is scoped +// per *user*, and a release source is a *system* plugin with no user context +// (the host rejects `storage/*` with "Storage is not available"). The host +// passes the saved etag back on every poll as `params.etag` and persists the +// `etag` we return on the poll response, so the source-state slot is the +// natural single-row bookmark for this single-feed source. /** - * Load the feed cursor bookmark from the KV store. Returns `null` when no - * cursor has been stored yet (first run) or when the read fails — a missing - * cursor simply restarts the walk from the beginning, which is safe given - * at-least-once delivery + host-side dedup. + * The cursor the host persisted from the previous poll. The host hands it + * back as `ReleasePollRequest.etag`; an empty/absent value restarts the walk + * from the beginning (safe — keyset pagination is gap-free and the host + * dedups re-delivered items). */ -export async function loadCursor(storage: PluginStorage): Promise { - try { - const res = await storage.get(CURSOR_STORAGE_KEY); - const data = res?.data; - return typeof data === "string" && data.length > 0 ? data : null; - } catch (err) { - const reason = err instanceof Error ? err.message : String(err); - logger.warn(`failed to load cursor; restarting from the beginning: ${reason}`); - return null; - } +export function loadCursor(params: ReleasePollRequest): string | null { + return params.etag && params.etag.length > 0 ? params.etag : null; } /** - * Persist the feed cursor bookmark. Best-effort: a failed write is logged but - * never aborts a poll — the worst case is re-walking already-seen pages on - * the next poll, which dedups host-side. + * Persist the feed cursor into the source's `etag` slot. Best-effort: a failed + * write is logged but never aborts a poll. The cursor is also returned on the + * poll response, which the host persists at the end of the poll — the per-page + * write here is what lets a long or interrupted walk resume mid-feed. */ -export async function saveCursor(storage: PluginStorage, cursor: string): Promise { +export async function saveCursor( + rpc: HostRpcClient, + sourceId: string, + cursor: string, +): Promise { try { - await storage.set(CURSOR_STORAGE_KEY, cursor); + await rpc.call(RELEASES_METHODS.SOURCE_STATE_SET, { sourceId, etag: cursor }); } catch (err) { const reason = err instanceof Error ? err.message : String(err); logger.warn(`failed to persist cursor "${cursor}": ${reason}`); @@ -261,7 +259,6 @@ async function reportProgress( /** Dependencies a poll needs, defaulted from plugin state at the call site. */ export interface PollDeps { - storage: PluginStorage; /** Tsundoku base URL (no trailing slash). */ baseUrl: string; /** Language stamped on every candidate. */ @@ -305,12 +302,13 @@ export async function poll( } // 2. Walk the feed from the stored cursor. - let cursor = await loadCursor(deps.storage); + let cursor = loadCursor(params); let parsed = 0; let matched = 0; let recorded = 0; let deduped = 0; let worstStatus = 200; + let pagesFetched = 0; while (true) { const result = await fetchFeedPage(deps.baseUrl, cursor, deps.pageLimit, { @@ -320,12 +318,20 @@ export async function poll( if (result.kind === "error") { worstStatus = Math.max(worstStatus, result.status); + // Couldn't fetch even the first page: surface a hard failure so the host + // records `last_error` and the source shows it (e.g. an unreachable or + // misconfigured `baseUrl`). A mid-walk failure, by contrast, keeps the + // pages already processed and just stops — the cursor is preserved. + if (pagesFetched === 0) { + throw new Error(`feed fetch failed (status ${result.status}): ${result.message}`); + } logger.warn( `feed fetch failed (status ${result.status}): ${result.message}; stopping walk, cursor preserved`, ); break; } + pagesFetched++; const page = result.data; for (const item of page.items) { parsed++; @@ -350,7 +356,7 @@ export async function poll( const next = page.nextCursor ?? null; if (next) { cursor = next; - await saveCursor(deps.storage, next); + await saveCursor(rpc, sourceId, next); } await reportProgress(rpc, parsed, parsed, `Processed ${parsed} feed items`); @@ -375,6 +381,9 @@ export async function poll( matched, recorded, deduped, + // Return the final cursor so the host persists it on the source row even + // if the per-page `source_state/set` writes were dropped. + ...(cursor !== null ? { etag: cursor } : {}), }; } @@ -386,14 +395,13 @@ createReleaseSourcePlugin({ manifest, provider: { async poll(params: ReleasePollRequest): Promise { - if (!state.hostRpc || !state.storage) { - throw new Error("Plugin not initialized: host RPC / storage client missing"); + if (!state.hostRpc) { + throw new Error("Plugin not initialized: host RPC client missing"); } if (!state.baseUrl) { throw new Error("Plugin not configured: baseUrl is required"); } return poll(params, state.hostRpc, { - storage: state.storage, baseUrl: state.baseUrl, language: state.defaultLanguage, pageLimit: state.pageLimit, @@ -404,7 +412,6 @@ createReleaseSourcePlugin({ logLevel: "info", async onInitialize(params: InitializeParams) { state.hostRpc = params.hostRpc; - state.storage = params.storage; const ac = params.adminConfig ?? {}; if (typeof ac.baseUrl === "string") { From 9b859728a9fcdc7c0427c747fbfc62160c99fd0e Mon Sep 17 00:00:00 2001 From: Sylvain Cau Date: Mon, 8 Jun 2026 17:28:01 -0700 Subject: [PATCH 06/10] feat(plugins): add system-scoped plugin KV store and use it for the tsundoku cursor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The plugin KV store (storage/*) was keyed by user_plugin_id, so system plugins — which have no user context — were rejected with "Storage is not available". Add a parallel per-plugin store so system plugins (e.g. release sources) get durable key-value storage. - DB: new plugin_data table keyed by plugin_id (FK to plugins, cascade, unique (plugin_id, key), TTL + indexes), a matching entity, and PluginDataRepository mirroring the per-user repository. - Services: StorageRequestHandler is now scope-aware (user vs system) behind a normalized entry type, with a system constructor; the system spawn path installs a system-scoped storage handler keyed by the plugin row. The per-user storage path is unchanged. - Tasks: the expired-data cleanup now sweeps both the user and system tables. Migrate the Tsundoku release plugin's feed cursor from the source-state etag slot to this KV store (feed_cursor), removing the bespoke source_state round-trips. The source-state etag slot remains for its intended use (HTTP ETag conditional GET); only the cursor moved. Also drop the registerSources retry loop, now redundant thanks to the host's reverse-RPC readiness barrier. Includes repository, handler, and plugin tests. --- crates/codex-db/src/entities/mod.rs | 3 + crates/codex-db/src/entities/plugin_data.rs | 81 +++++ crates/codex-db/src/entities/prelude.rs | 3 + crates/codex-db/src/repositories/mod.rs | 7 + .../codex-db/src/repositories/plugin_data.rs | 290 ++++++++++++++++++ crates/codex-services/src/plugin/manager.rs | 9 +- .../src/plugin/storage_handler.rs | 223 +++++++++++--- .../src/handlers/cleanup_plugin_data.rs | 14 +- migration/src/lib.rs | 3 + .../m20260608_000095_create_plugin_data.rs | 117 +++++++ plugins/release-tsundoku/README.md | 9 +- plugins/release-tsundoku/src/index.test.ts | 153 +++++---- plugins/release-tsundoku/src/index.ts | 106 +++---- 13 files changed, 849 insertions(+), 169 deletions(-) create mode 100644 crates/codex-db/src/entities/plugin_data.rs create mode 100644 crates/codex-db/src/repositories/plugin_data.rs create mode 100644 migration/src/m20260608_000095_create_plugin_data.rs diff --git a/crates/codex-db/src/entities/mod.rs b/crates/codex-db/src/entities/mod.rs index b96c4bac..3a2c78e9 100644 --- a/crates/codex-db/src/entities/mod.rs +++ b/crates/codex-db/src/entities/mod.rs @@ -38,6 +38,9 @@ pub mod users; // OIDC authentication pub mod oidc_connections; +// System-scoped plugin KV store (per-plugin, no user context) +pub mod plugin_data; + // User plugin system (per-user plugin instances and data storage) pub mod user_plugin_data; pub mod user_plugins; diff --git a/crates/codex-db/src/entities/plugin_data.rs b/crates/codex-db/src/entities/plugin_data.rs new file mode 100644 index 00000000..e12e412f --- /dev/null +++ b/crates/codex-db/src/entities/plugin_data.rs @@ -0,0 +1,81 @@ +//! Plugin Data entity for system-scoped (per-plugin) key-value storage. +//! +//! Mirrors [`super::user_plugin_data`] but is keyed by `plugin_id` rather than +//! `user_plugin_id`. System plugins (e.g. release sources) run with no user +//! context, so they can't use the per-user store; this is their durable KV +//! bucket — used, for example, to persist a release feed cursor. +//! +//! ## Storage Isolation +//! +//! Each entry is scoped to a specific `plugin_id`. Plugins can only address +//! their own data by key; the host resolves the plugin scope from the +//! connection context. +//! +//! ## TTL Support +//! +//! Entries can optionally have an `expires_at` timestamp for cached data. A +//! background cleanup task removes expired entries periodically. + +use chrono::{DateTime, Utc}; +use sea_orm::entity::prelude::*; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq, Serialize, Deserialize)] +#[sea_orm(table_name = "plugin_data")] +pub struct Model { + /// Unique identifier for this data entry + #[sea_orm(primary_key, auto_increment = false)] + pub id: Uuid, + + /// Reference to the system plugin (provides plugin scoping) + pub plugin_id: Uuid, + + /// Storage key (e.g., "feed_cursor") + pub key: String, + + /// Plugin-managed JSON data + pub data: serde_json::Value, + + /// Optional TTL — entry is considered expired after this timestamp + pub expires_at: Option>, + + /// When this entry was first created + pub created_at: DateTime, + + /// When this entry was last updated + pub updated_at: DateTime, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation { + #[sea_orm( + belongs_to = "super::plugins::Entity", + from = "Column::PluginId", + to = "super::plugins::Column::Id", + on_delete = "Cascade" + )] + Plugin, +} + +impl Related for Entity { + fn to() -> RelationDef { + Relation::Plugin.def() + } +} + +impl ActiveModelBehavior for ActiveModel {} + +// ============================================================================= +// Helper Methods +// ============================================================================= + +impl Model { + /// Check if this entry has expired + pub fn is_expired(&self) -> bool { + match self.expires_at { + Some(expires_at) => Utc::now() >= expires_at, + None => false, // No expiry means never expires + } + } +} diff --git a/crates/codex-db/src/entities/prelude.rs b/crates/codex-db/src/entities/prelude.rs index 475e3e59..cfc02564 100644 --- a/crates/codex-db/src/entities/prelude.rs +++ b/crates/codex-db/src/entities/prelude.rs @@ -41,6 +41,9 @@ pub use super::series_duplicates::Entity as SeriesDuplicates; pub use super::series_external_ids::Entity as SeriesExternalIds; pub use super::series_metadata::Entity as SeriesMetadata; +// System-scoped plugin KV store +#[allow(unused_imports)] +pub use super::plugin_data::Entity as PluginData; // User plugin system #[allow(unused_imports)] pub use super::user_plugin_data::Entity as UserPluginData; diff --git a/crates/codex-db/src/repositories/mod.rs b/crates/codex-db/src/repositories/mod.rs index 62e88a91..1b7522ac 100644 --- a/crates/codex-db/src/repositories/mod.rs +++ b/crates/codex-db/src/repositories/mod.rs @@ -50,6 +50,9 @@ pub mod access_group; // OIDC authentication pub mod oidc_connection; +// System-scoped plugin KV store (per-plugin, no user context) +pub mod plugin_data; + // User plugin system pub mod user_plugin_data; pub mod user_plugins; @@ -119,6 +122,10 @@ pub use access_group::AccessGroupRepository; // OIDC authentication pub use oidc_connection::OidcConnectionRepository; +// System-scoped plugin KV store +#[allow(unused_imports)] +pub use plugin_data::PluginDataRepository; + // User plugin system #[allow(unused_imports)] pub use user_plugin_data::UserPluginDataRepository; diff --git a/crates/codex-db/src/repositories/plugin_data.rs b/crates/codex-db/src/repositories/plugin_data.rs new file mode 100644 index 00000000..3113655b --- /dev/null +++ b/crates/codex-db/src/repositories/plugin_data.rs @@ -0,0 +1,290 @@ +//! Plugin Data Repository (system-scoped) +//! +//! Key-value storage operations for system plugins, keyed by `plugin_id`. +//! The per-user counterpart is [`super::user_plugin_data`]; this one exists +//! so plugins with no user context (e.g. release sources) get a durable KV +//! bucket — used, for example, to persist a release feed cursor. + +#![allow(dead_code)] + +use crate::entities::plugin_data::{self, Entity as PluginData}; +use anyhow::Result; +use chrono::{DateTime, Utc}; +use sea_orm::*; +use uuid::Uuid; + +pub struct PluginDataRepository; + +impl PluginDataRepository { + // ========================================================================= + // Read Operations + // ========================================================================= + + /// Get a value by key for a plugin. Returns `None` if the key doesn't + /// exist or the entry has expired (expired entries are auto-deleted). + pub async fn get( + db: &DatabaseConnection, + plugin_id: Uuid, + key: &str, + ) -> Result> { + let entry = PluginData::find() + .filter(plugin_data::Column::PluginId.eq(plugin_id)) + .filter(plugin_data::Column::Key.eq(key)) + .one(db) + .await?; + + match entry { + Some(e) if e.is_expired() => { + PluginData::delete_by_id(e.id).exec(db).await?; + Ok(None) + } + other => Ok(other), + } + } + + /// List all (non-expired) keys for a plugin. + pub async fn list_keys( + db: &DatabaseConnection, + plugin_id: Uuid, + ) -> Result> { + let entries = PluginData::find() + .filter(plugin_data::Column::PluginId.eq(plugin_id)) + .filter( + Condition::any() + .add(plugin_data::Column::ExpiresAt.is_null()) + .add(plugin_data::Column::ExpiresAt.gt(Utc::now())), + ) + .order_by_asc(plugin_data::Column::Key) + .all(db) + .await?; + Ok(entries) + } + + // ========================================================================= + // Write Operations + // ========================================================================= + + /// Set a value by key (upsert — creates or updates). + pub async fn set( + db: &DatabaseConnection, + plugin_id: Uuid, + key: &str, + data: serde_json::Value, + expires_at: Option>, + ) -> Result { + let now = Utc::now(); + + let existing = PluginData::find() + .filter(plugin_data::Column::PluginId.eq(plugin_id)) + .filter(plugin_data::Column::Key.eq(key)) + .one(db) + .await?; + + match existing { + Some(entry) => { + let mut active_model: plugin_data::ActiveModel = entry.into(); + active_model.data = Set(data); + active_model.expires_at = Set(expires_at); + active_model.updated_at = Set(now); + Ok(active_model.update(db).await?) + } + None => { + let entry = plugin_data::ActiveModel { + id: Set(Uuid::new_v4()), + plugin_id: Set(plugin_id), + key: Set(key.to_string()), + data: Set(data), + expires_at: Set(expires_at), + created_at: Set(now), + updated_at: Set(now), + }; + Ok(entry.insert(db).await?) + } + } + } + + // ========================================================================= + // Delete Operations + // ========================================================================= + + /// Delete a value by key. Returns true if the key existed. + pub async fn delete(db: &DatabaseConnection, plugin_id: Uuid, key: &str) -> Result { + let result = PluginData::delete_many() + .filter(plugin_data::Column::PluginId.eq(plugin_id)) + .filter(plugin_data::Column::Key.eq(key)) + .exec(db) + .await?; + Ok(result.rows_affected > 0) + } + + /// Clear all data for a plugin. Returns the number of entries deleted. + pub async fn clear_all(db: &DatabaseConnection, plugin_id: Uuid) -> Result { + let result = PluginData::delete_many() + .filter(plugin_data::Column::PluginId.eq(plugin_id)) + .exec(db) + .await?; + Ok(result.rows_affected) + } + + /// Cleanup expired data across all plugins. Intended for a background task. + /// Returns the number of expired entries deleted. + pub async fn cleanup_expired(db: &DatabaseConnection) -> Result { + let result = PluginData::delete_many() + .filter(plugin_data::Column::ExpiresAt.is_not_null()) + .filter(plugin_data::Column::ExpiresAt.lte(Utc::now())) + .exec(db) + .await?; + Ok(result.rows_affected) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::repositories::PluginsRepository; + use crate::test_helpers::setup_test_db; + use chrono::Duration; + + /// Create a system plugin row and return its id. + async fn make_plugin(db: &DatabaseConnection, name: &str) -> Uuid { + let plugin = PluginsRepository::create( + db, + name, + name, + None, + "system", + "node", + vec!["x".to_string()], + vec![], + None, + vec![], + vec![], + vec![], + None, + "env", + None, + true, + None, + None, + None, + ) + .await + .expect("create plugin"); + plugin.id + } + + #[tokio::test] + async fn set_get_roundtrip() { + let db = setup_test_db().await; + let conn = &db; + let plugin_id = make_plugin(conn, "release-x").await; + + PluginDataRepository::set( + conn, + plugin_id, + "feed_cursor", + serde_json::json!("abc"), + None, + ) + .await + .unwrap(); + let got = PluginDataRepository::get(conn, plugin_id, "feed_cursor") + .await + .unwrap() + .expect("entry"); + assert_eq!(got.data, serde_json::json!("abc")); + } + + #[tokio::test] + async fn set_upserts_in_place() { + let db = setup_test_db().await; + let conn = &db; + let plugin_id = make_plugin(conn, "release-x").await; + + PluginDataRepository::set(conn, plugin_id, "k", serde_json::json!(1), None) + .await + .unwrap(); + PluginDataRepository::set(conn, plugin_id, "k", serde_json::json!(2), None) + .await + .unwrap(); + let keys = PluginDataRepository::list_keys(conn, plugin_id) + .await + .unwrap(); + assert_eq!(keys.len(), 1); + assert_eq!(keys[0].data, serde_json::json!(2)); + } + + #[tokio::test] + async fn data_is_isolated_per_plugin() { + let db = setup_test_db().await; + let conn = &db; + let a = make_plugin(conn, "plugin-a").await; + let b = make_plugin(conn, "plugin-b").await; + + PluginDataRepository::set(conn, a, "k", serde_json::json!("a"), None) + .await + .unwrap(); + assert!( + PluginDataRepository::get(conn, b, "k") + .await + .unwrap() + .is_none() + ); + } + + #[tokio::test] + async fn expired_entries_are_hidden_and_cleaned() { + let db = setup_test_db().await; + let conn = &db; + let plugin_id = make_plugin(conn, "release-x").await; + + let past = Utc::now() - Duration::hours(1); + PluginDataRepository::set(conn, plugin_id, "k", serde_json::json!(1), Some(past)) + .await + .unwrap(); + + // Hidden on read (and auto-deleted)... + assert!( + PluginDataRepository::get(conn, plugin_id, "k") + .await + .unwrap() + .is_none() + ); + // ...and counted by the cleanup sweep when present. + PluginDataRepository::set(conn, plugin_id, "k2", serde_json::json!(1), Some(past)) + .await + .unwrap(); + let removed = PluginDataRepository::cleanup_expired(conn).await.unwrap(); + assert!(removed >= 1); + } + + #[tokio::test] + async fn delete_and_clear() { + let db = setup_test_db().await; + let conn = &db; + let plugin_id = make_plugin(conn, "release-x").await; + + PluginDataRepository::set(conn, plugin_id, "k1", serde_json::json!(1), None) + .await + .unwrap(); + PluginDataRepository::set(conn, plugin_id, "k2", serde_json::json!(2), None) + .await + .unwrap(); + + assert!( + PluginDataRepository::delete(conn, plugin_id, "k1") + .await + .unwrap() + ); + assert!( + !PluginDataRepository::delete(conn, plugin_id, "missing") + .await + .unwrap() + ); + + let cleared = PluginDataRepository::clear_all(conn, plugin_id) + .await + .unwrap(); + assert_eq!(cleared, 1); + } +} diff --git a/crates/codex-services/src/plugin/manager.rs b/crates/codex-services/src/plugin/manager.rs index f7630a68..a728bb36 100644 --- a/crates/codex-services/src/plugin/manager.rs +++ b/crates/codex-services/src/plugin/manager.rs @@ -646,7 +646,14 @@ impl PluginManager { // Need to spawn/initialize the plugin let handle_config = self.create_plugin_config(&entry.db_config).await?; - let mut handle = PluginHandle::new(handle_config).with_release_db(self.db.as_ref().clone()); + // System plugins have no user context, so they get a per-plugin KV + // bucket keyed by `plugins.id` (used e.g. by release sources to persist + // a feed cursor). Per-user storage is wired separately on the + // user-plugin spawn path. + let storage_handler = + StorageRequestHandler::new_system(self.db.as_ref().clone(), plugin_id); + let mut handle = PluginHandle::new_with_storage(handle_config, storage_handler) + .with_release_db(self.db.as_ref().clone()); if let Some(ref s) = self.scheduler { handle = handle.with_scheduler(s.clone()); } diff --git a/crates/codex-services/src/plugin/storage_handler.rs b/crates/codex-services/src/plugin/storage_handler.rs index fc710a53..c2074b9d 100644 --- a/crates/codex-services/src/plugin/storage_handler.rs +++ b/crates/codex-services/src/plugin/storage_handler.rs @@ -19,7 +19,8 @@ use super::storage::{ StorageGetResponse, StorageKeyEntry, StorageListResponse, StorageSetRequest, StorageSetResponse, }; -use codex_db::repositories::UserPluginDataRepository; +use anyhow::Result; +use codex_db::repositories::{PluginDataRepository, UserPluginDataRepository}; /// Maximum number of storage keys allowed per plugin instance const MAX_KEYS_PER_PLUGIN: usize = 100; @@ -27,21 +28,147 @@ const MAX_KEYS_PER_PLUGIN: usize = 100; /// Maximum serialized size of a single storage value (1 MB) const MAX_VALUE_SIZE_BYTES: usize = 1_048_576; +/// Which storage bucket a handler is bound to. +/// +/// User plugins get a per-(user, plugin) bucket; system plugins (which have no +/// user context) get a per-plugin bucket. The handler is created per-connection +/// with exactly one scope, so a plugin can only ever address its own data. +#[derive(Clone, Copy)] +enum StorageScope { + /// Per-(user, plugin) bucket, keyed by `user_plugins.id`. + User(Uuid), + /// Per-plugin bucket, keyed by `plugins.id` (system plugins). + System(Uuid), +} + +impl StorageScope { + fn describe(&self) -> String { + match self { + StorageScope::User(id) => format!("user_plugin:{id}"), + StorageScope::System(id) => format!("plugin:{id}"), + } + } +} + +/// A storage entry normalized across the user- and system-scoped tables, so +/// the JSON-RPC handlers don't care which backing table produced it. +struct StoredEntry { + key: String, + data: Value, + expires_at: Option>, + updated_at: DateTime, +} + +impl From for StoredEntry { + fn from(m: codex_db::entities::user_plugin_data::Model) -> Self { + Self { + key: m.key, + data: m.data, + expires_at: m.expires_at, + updated_at: m.updated_at, + } + } +} + +impl From for StoredEntry { + fn from(m: codex_db::entities::plugin_data::Model) -> Self { + Self { + key: m.key, + data: m.data, + expires_at: m.expires_at, + updated_at: m.updated_at, + } + } +} + /// Handles storage requests from plugins. /// -/// This handler is created per-connection with a specific `user_plugin_id`, -/// providing architectural isolation - each handler can only access its own -/// user-plugin instance's data. +/// This handler is created per-connection bound to a single [`StorageScope`], +/// providing architectural isolation — each handler can only access its own +/// bucket's data. #[derive(Clone)] pub struct StorageRequestHandler { db: DatabaseConnection, - user_plugin_id: Uuid, + scope: StorageScope, } impl StorageRequestHandler { - /// Create a new storage handler for a specific user-plugin instance + /// Create a new storage handler for a specific user-plugin instance. pub fn new(db: DatabaseConnection, user_plugin_id: Uuid) -> Self { - Self { db, user_plugin_id } + Self { + db, + scope: StorageScope::User(user_plugin_id), + } + } + + /// Create a new storage handler for a system plugin (no user context), + /// scoped by the `plugins.id` row. + pub fn new_system(db: DatabaseConnection, plugin_id: Uuid) -> Self { + Self { + db, + scope: StorageScope::System(plugin_id), + } + } + + // ========================================================================= + // Scope-dispatched data access (routes to the right backing table) + // ========================================================================= + + async fn data_get(&self, key: &str) -> Result> { + Ok(match self.scope { + StorageScope::User(id) => UserPluginDataRepository::get(&self.db, id, key) + .await? + .map(StoredEntry::from), + StorageScope::System(id) => PluginDataRepository::get(&self.db, id, key) + .await? + .map(StoredEntry::from), + }) + } + + async fn data_list_keys(&self) -> Result> { + Ok(match self.scope { + StorageScope::User(id) => UserPluginDataRepository::list_keys(&self.db, id) + .await? + .into_iter() + .map(StoredEntry::from) + .collect(), + StorageScope::System(id) => PluginDataRepository::list_keys(&self.db, id) + .await? + .into_iter() + .map(StoredEntry::from) + .collect(), + }) + } + + async fn data_set( + &self, + key: &str, + data: Value, + expires_at: Option>, + ) -> Result<()> { + match self.scope { + StorageScope::User(id) => { + UserPluginDataRepository::set(&self.db, id, key, data, expires_at).await?; + } + StorageScope::System(id) => { + PluginDataRepository::set(&self.db, id, key, data, expires_at).await?; + } + } + Ok(()) + } + + async fn data_delete(&self, key: &str) -> Result { + Ok(match self.scope { + StorageScope::User(id) => UserPluginDataRepository::delete(&self.db, id, key).await?, + StorageScope::System(id) => PluginDataRepository::delete(&self.db, id, key).await?, + }) + } + + async fn data_clear_all(&self) -> Result { + Ok(match self.scope { + StorageScope::User(id) => UserPluginDataRepository::clear_all(&self.db, id).await?, + StorageScope::System(id) => PluginDataRepository::clear_all(&self.db, id).await?, + }) } /// Handle a storage JSON-RPC request and return a response @@ -51,7 +178,7 @@ impl StorageRequestHandler { debug!( method = method, - user_plugin_id = %self.user_plugin_id, + scope = %self.scope.describe(), "Handling storage request" ); @@ -79,7 +206,7 @@ impl StorageRequestHandler { Err(resp) => return resp.with_id(id), }; - match UserPluginDataRepository::get(&self.db, self.user_plugin_id, ¶ms.key).await { + match self.data_get(¶ms.key).await { Ok(Some(entry)) => { let response = StorageGetResponse { data: Some(entry.data), @@ -118,7 +245,7 @@ impl StorageRequestHandler { .unwrap_or(0); if serialized_size > MAX_VALUE_SIZE_BYTES { warn!( - user_plugin_id = %self.user_plugin_id, + scope = %self.scope.describe(), key = %params.key, size = serialized_size, max = MAX_VALUE_SIZE_BYTES, @@ -137,27 +264,23 @@ impl StorageRequestHandler { } // Enforce key count limit (only for new keys, not upserts) - let is_new_key = - match UserPluginDataRepository::get(&self.db, self.user_plugin_id, ¶ms.key).await { - Ok(existing) => existing.is_none(), - Err(e) => { - error!(error = %e, "Storage key existence check failed"); - return JsonRpcResponse::error( - Some(id), - JsonRpcError::new( - error_codes::INTERNAL_ERROR, - format!("Storage error: {}", e), - ), - ); - } - }; + let is_new_key = match self.data_get(¶ms.key).await { + Ok(existing) => existing.is_none(), + Err(e) => { + error!(error = %e, "Storage key existence check failed"); + return JsonRpcResponse::error( + Some(id), + JsonRpcError::new(error_codes::INTERNAL_ERROR, format!("Storage error: {}", e)), + ); + } + }; if is_new_key { - match UserPluginDataRepository::list_keys(&self.db, self.user_plugin_id).await { + match self.data_list_keys().await { Ok(keys) => { if keys.len() >= MAX_KEYS_PER_PLUGIN { warn!( - user_plugin_id = %self.user_plugin_id, + scope = %self.scope.describe(), key_count = keys.len(), max = MAX_KEYS_PER_PLUGIN, "Storage key limit exceeded" @@ -205,15 +328,7 @@ impl StorageRequestHandler { None => None, }; - match UserPluginDataRepository::set( - &self.db, - self.user_plugin_id, - ¶ms.key, - params.data, - expires_at, - ) - .await - { + match self.data_set(¶ms.key, params.data, expires_at).await { Ok(_) => { let response = StorageSetResponse { success: true }; JsonRpcResponse::success(id, serde_json::to_value(response).unwrap()) @@ -236,7 +351,7 @@ impl StorageRequestHandler { Err(resp) => return resp.with_id(id), }; - match UserPluginDataRepository::delete(&self.db, self.user_plugin_id, ¶ms.key).await { + match self.data_delete(¶ms.key).await { Ok(deleted) => { let response = StorageDeleteResponse { deleted }; JsonRpcResponse::success(id, serde_json::to_value(response).unwrap()) @@ -254,7 +369,7 @@ impl StorageRequestHandler { async fn handle_list(&self, request: &JsonRpcRequest) -> JsonRpcResponse { let id = request.id.clone(); - match UserPluginDataRepository::list_keys(&self.db, self.user_plugin_id).await { + match self.data_list_keys().await { Ok(entries) => { let keys: Vec = entries .into_iter() @@ -280,7 +395,7 @@ impl StorageRequestHandler { async fn handle_clear(&self, request: &JsonRpcRequest) -> JsonRpcResponse { let id = request.id.clone(); - match UserPluginDataRepository::clear_all(&self.db, self.user_plugin_id).await { + match self.data_clear_all().await { Ok(count) => { let response = StorageClearResponse { deleted_count: count, @@ -396,6 +511,13 @@ mod tests { (handler, user_plugin.id) } + /// A system-scoped handler keyed by the `plugins` row (no user context). + async fn setup_system_handler(db: &DatabaseConnection) -> (StorageRequestHandler, Uuid) { + let plugin = create_test_plugin(db).await; + let handler = StorageRequestHandler::new_system(db.clone(), plugin.id); + (handler, plugin.id) + } + fn make_request(method: &str, params: Option) -> JsonRpcRequest { JsonRpcRequest::new(1i64, method, params) } @@ -734,6 +856,31 @@ mod tests { assert!(err.message.contains("key limit exceeded")); } + #[tokio::test] + async fn test_system_scope_set_get_and_isolation() { + let db = setup_test_db().await; + let (handler, _) = setup_system_handler(&db).await; + + // A system plugin (no user context) can persist and read back data. + let set = make_request( + "storage/set", + Some(json!({"key": "feed_cursor", "data": "abc"})), + ); + assert!(!handler.handle_request(&set).await.is_error()); + + let get = make_request("storage/get", Some(json!({"key": "feed_cursor"}))); + let resp = handler.handle_request(&get).await; + let result: StorageGetResponse = serde_json::from_value(resp.result.unwrap()).unwrap(); + assert_eq!(result.data.unwrap(), json!("abc")); + + // A different system plugin's bucket is isolated. + let (other, _) = setup_system_handler(&db).await; + let get2 = make_request("storage/get", Some(json!({"key": "feed_cursor"}))); + let resp2 = other.handle_request(&get2).await; + let result2: StorageGetResponse = serde_json::from_value(resp2.result.unwrap()).unwrap(); + assert!(result2.data.is_none()); + } + #[tokio::test] async fn test_storage_upsert_at_key_limit_succeeds() { let db = setup_test_db().await; diff --git a/crates/codex-tasks/src/handlers/cleanup_plugin_data.rs b/crates/codex-tasks/src/handlers/cleanup_plugin_data.rs index 0004a51f..79c5acd2 100644 --- a/crates/codex-tasks/src/handlers/cleanup_plugin_data.rs +++ b/crates/codex-tasks/src/handlers/cleanup_plugin_data.rs @@ -1,9 +1,10 @@ //! Handler for CleanupPluginData task //! -//! Periodically cleans up expired key-value data from plugin storage -//! (`user_plugin_data` table). Entries with a past `expires_at` timestamp -//! are deleted in bulk. Also cleans up expired OAuth state flows from the -//! in-memory `OAuthStateManager` to prevent memory leaks. +//! Periodically cleans up expired key-value data from plugin storage — both +//! the per-user `user_plugin_data` table and the system-scoped `plugin_data` +//! table. Entries with a past `expires_at` timestamp are deleted in bulk. +//! Also cleans up expired OAuth state flows from the in-memory +//! `OAuthStateManager` to prevent memory leaks. use anyhow::Result; use sea_orm::DatabaseConnection; @@ -14,7 +15,7 @@ use tracing::info; use crate::handlers::TaskHandler; use crate::types::TaskResult; use codex_db::entities::tasks; -use codex_db::repositories::UserPluginDataRepository; +use codex_db::repositories::{PluginDataRepository, UserPluginDataRepository}; use codex_events::EventBroadcaster; use codex_services::user_plugin::OAuthStateManager; @@ -46,7 +47,8 @@ impl TaskHandler for CleanupPluginDataHandler { Box::pin(async move { info!("Task {}: Starting plugin data cleanup", task.id); - let deleted_count = UserPluginDataRepository::cleanup_expired(db).await?; + let deleted_count = UserPluginDataRepository::cleanup_expired(db).await? + + PluginDataRepository::cleanup_expired(db).await?; // Clean up expired OAuth pending flows from in-memory state let (oauth_cleaned, oauth_remaining) = diff --git a/migration/src/lib.rs b/migration/src/lib.rs index 6fe0b629..d218b1e1 100644 --- a/migration/src/lib.rs +++ b/migration/src/lib.rs @@ -192,6 +192,7 @@ mod m20260606_000092_add_plugin_sync_cron; mod m20260607_000093_create_scheduled_firing_claims; // Atomic per-(plugin,user) dedup for scheduled user-plugin syncs mod m20260607_000094_add_user_plugin_sync_unique_index; +mod m20260608_000095_create_plugin_data; pub struct Migrator; @@ -355,6 +356,8 @@ impl MigratorTrait for Migrator { Box::new(m20260607_000093_create_scheduled_firing_claims::Migration), // Atomic per-(plugin,user) dedup for scheduled user-plugin syncs Box::new(m20260607_000094_add_user_plugin_sync_unique_index::Migration), + // System-scoped plugin KV store (keyed by plugin, no user context) + Box::new(m20260608_000095_create_plugin_data::Migration), ] } } diff --git a/migration/src/m20260608_000095_create_plugin_data.rs b/migration/src/m20260608_000095_create_plugin_data.rs new file mode 100644 index 00000000..c20de836 --- /dev/null +++ b/migration/src/m20260608_000095_create_plugin_data.rs @@ -0,0 +1,117 @@ +use sea_orm_migration::prelude::*; + +/// System-scoped plugin key-value store. +/// +/// Mirrors `user_plugin_data` but is keyed by `plugin_id` (the `plugins` row) +/// rather than `user_plugin_id`. System plugins (e.g. release sources) run +/// with no user context, so they can't use the per-user store; this table is +/// their durable KV bucket (used for things like a release feed cursor). +#[derive(DeriveMigrationName)] +pub struct Migration; + +#[async_trait::async_trait] +impl MigrationTrait for Migration { + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .create_table( + Table::create() + .table(PluginData::Table) + .if_not_exists() + .col( + ColumnDef::new(PluginData::Id) + .uuid() + .not_null() + .primary_key(), + ) + // Reference to the system plugin (no user scoping) + .col(ColumnDef::new(PluginData::PluginId).uuid().not_null()) + // Key-value storage + .col(ColumnDef::new(PluginData::Key).text().not_null()) + .col(ColumnDef::new(PluginData::Data).json().not_null()) + // Optional TTL for cached data + .col(ColumnDef::new(PluginData::ExpiresAt).timestamp_with_time_zone()) + // Timestamps + .col( + ColumnDef::new(PluginData::CreatedAt) + .timestamp_with_time_zone() + .not_null(), + ) + .col( + ColumnDef::new(PluginData::UpdatedAt) + .timestamp_with_time_zone() + .not_null(), + ) + // Foreign key — data is removed when the plugin is deleted. + .foreign_key( + ForeignKey::create() + .name("fk_plugin_data_plugin") + .from(PluginData::Table, PluginData::PluginId) + .to(Plugins::Table, Plugins::Id) + .on_delete(ForeignKeyAction::Cascade), + ) + .to_owned(), + ) + .await?; + + // Unique constraint: one value per key per plugin. + manager + .create_index( + Index::create() + .name("idx_plugin_data_plugin_key") + .table(PluginData::Table) + .col(PluginData::PluginId) + .col(PluginData::Key) + .unique() + .to_owned(), + ) + .await?; + + // Index on plugin_id for fast lookups. + manager + .create_index( + Index::create() + .name("idx_plugin_data_plugin_id") + .table(PluginData::Table) + .col(PluginData::PluginId) + .to_owned(), + ) + .await?; + + // Index on expires_at for cleanup of expired data. + manager + .create_index( + Index::create() + .name("idx_plugin_data_expires_at") + .table(PluginData::Table) + .col(PluginData::ExpiresAt) + .to_owned(), + ) + .await?; + + Ok(()) + } + + async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> { + manager + .drop_table(Table::drop().table(PluginData::Table).to_owned()) + .await + } +} + +#[derive(DeriveIden)] +pub enum PluginData { + Table, + Id, + PluginId, + Key, + Data, + ExpiresAt, + CreatedAt, + UpdatedAt, +} + +#[derive(DeriveIden)] +enum Plugins { + Table, + Id, +} diff --git a/plugins/release-tsundoku/README.md b/plugins/release-tsundoku/README.md index 12b34e74..88c9e988 100644 --- a/plugins/release-tsundoku/README.md +++ b/plugins/release-tsundoku/README.md @@ -11,8 +11,9 @@ incremental series feed. **Notification-only** — Codex does not download anyth Shikimori, Anime-Planet, Anime News Network), so announcements always land on the right series with full confidence. - **Incremental, cursor-based.** Walks Tsundoku's keyset-paginated - `/api/v1/series/feed`, persisting its position in the source's state (etag - slot) so each poll only processes activity since the last run. + `/api/v1/series/feed`, persisting its position in the plugin's + (system-scoped) KV store so each poll only processes activity since the last + run. - **Volume- and chapter-aware.** The feed's merged, gap-preserving coverage spans map directly onto Codex's release model. @@ -63,8 +64,8 @@ providers, in match-priority order: On each poll the plugin: -1. Loads its stored feed cursor (the host hands it back as the source's - persisted `etag`). +1. Loads its stored feed cursor from the plugin's system-scoped KV store + (`feed_cursor`). 2. Builds a reverse index (`provider:id → Codex series`) from your tracked series via the host's `releases/list_tracked`. 3. Walks the feed from the cursor. Each item is matched against the index by diff --git a/plugins/release-tsundoku/src/index.test.ts b/plugins/release-tsundoku/src/index.test.ts index 34042e5c..e70d2250 100644 --- a/plugins/release-tsundoku/src/index.test.ts +++ b/plugins/release-tsundoku/src/index.test.ts @@ -1,7 +1,14 @@ -import { HostRpcClient } from "@ashdev/codex-plugin-sdk"; +import { HostRpcClient, type PluginStorage } from "@ashdev/codex-plugin-sdk"; import { describe, expect, it, vi } from "vitest"; import type { FeedItem, FeedResponse } from "./fetcher.js"; -import { loadCursor, normalizeBaseUrl, poll, registerSources, saveCursor } from "./index.js"; +import { + CURSOR_STORAGE_KEY, + loadCursor, + normalizeBaseUrl, + poll, + registerSources, + saveCursor, +} from "./index.js"; // ----------------------------------------------------------------------------- // Mock host RPC @@ -76,32 +83,64 @@ describe("normalizeBaseUrl", () => { }); // ----------------------------------------------------------------------------- -// Cursor persistence (per-source state / etag slot) +// Cursor persistence (system-scoped KV store) // ----------------------------------------------------------------------------- +/** A stateful in-memory `PluginStorage` double recording every `set`. */ +function makeStorage(initialCursor: string | null = null): { + storage: PluginStorage; + sets: string[]; +} { + let value = initialCursor; + const sets: string[] = []; + const storage = { + get: vi.fn(async () => ({ data: value })), + set: vi.fn(async (_key: string, data: unknown) => { + value = data as string; + sets.push(value); + return { success: true }; + }), + } as unknown as PluginStorage; + return { storage, sets }; +} + describe("loadCursor", () => { - it("returns the etag the host passed back from the last poll", () => { - expect(loadCursor({ sourceId: "s", etag: "cursor-42" })).toBe("cursor-42"); + it("returns the stored cursor string", async () => { + const { storage } = makeStorage("cursor-42"); + expect(await loadCursor(storage)).toBe("cursor-42"); }); - it("returns null when no etag was supplied", () => { - expect(loadCursor({ sourceId: "s" })).toBeNull(); - expect(loadCursor({ sourceId: "s", etag: "" })).toBeNull(); + it("returns null when nothing is stored", async () => { + expect(await loadCursor(makeStorage(null).storage)).toBeNull(); + }); + + it("returns null and does not throw when the read fails", async () => { + const storage = { + get: vi.fn(async () => { + throw new Error("kv down"); + }), + set: vi.fn(), + } as unknown as PluginStorage; + expect(await loadCursor(storage)).toBeNull(); }); }); describe("saveCursor", () => { - it("persists the cursor into the source-state etag slot", async () => { - const { rpc, calls } = makeMockRpc(() => ({ success: true })); - await saveCursor(rpc, "src-1", "cursor-99"); - expect(calls).toHaveLength(1); - expect(calls[0].method).toBe("releases/source_state/set"); - expect(calls[0].params).toEqual({ sourceId: "src-1", etag: "cursor-99" }); + it("writes the cursor under the feed-cursor key", async () => { + const { storage, sets } = makeStorage(); + await saveCursor(storage, "cursor-99"); + expect(sets).toEqual(["cursor-99"]); + expect((storage.set as ReturnType).mock.calls[0][0]).toBe(CURSOR_STORAGE_KEY); }); it("swallows a write failure without throwing", async () => { - const { rpc } = makeMockRpc(() => ({ __error: { code: -32000, message: "db error" } })); - await expect(saveCursor(rpc, "src-1", "cursor-99")).resolves.toBeUndefined(); + const storage = { + get: vi.fn(), + set: vi.fn(async () => { + throw new Error("kv full"); + }), + } as unknown as PluginStorage; + await expect(saveCursor(storage, "cursor-99")).resolves.toBeUndefined(); }); }); @@ -126,29 +165,9 @@ describe("registerSources", () => { }); }); - it("retries on METHOD_NOT_FOUND then succeeds", async () => { - const { rpc, calls } = makeMockRpc((_m, _p, attempt) => - attempt < 3 - ? { __error: { code: -32601, message: "method not found" } } - : { registered: 1, pruned: 0 }, - ); - const result = await registerSources(rpc); - - expect(result).toEqual({ registered: 1, pruned: 0 }); - expect(calls.length).toBe(3); - }); - - it("returns null after exhausting retries on METHOD_NOT_FOUND", async () => { - const { rpc, calls } = makeMockRpc(() => ({ - __error: { code: -32601, message: "method not found" }, - })); - const result = await registerSources(rpc); - - expect(result).toBeNull(); - expect(calls.length).toBe(5); - }); - - it("does not retry on a non-METHOD_NOT_FOUND error", async () => { + it("issues a single call (no retry) and returns null on failure", async () => { + // The host's readiness barrier makes registration race-free, so the + // plugin no longer retries: one call, and a failure surfaces as null. const { rpc, calls } = makeMockRpc(() => ({ __error: { code: -32000, message: "db error" }, })); @@ -186,13 +205,6 @@ function makeFetchSequence(pages: PageOrError[]): { return { fetchImpl, urls }; } -/** Cursors persisted via `releases/source_state/set`, in order. */ -function cursorsPersisted(calls: CapturedCall[]): string[] { - return calls - .filter((c) => c.method === "releases/source_state/set") - .map((c) => (c.params as { etag: string }).etag); -} - function item( seriesId: number, provider: string, @@ -234,14 +246,12 @@ function makePollRpc(opts: { if (method === "releases/report_progress") { return { emitted: true }; } - if (method === "releases/source_state/set") { - return { success: true }; - } return {}; }); } -const pollDeps = (fetchImpl: typeof fetch) => ({ +const pollDeps = (fetchImpl: typeof fetch, storage: PluginStorage) => ({ + storage, baseUrl: "https://t.example.com", language: "en", pageLimit: 100, @@ -251,9 +261,10 @@ const pollDeps = (fetchImpl: typeof fetch) => ({ describe("poll", () => { it("walks pages, matches by external id, records, and persists the cursor per page", async () => { - const { rpc, calls } = makePollRpc({ + const { rpc } = makePollRpc({ tracked: [{ seriesId: "uuid-a", externalIds: { mangabaka: "9741" } }], }); + const { storage, sets } = makeStorage(); const { fetchImpl } = makeFetchSequence([ { items: [item(87, "mangabaka", "9741", 16), item(99, "anilist", "999")], @@ -263,7 +274,7 @@ describe("poll", () => { { items: [item(87, "mangabaka", "9741", 17)], hasMore: false, nextCursor: "c2" }, ]); - const res = await poll({ sourceId: "src-1" }, rpc, pollDeps(fetchImpl)); + const res = await poll({ sourceId: "src-1" }, rpc, pollDeps(fetchImpl, storage)); expect(res).toMatchObject({ parsed: 3, @@ -271,9 +282,8 @@ describe("poll", () => { recorded: 2, deduped: 0, upstreamStatus: 200, - etag: "c2", }); - expect(cursorsPersisted(calls)).toEqual(["c1", "c2"]); + expect(sets).toEqual(["c1", "c2"]); }); it("counts host dedup separately from inserts", async () => { @@ -281,6 +291,7 @@ describe("poll", () => { tracked: [{ seriesId: "uuid-a", externalIds: { mangabaka: "9741" } }], onRecord: (n) => ({ ledgerId: `l${n}`, deduped: n > 1 }), }); + const { storage } = makeStorage(); const { fetchImpl } = makeFetchSequence([ { items: [item(87, "mangabaka", "9741", 16), item(87, "mangabaka", "9741", 17)], @@ -289,7 +300,7 @@ describe("poll", () => { }, ]); - const res = await poll({ sourceId: "src-1" }, rpc, pollDeps(fetchImpl)); + const res = await poll({ sourceId: "src-1" }, rpc, pollDeps(fetchImpl, storage)); expect(res).toMatchObject({ matched: 2, recorded: 1, deduped: 1 }); }); @@ -297,66 +308,72 @@ describe("poll", () => { const { rpc, calls } = makePollRpc({ tracked: [{ seriesId: "uuid-a", externalIds: { mangabaka: "9741" } }], }); + const { storage } = makeStorage(); const { fetchImpl } = makeFetchSequence([ { items: [item(99, "anilist", "999")], hasMore: false, nextCursor: "c1" }, ]); - const res = await poll({ sourceId: "src-1" }, rpc, pollDeps(fetchImpl)); + const res = await poll({ sourceId: "src-1" }, rpc, pollDeps(fetchImpl, storage)); expect(res).toMatchObject({ parsed: 1, matched: 0, recorded: 0 }); expect(calls.some((c) => c.method === "releases/record")).toBe(false); }); it("tolerates a record failure without aborting the walk", async () => { - const { rpc, calls } = makePollRpc({ + const { rpc } = makePollRpc({ tracked: [{ seriesId: "uuid-a", externalIds: { mangabaka: "9741" } }], onRecord: () => ({ __error: { code: -32000, message: "ledger down" } }), }); + const { storage, sets } = makeStorage(); const { fetchImpl } = makeFetchSequence([ { items: [item(87, "mangabaka", "9741", 16)], hasMore: false, nextCursor: "c1" }, ]); - const res = await poll({ sourceId: "src-1" }, rpc, pollDeps(fetchImpl)); + const res = await poll({ sourceId: "src-1" }, rpc, pollDeps(fetchImpl, storage)); expect(res).toMatchObject({ parsed: 1, matched: 1, recorded: 0, deduped: 0 }); - expect(cursorsPersisted(calls)).toEqual(["c1"]); // walk still completed and advanced the cursor + expect(sets).toEqual(["c1"]); // walk still completed and advanced the cursor }); - it("resumes from the etag the host passed in", async () => { + it("resumes from the cursor stored in the KV store", async () => { const { rpc } = makePollRpc({ tracked: [] }); + const { storage } = makeStorage("resume-here"); const { fetchImpl, urls } = makeFetchSequence([ { items: [], hasMore: false, nextCursor: null }, ]); - await poll({ sourceId: "src-1", etag: "resume-here" }, rpc, pollDeps(fetchImpl)); + await poll({ sourceId: "src-1" }, rpc, pollDeps(fetchImpl, storage)); expect(new URL(urls[0]).searchParams.get("cursor")).toBe("resume-here"); }); it("throws when even the first page can't be fetched (so the source shows last_error)", async () => { - const { rpc, calls } = makePollRpc({ + const { rpc } = makePollRpc({ tracked: [{ seriesId: "uuid-a", externalIds: { mangabaka: "9741" } }], }); + const { storage, sets } = makeStorage(); const { fetchImpl } = makeFetchSequence([{ errorStatus: 503 }]); - await expect(poll({ sourceId: "src-1" }, rpc, pollDeps(fetchImpl))).rejects.toThrow(/503/); - expect(cursorsPersisted(calls)).toEqual([]); // nothing persisted on a hard failure + await expect(poll({ sourceId: "src-1" }, rpc, pollDeps(fetchImpl, storage))).rejects.toThrow( + /503/, + ); + expect(sets).toEqual([]); // nothing persisted on a hard failure }); it("stops without throwing on a mid-walk fetch error, keeping prior progress", async () => { - const { rpc, calls } = makePollRpc({ + const { rpc } = makePollRpc({ tracked: [{ seriesId: "uuid-a", externalIds: { mangabaka: "9741" } }], }); + const { storage, sets } = makeStorage(); const { fetchImpl } = makeFetchSequence([ { items: [item(87, "mangabaka", "9741", 16)], hasMore: true, nextCursor: "c1" }, { errorStatus: 503 }, ]); - const res = await poll({ sourceId: "src-1" }, rpc, pollDeps(fetchImpl)); + const res = await poll({ sourceId: "src-1" }, rpc, pollDeps(fetchImpl, storage)); expect(res).toMatchObject({ parsed: 1, matched: 1, recorded: 1, upstreamStatus: 503, - etag: "c1", }); - expect(cursorsPersisted(calls)).toEqual(["c1"]); + expect(sets).toEqual(["c1"]); }); }); diff --git a/plugins/release-tsundoku/src/index.ts b/plugins/release-tsundoku/src/index.ts index 53e181cd..2b33d3c0 100644 --- a/plugins/release-tsundoku/src/index.ts +++ b/plugins/release-tsundoku/src/index.ts @@ -29,6 +29,7 @@ import { type HostRpcClient, HostRpcError, type InitializeParams, + type PluginStorage, RELEASES_METHODS, type ReleaseCandidate, type ReleasePollRequest, @@ -42,6 +43,9 @@ import { buildIndex, matchItem } from "./matcher.js"; const logger = createLogger({ name: manifest.name, level: "info" }); +/** KV-store key under which the feed cursor bookmark is persisted. */ +export const CURSOR_STORAGE_KEY = "feed_cursor"; + /** Default feed page size when config omits / mis-types `pageLimit`. */ const DEFAULT_PAGE_LIMIT = 100; /** Tsundoku caps the feed page size at 500. */ @@ -58,6 +62,8 @@ const DEFAULT_LANGUAGE = "en"; interface PluginState { hostRpc: HostRpcClient | null; + /** Per-plugin (system-scoped) KV store for the feed cursor bookmark. */ + storage: PluginStorage | null; /** Tsundoku instance base URL (no trailing slash), e.g. `https://t.example.com`. */ baseUrl: string; /** ISO 639-1 tag stamped on every candidate (the feed carries none). */ @@ -70,6 +76,7 @@ interface PluginState { const state: PluginState = { hostRpc: null, + storage: null, baseUrl: "", defaultLanguage: DEFAULT_LANGUAGE, pageLimit: DEFAULT_PAGE_LIMIT, @@ -79,6 +86,7 @@ const state: PluginState = { /** Reset state. Exported for tests; not part of the plugin contract. */ export function _resetState(): void { state.hostRpc = null; + state.storage = null; state.baseUrl = ""; state.defaultLanguage = DEFAULT_LANGUAGE; state.pageLimit = DEFAULT_PAGE_LIMIT; @@ -91,40 +99,40 @@ export function normalizeBaseUrl(raw: string): string { } // ============================================================================= -// Cursor persistence (per-source state) +// Cursor persistence (system-scoped KV store) // ============================================================================= // -// The feed cursor is persisted in the source's `etag` slot via -// `releases/source_state`, NOT the plugin KV store: the KV store is scoped -// per *user*, and a release source is a *system* plugin with no user context -// (the host rejects `storage/*` with "Storage is not available"). The host -// passes the saved etag back on every poll as `params.etag` and persists the -// `etag` we return on the poll response, so the source-state slot is the -// natural single-row bookmark for this single-feed source. +// The feed cursor lives in the plugin's KV store under `feed_cursor`. Release +// sources are *system* plugins (no user context), so this is the per-plugin +// (system) bucket — the host resolves the scope from the connection. Persisted +// after each processed page so a long or interrupted walk resumes mid-feed. /** - * The cursor the host persisted from the previous poll. The host hands it - * back as `ReleasePollRequest.etag`; an empty/absent value restarts the walk - * from the beginning (safe — keyset pagination is gap-free and the host - * dedups re-delivered items). + * Load the feed cursor bookmark from the KV store. Returns `null` when no + * cursor has been stored yet (first run) or when the read fails — a missing + * cursor simply restarts the walk from the beginning, which is safe given + * keyset pagination is gap-free and the host dedups re-delivered items. */ -export function loadCursor(params: ReleasePollRequest): string | null { - return params.etag && params.etag.length > 0 ? params.etag : null; +export async function loadCursor(storage: PluginStorage): Promise { + try { + const res = await storage.get(CURSOR_STORAGE_KEY); + const data = res?.data; + return typeof data === "string" && data.length > 0 ? data : null; + } catch (err) { + const reason = err instanceof Error ? err.message : String(err); + logger.warn(`failed to load cursor; restarting from the beginning: ${reason}`); + return null; + } } /** - * Persist the feed cursor into the source's `etag` slot. Best-effort: a failed - * write is logged but never aborts a poll. The cursor is also returned on the - * poll response, which the host persists at the end of the poll — the per-page - * write here is what lets a long or interrupted walk resume mid-feed. + * Persist the feed cursor bookmark. Best-effort: a failed write is logged but + * never aborts a poll — the worst case is re-walking already-seen pages on the + * next poll, which dedups host-side. */ -export async function saveCursor( - rpc: HostRpcClient, - sourceId: string, - cursor: string, -): Promise { +export async function saveCursor(storage: PluginStorage, cursor: string): Promise { try { - await rpc.call(RELEASES_METHODS.SOURCE_STATE_SET, { sourceId, etag: cursor }); + await storage.set(CURSOR_STORAGE_KEY, cursor); } catch (err) { const reason = err instanceof Error ? err.message : String(err); logger.warn(`failed to persist cursor "${cursor}": ${reason}`); @@ -137,9 +145,11 @@ export async function saveCursor( /** * Register the single static source row representing the Tsundoku feed. The - * whole catalog is polled under one logical source keyed `default`. Retries - * on `METHOD_NOT_FOUND` to absorb the brief race where the host has not yet - * installed the releases reverse-RPC handler at startup. + * whole catalog is polled under one logical source keyed `default`. + * + * No retry needed: the host parks an early reverse-RPC on its readiness + * barrier until the plugin's capabilities + handlers are installed, so this + * single call resolves cleanly even when fired from `onInitialize`. */ export async function registerSources( rpc: HostRpcClient, @@ -152,25 +162,16 @@ export async function registerSources( config: null, }, ]; - const maxAttempts = 5; - for (let attempt = 1; attempt <= maxAttempts; attempt++) { - try { - return await rpc.call<{ registered: number; pruned: number }>( - RELEASES_METHODS.REGISTER_SOURCES, - { sources }, - ); - } catch (err) { - const isMethodNotFound = err instanceof HostRpcError && err.code === -32601; - if (isMethodNotFound && attempt < maxAttempts) { - await new Promise((r) => setTimeout(r, 50 * attempt)); - continue; - } - const reason = err instanceof Error ? err.message : String(err); - logger.error(`register_sources failed: ${reason}`); - return null; - } + try { + return await rpc.call<{ registered: number; pruned: number }>( + RELEASES_METHODS.REGISTER_SOURCES, + { sources }, + ); + } catch (err) { + const reason = err instanceof Error ? err.message : String(err); + logger.error(`register_sources failed: ${reason}`); + return null; } - return null; } // ============================================================================= @@ -259,6 +260,8 @@ async function reportProgress( /** Dependencies a poll needs, defaulted from plugin state at the call site. */ export interface PollDeps { + /** System-scoped KV store holding the feed cursor bookmark. */ + storage: PluginStorage; /** Tsundoku base URL (no trailing slash). */ baseUrl: string; /** Language stamped on every candidate. */ @@ -302,7 +305,7 @@ export async function poll( } // 2. Walk the feed from the stored cursor. - let cursor = loadCursor(params); + let cursor = await loadCursor(deps.storage); let parsed = 0; let matched = 0; let recorded = 0; @@ -356,7 +359,7 @@ export async function poll( const next = page.nextCursor ?? null; if (next) { cursor = next; - await saveCursor(rpc, sourceId, next); + await saveCursor(deps.storage, next); } await reportProgress(rpc, parsed, parsed, `Processed ${parsed} feed items`); @@ -381,9 +384,6 @@ export async function poll( matched, recorded, deduped, - // Return the final cursor so the host persists it on the source row even - // if the per-page `source_state/set` writes were dropped. - ...(cursor !== null ? { etag: cursor } : {}), }; } @@ -395,13 +395,14 @@ createReleaseSourcePlugin({ manifest, provider: { async poll(params: ReleasePollRequest): Promise { - if (!state.hostRpc) { - throw new Error("Plugin not initialized: host RPC client missing"); + if (!state.hostRpc || !state.storage) { + throw new Error("Plugin not initialized: host RPC / storage client missing"); } if (!state.baseUrl) { throw new Error("Plugin not configured: baseUrl is required"); } return poll(params, state.hostRpc, { + storage: state.storage, baseUrl: state.baseUrl, language: state.defaultLanguage, pageLimit: state.pageLimit, @@ -412,6 +413,7 @@ createReleaseSourcePlugin({ logLevel: "info", async onInitialize(params: InitializeParams) { state.hostRpc = params.hostRpc; + state.storage = params.storage; const ac = params.adminConfig ?? {}; if (typeof ac.baseUrl === "string") { From c6888522d546613b8a24e37a2fa0c7076fb82be4 Mon Sep 17 00:00:00 2001 From: Sylvain Cau Date: Mon, 8 Jun 2026 18:57:34 -0700 Subject: [PATCH 07/10] fix(tasks): replay recorded events for failed tasks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In distributed mode the worker records the entity events a task emits and the web TaskListener replays them to SSE subscribers — but only for tasks that completed. Failed tasks dropped their events, so a release poll that errored never delivered its `release_source_polled` event and the Release tracking UI didn't refresh until a manual reload. Carry recorded events through the failure path: `mark_failed` now accepts and persists result data, `fail_task` passes the recorded events into it, and the TaskListener replays recorded events on the Failed terminal state as well as Completed. Adds repository tests for the new result-data persistence. --- crates/codex-db/src/repositories/task.rs | 70 +++++++++++++++++++++- crates/codex-services/src/task_listener.rs | 9 ++- crates/codex-tasks/src/worker.rs | 22 +++++-- 3 files changed, 91 insertions(+), 10 deletions(-) diff --git a/crates/codex-db/src/repositories/task.rs b/crates/codex-db/src/repositories/task.rs index c17476ce..147049eb 100644 --- a/crates/codex-db/src/repositories/task.rs +++ b/crates/codex-db/src/repositories/task.rs @@ -847,7 +847,21 @@ impl TaskRepository { } /// Mark task as failed (will retry if attempts < max_attempts) - pub async fn mark_failed(db: &DatabaseConnection, task_id: Uuid, error: String) -> Result<()> { + /// Mark a task failed (or reschedule for retry if attempts remain). + /// + /// `result_data`, when provided, is persisted on the row. The worker uses + /// it to carry recorded entity events (`emitted_events`) so the web + /// server's `TaskListener` can replay them to SSE subscribers even for + /// *failed* tasks in distributed deployments — otherwise events emitted + /// during a failing task (e.g. a `release_source_polled` from a poll that + /// errored) would be lost and the UI wouldn't refresh until a manual + /// reload. + pub async fn mark_failed( + db: &DatabaseConnection, + task_id: Uuid, + error: String, + result_data: Option, + ) -> Result<()> { let task = Tasks::find_by_id(task_id) .one(db) .await @@ -856,6 +870,10 @@ impl TaskRepository { let mut active: tasks::ActiveModel = task.clone().into(); + if let Some(data) = result_data { + active.result = Set(Some(data)); + } + // Check if we should retry if task.attempts < task.max_attempts { // Retry - back to pending with exponential backoff @@ -1306,3 +1324,53 @@ impl TaskRepository { Ok(recovered) } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_helpers::setup_test_db; + + /// `mark_failed` persists `result_data` so the web `TaskListener` can replay + /// recorded events for failed tasks (distributed mode). Without this, events + /// emitted during a failing task would be lost. + #[tokio::test] + async fn mark_failed_persists_result_data() { + let db = setup_test_db().await; + let task_id = TaskRepository::enqueue(&db, TaskType::CleanupPluginData, None) + .await + .unwrap(); + + let result_data = serde_json::json!({ + "emitted_events": [{ "marker": "release_source_polled" }] + }); + TaskRepository::mark_failed(&db, task_id, "boom".to_string(), Some(result_data.clone())) + .await + .unwrap(); + + let task = TaskRepository::get_by_id(&db, task_id) + .await + .unwrap() + .expect("task exists"); + assert_eq!(task.last_error.as_deref(), Some("boom")); + assert_eq!(task.result, Some(result_data)); + } + + /// Passing `None` leaves the result column untouched. + #[tokio::test] + async fn mark_failed_without_result_data_leaves_result_null() { + let db = setup_test_db().await; + let task_id = TaskRepository::enqueue(&db, TaskType::CleanupPluginData, None) + .await + .unwrap(); + + TaskRepository::mark_failed(&db, task_id, "boom".to_string(), None) + .await + .unwrap(); + + let task = TaskRepository::get_by_id(&db, task_id) + .await + .unwrap() + .expect("task exists"); + assert!(task.result.is_none()); + } +} diff --git a/crates/codex-services/src/task_listener.rs b/crates/codex-services/src/task_listener.rs index 3c6ee424..c70a7557 100644 --- a/crates/codex-services/src/task_listener.rs +++ b/crates/codex-services/src/task_listener.rs @@ -178,9 +178,12 @@ impl TaskListener { } } - // Replay any recorded entity events from the task result - // This bridges events from worker processes to the web server - if status == TaskStatus::Completed + // Replay any recorded entity events from the task result. This bridges + // events from worker processes to the web server. Failed tasks can + // carry events too (e.g. a `release_source_polled` from a poll that + // errored), so replay on both terminal states — otherwise the UI + // wouldn't refresh after a failed poll until a manual reload. + if matches!(status, TaskStatus::Completed | TaskStatus::Failed) && let Err(e) = self.replay_recorded_events(task_id).await { warn!( diff --git a/crates/codex-tasks/src/worker.rs b/crates/codex-tasks/src/worker.rs index c2dd679a..43000b1a 100644 --- a/crates/codex-tasks/src/worker.rs +++ b/crates/codex-tasks/src/worker.rs @@ -738,10 +738,10 @@ impl TaskWorker { .message .unwrap_or_else(|| "task reported failure".to_string()) ); - self.fail_task(&task, err, started_at).await?; + self.fail_task(&task, err, started_at, events).await?; } Err(e) => { - self.fail_task(&task, e, started_at).await?; + self.fail_task(&task, e, started_at, events).await?; } } @@ -794,10 +794,12 @@ impl TaskWorker { .message .unwrap_or_else(|| "task reported failure".to_string()) ); - self.fail_task(&task, err, started_at).await?; + // Single-process mode: events flow live to the shared + // broadcaster, so there are none to record/replay. + self.fail_task(&task, err, started_at, None).await?; } Err(e) => { - self.fail_task(&task, e, started_at).await?; + self.fail_task(&task, e, started_at, None).await?; } } @@ -901,6 +903,7 @@ impl TaskWorker { task: &codex_db::entities::tasks::Model, error: anyhow::Error, started_at: chrono::DateTime, + recorded_events: Option>, ) -> Result<()> { let completed_at = Utc::now(); let error_string = error.to_string(); @@ -963,9 +966,16 @@ impl TaskWorker { return Ok(()); } - // Not rate-limited: handle as normal failure + // Not rate-limited: handle as normal failure. Carry any recorded + // events through so the web `TaskListener` can replay them for failed + // tasks too (distributed mode) — e.g. a `release_source_polled` from a + // poll that errored, so the Release tracking UI updates without a + // manual reload. error!("Task {} failed: {}", task.id, error_string); - TaskRepository::mark_failed(&self.db, task.id, error_string.clone()).await?; + let result_data = recorded_events + .filter(|events| !events.is_empty()) + .map(|events| json!({ "emitted_events": events })); + TaskRepository::mark_failed(&self.db, task.id, error_string.clone(), result_data).await?; // Record metrics if let Some(ref metrics_service) = self.task_metrics_service { From 31720490f2b5ca27a4024d361f5130f9a218e340 Mon Sep 17 00:00:00 2001 From: Sylvain Cau Date: Mon, 8 Jun 2026 18:57:45 -0700 Subject: [PATCH 08/10] feat(plugins): match tsundoku series by weighted external-id voting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Matching on the first shared provider id was unsafe: non-MangaBaka providers (MAL, MangaUpdates, …) sometimes share or merge ids across distinct series, so a single matching low-trust id could attribute one series' coverage to another. Match by weighted voting instead. For each provider both the feed item and a candidate series carry, an agreeing id adds its weight (MangaBaka 3, AniList 2, rest 1) and a disagreeing one subtracts it; a series matches only on a net-positive score, so a trusted disagreement (different MangaBaka ids) overrides a sloppy agreement (a shared MAL id). Confidence is derived from the score, and a post-walk cross-item pass keeps at most one feed entry per Codex series — best score wins, and genuinely ambiguous ties between different series are skipped rather than mis-attributed. Candidate confidence/reason now reflect the vote instead of a hardcoded 1.0. Tests cover the voting, conflict rejection, ambiguity, and cross-item resolution. --- plugins/release-tsundoku/README.md | 26 ++- .../release-tsundoku/src/candidate.test.ts | 9 +- plugins/release-tsundoku/src/candidate.ts | 9 +- plugins/release-tsundoku/src/index.test.ts | 59 +++++- plugins/release-tsundoku/src/index.ts | 90 ++++++--- plugins/release-tsundoku/src/matcher.test.ts | 122 +++++++----- plugins/release-tsundoku/src/matcher.ts | 177 +++++++++++++----- 7 files changed, 359 insertions(+), 133 deletions(-) diff --git a/plugins/release-tsundoku/README.md b/plugins/release-tsundoku/README.md index 88c9e988..9c86175c 100644 --- a/plugins/release-tsundoku/README.md +++ b/plugins/release-tsundoku/README.md @@ -6,10 +6,15 @@ incremental series feed. **Notification-only** — Codex does not download anyth ## Features -- **Exact external-ID matching, no fuzzy logic.** Series are matched to your - Codex catalog by provider IDs (MangaBaka, AniList, MAL, MangaUpdates, Kitsu, - Shikimori, Anime-Planet, Anime News Network), so announcements always land on - the right series with full confidence. +- **External-ID matching by weighted voting, no title fuzzing.** Series are + matched to your Codex catalog by provider IDs (MangaBaka, AniList, MAL, + MangaUpdates, Kitsu, Shikimori, Anime-Planet, Anime News Network). Because + some providers occasionally share/merge IDs across distinct series, each + shared ID *votes*: an agreeing ID adds its weight (MangaBaka 3, AniList 2, + rest 1), a disagreeing one subtracts it, and a series matches only when + agreement wins. So a trusted disagreement (different MangaBaka IDs) overrides + a sloppy agreement (a shared MAL ID), and genuinely ambiguous ties are + skipped rather than mis-attributed. - **Incremental, cursor-based.** Walks Tsundoku's keyset-paginated `/api/v1/series/feed`, persisting its position in the plugin's (system-scoped) KV store so each poll only processes activity since the last @@ -68,10 +73,13 @@ On each poll the plugin: (`feed_cursor`). 2. Builds a reverse index (`provider:id → Codex series`) from your tracked series via the host's `releases/list_tracked`. -3. Walks the feed from the cursor. Each item is matched against the index by - external ID; on a hit it records a release candidate (confidence `1.0`) whose - `volumes`/`chapters` mirror the item's coverage. The cursor is persisted after - each processed page, so an interrupted walk resumes cleanly. +3. Walks the feed from the cursor. Each item is matched to a tracked series by + weighted external-ID voting (see Features); on a confident match it records a + release candidate whose `volumes`/`chapters` mirror the item's coverage and + whose confidence reflects the vote. When several feed entries map to the same + Codex series, only the best-scoring one is recorded (ties are skipped). The + cursor is persisted after each processed page, so an interrupted walk resumes + cleanly. 4. Reports counters back to the host; the host applies its own threshold, auto-ignore (for coverage you already own), and dedup. @@ -125,7 +133,7 @@ src/ ├── index.ts # Plugin lifecycle, config, source registration, poll loop ├── manifest.ts # Capability + config schema + supported providers ├── fetcher.ts # Feed wire types + paginated fetchFeedPage -├── matcher.ts # Reverse index + exact external-ID matching +├── matcher.ts # Weighted-vote external-ID matching + cross-item resolution └── candidate.ts # Feed item → ReleaseCandidate mapping ``` diff --git a/plugins/release-tsundoku/src/candidate.test.ts b/plugins/release-tsundoku/src/candidate.test.ts index b16b247b..019cc039 100644 --- a/plugins/release-tsundoku/src/candidate.test.ts +++ b/plugins/release-tsundoku/src/candidate.test.ts @@ -23,8 +23,9 @@ function feedItem(overrides: Partial = {}): FeedItem { const match: MatchResult = { codexSeriesId: "uuid-a", - provider: "mangabaka", - externalId: "9741", + score: 4, + confidence: 1.0, + agreeingProviders: ["mangabaka", "mal"], }; const opts = { @@ -95,12 +96,12 @@ describe("externalReleaseId", () => { // ----------------------------------------------------------------------------- describe("feedItemToCandidate", () => { - it("builds an exact-match candidate (confidence 1.0)", () => { + it("carries the vote confidence and agreeing providers into seriesMatch", () => { const c = feedItemToCandidate(feedItem(), match, opts); expect(c.seriesMatch).toEqual({ codexSeriesId: "uuid-a", confidence: 1.0, - reason: "tsundoku:mangabaka:9741", + reason: "tsundoku:vote:mangabaka+mal", }); expect(c.externalReleaseId).toBe("tsundoku:87:v16:c-"); expect(c.language).toBe("en"); diff --git a/plugins/release-tsundoku/src/candidate.ts b/plugins/release-tsundoku/src/candidate.ts index c2399647..078ecbf9 100644 --- a/plugins/release-tsundoku/src/candidate.ts +++ b/plugins/release-tsundoku/src/candidate.ts @@ -51,8 +51,9 @@ export function externalReleaseId(item: FeedItem): string { } /** - * Build a `ReleaseCandidate` for a matched feed item. Confidence is 1.0 — the - * match is an exact external-ID hit, never fuzzy. + * Build a `ReleaseCandidate` for a matched feed item. Confidence and `reason` + * come from the weighted-vote match: confidence reflects the net agreement + * score, and `reason` lists the providers that agreed (highest-weight first). */ export function feedItemToCandidate( item: FeedItem, @@ -63,8 +64,8 @@ export function feedItemToCandidate( return { seriesMatch: { codexSeriesId: match.codexSeriesId, - confidence: 1.0, - reason: `tsundoku:${match.provider}:${match.externalId}`, + confidence: match.confidence, + reason: `tsundoku:vote:${match.agreeingProviders.join("+")}`, }, externalReleaseId: externalReleaseId(item), volumes: toSpans(item.volumeCoverage), diff --git a/plugins/release-tsundoku/src/index.test.ts b/plugins/release-tsundoku/src/index.test.ts index e70d2250..d3113f79 100644 --- a/plugins/release-tsundoku/src/index.test.ts +++ b/plugins/release-tsundoku/src/index.test.ts @@ -262,7 +262,10 @@ const pollDeps = (fetchImpl: typeof fetch, storage: PluginStorage) => ({ describe("poll", () => { it("walks pages, matches by external id, records, and persists the cursor per page", async () => { const { rpc } = makePollRpc({ - tracked: [{ seriesId: "uuid-a", externalIds: { mangabaka: "9741" } }], + tracked: [ + { seriesId: "uuid-a", externalIds: { mangabaka: "9741" } }, + { seriesId: "uuid-b", externalIds: { mangabaka: "5555" } }, + ], }); const { storage, sets } = makeStorage(); const { fetchImpl } = makeFetchSequence([ @@ -271,14 +274,14 @@ describe("poll", () => { hasMore: true, nextCursor: "c1", }, - { items: [item(87, "mangabaka", "9741", 17)], hasMore: false, nextCursor: "c2" }, + { items: [item(88, "mangabaka", "5555", 3)], hasMore: false, nextCursor: "c2" }, ]); const res = await poll({ sourceId: "src-1" }, rpc, pollDeps(fetchImpl, storage)); expect(res).toMatchObject({ - parsed: 3, - matched: 2, + parsed: 3, // 87, 99, 88 + matched: 2, // 87 -> uuid-a, 88 -> uuid-b; 99 unmatched recorded: 2, deduped: 0, upstreamStatus: 200, @@ -289,19 +292,61 @@ describe("poll", () => { it("counts host dedup separately from inserts", async () => { const { rpc } = makePollRpc({ tracked: [{ seriesId: "uuid-a", externalIds: { mangabaka: "9741" } }], - onRecord: (n) => ({ ledgerId: `l${n}`, deduped: n > 1 }), + onRecord: () => ({ ledgerId: "l1", deduped: true }), + }); + const { storage } = makeStorage(); + const { fetchImpl } = makeFetchSequence([ + { items: [item(87, "mangabaka", "9741", 16)], hasMore: false, nextCursor: "c1" }, + ]); + + const res = await poll({ sourceId: "src-1" }, rpc, pollDeps(fetchImpl, storage)); + expect(res).toMatchObject({ matched: 1, recorded: 0, deduped: 1 }); + }); + + it("resolves a collision to the highest-scoring feed entry", async () => { + // Two different Tsundoku series both map to uuid-a: #87 via mangabaka + // (score 3) and #88 via mal (score 1). The mangabaka match wins; the mal + // one is superseded, so only one record call is made — for series #87. + const { rpc, calls } = makePollRpc({ + tracked: [{ seriesId: "uuid-a", externalIds: { mangabaka: "9741", mal: "555" } }], + }); + const { storage } = makeStorage(); + const { fetchImpl } = makeFetchSequence([ + { + items: [item(87, "mangabaka", "9741", 16), item(88, "mal", "555", 9)], + hasMore: false, + nextCursor: "c1", + }, + ]); + + const res = await poll({ sourceId: "src-1" }, rpc, pollDeps(fetchImpl, storage)); + expect(res).toMatchObject({ parsed: 2, matched: 1, recorded: 1 }); + + const recordCalls = calls.filter((c) => c.method === "releases/record"); + expect(recordCalls).toHaveLength(1); + const candidate = (recordCalls[0].params as { candidate: { externalReleaseId: string } }) + .candidate; + expect(candidate.externalReleaseId).toContain("tsundoku:87:"); + }); + + it("skips an ambiguous collision (different series tie at the same score)", async () => { + // Both #87 and #88 match uuid-a only via the same low-trust mal id (score + // 1 each) — genuinely ambiguous, so neither is recorded. + const { rpc, calls } = makePollRpc({ + tracked: [{ seriesId: "uuid-a", externalIds: { mal: "555" } }], }); const { storage } = makeStorage(); const { fetchImpl } = makeFetchSequence([ { - items: [item(87, "mangabaka", "9741", 16), item(87, "mangabaka", "9741", 17)], + items: [item(87, "mal", "555", 16), item(88, "mal", "555", 9)], hasMore: false, nextCursor: "c1", }, ]); const res = await poll({ sourceId: "src-1" }, rpc, pollDeps(fetchImpl, storage)); - expect(res).toMatchObject({ matched: 2, recorded: 1, deduped: 1 }); + expect(res).toMatchObject({ parsed: 2, matched: 0, recorded: 0 }); + expect(calls.some((c) => c.method === "releases/record")).toBe(false); }); it("skips items with no tracked match (no record calls)", async () => { diff --git a/plugins/release-tsundoku/src/index.ts b/plugins/release-tsundoku/src/index.ts index 2b33d3c0..aba60b70 100644 --- a/plugins/release-tsundoku/src/index.ts +++ b/plugins/release-tsundoku/src/index.ts @@ -37,9 +37,9 @@ import { type TrackedSeriesEntry, } from "@ashdev/codex-plugin-sdk"; import { feedItemToCandidate } from "./candidate.js"; -import { fetchFeedPage } from "./fetcher.js"; +import { type FeedItem, fetchFeedPage } from "./fetcher.js"; import { manifest } from "./manifest.js"; -import { buildIndex, matchItem } from "./matcher.js"; +import { buildMatchContext, type MatchResult, matchItem } from "./matcher.js"; const logger = createLogger({ name: manifest.name, level: "info" }); @@ -291,27 +291,28 @@ export async function poll( ): Promise { const sourceId = params.sourceId; - // 1. Build the reverse index from the user's tracked series. The feed spans + // 1. Build the match context from the user's tracked series. The feed spans // the whole Tsundoku catalog, so this is what scopes it to the user. const trackedEntries: TrackedSeriesEntry[] = []; for await (const entry of iterateTrackedSeries(rpc, sourceId)) { trackedEntries.push(entry); } - const index = buildIndex(trackedEntries); - if (index.size === 0) { + const ctx = buildMatchContext(trackedEntries); + if (ctx.series.size === 0) { logger.info( `poll: no tracked series carry a Tsundoku-known external ID (source=${sourceId}); nothing to match`, ); } - // 2. Walk the feed from the stored cursor. + // 2. Walk the feed from the stored cursor, collecting per-item matches. We + // resolve them after the walk (cross-item) rather than recording inline, + // so that when several feed entries map to the same Codex series we keep + // only the best one instead of polluting the ledger. let cursor = await loadCursor(deps.storage); let parsed = 0; - let matched = 0; - let recorded = 0; - let deduped = 0; let worstStatus = 200; let pagesFetched = 0; + const hits: Array<{ item: FeedItem; match: MatchResult }> = []; while (true) { const result = await fetchFeedPage(deps.baseUrl, cursor, deps.pageLimit, { @@ -338,19 +339,9 @@ export async function poll( const page = result.data; for (const item of page.items) { parsed++; - const match = matchItem(item, index); - if (!match) continue; - matched++; - const candidate = feedItemToCandidate(item, match, { - baseUrl: deps.baseUrl, - language: deps.language, - }); - const outcome = await recordCandidate(rpc, sourceId, candidate); - if (!outcome) continue; - if (outcome.deduped) { - deduped++; - } else { - recorded++; + const match = matchItem(item, ctx); + if (match) { + hits.push({ item, match }); } } @@ -373,8 +364,61 @@ export async function poll( if (page.items.length === 0) break; } + // 3. Cross-item resolution: a Codex series should map to at most one feed + // entry. Group hits by Codex series; keep the highest-scoring one. If the + // top two tie (e.g. two entries match only via the same low-trust ID), + // it's genuinely ambiguous — skip both rather than record the wrong one. + const byCodex = new Map>(); + for (const hit of hits) { + const arr = byCodex.get(hit.match.codexSeriesId); + if (arr) { + arr.push(hit); + } else { + byCodex.set(hit.match.codexSeriesId, [hit]); + } + } + + let matched = 0; + let recorded = 0; + let deduped = 0; + let ambiguous = 0; + let superseded = 0; + + for (const [codexSeriesId, group] of byCodex) { + // Best score first; for ties prefer the most recently updated entry (newest + // coverage). The same Tsundoku series appearing twice in one walk is not a + // conflict — only *different* series tying is. + group.sort((a, b) => b.match.score - a.match.score || b.item.updatedAt - a.item.updatedAt); + if ( + group.length > 1 && + group[0].match.score === group[1].match.score && + group[0].item.seriesId !== group[1].item.seriesId + ) { + ambiguous += group.length; + logger.warn( + `ambiguous: feed entries from different Tsundoku series match Codex series ${codexSeriesId} at score ${group[0].match.score}; skipping`, + ); + continue; + } + superseded += group.length - 1; + + const { item, match } = group[0]; + matched++; + const candidate = feedItemToCandidate(item, match, { + baseUrl: deps.baseUrl, + language: deps.language, + }); + const outcome = await recordCandidate(rpc, sourceId, candidate); + if (!outcome) continue; + if (outcome.deduped) { + deduped++; + } else { + recorded++; + } + } + logger.info( - `poll complete: source=${sourceId} tracked=${trackedEntries.length} parsed=${parsed} matched=${matched} recorded=${recorded} deduped=${deduped} worst_status=${worstStatus}`, + `poll complete: source=${sourceId} tracked=${trackedEntries.length} parsed=${parsed} matched=${matched} recorded=${recorded} deduped=${deduped} ambiguous=${ambiguous} superseded=${superseded} worst_status=${worstStatus}`, ); return { diff --git a/plugins/release-tsundoku/src/matcher.test.ts b/plugins/release-tsundoku/src/matcher.test.ts index 7af2277f..60bedf96 100644 --- a/plugins/release-tsundoku/src/matcher.test.ts +++ b/plugins/release-tsundoku/src/matcher.test.ts @@ -1,7 +1,7 @@ import type { TrackedSeriesEntry } from "@ashdev/codex-plugin-sdk"; import { describe, expect, it } from "vitest"; import type { FeedExternalId, FeedItem } from "./fetcher.js"; -import { buildIndex, matchItem } from "./matcher.js"; +import { buildMatchContext, externalIdFilter, matchItem } from "./matcher.js"; // ----------------------------------------------------------------------------- // Helpers @@ -29,73 +29,111 @@ function ext(provider: string, externalId: string): FeedExternalId { } // ----------------------------------------------------------------------------- -// buildIndex +// buildMatchContext / externalIdFilter // ----------------------------------------------------------------------------- -describe("buildIndex", () => { - it("indexes each provider/id pair to its series", () => { - const index = buildIndex([ +describe("buildMatchContext", () => { + it("indexes provider/id pairs and keeps each series' full id map", () => { + const ctx = buildMatchContext([ tracked("uuid-a", { mangabaka: "9741", anilist: "122180" }), tracked("uuid-b", { mal: "128555" }), ]); - expect(index.get("mangabaka:9741")).toBe("uuid-a"); - expect(index.get("anilist:122180")).toBe("uuid-a"); - expect(index.get("mal:128555")).toBe("uuid-b"); - expect(index.size).toBe(3); + expect(ctx.byKey.get("mangabaka:9741")).toEqual(["uuid-a"]); + expect(ctx.byKey.get("mal:128555")).toEqual(["uuid-b"]); + expect(ctx.series.get("uuid-a")?.get("anilist")).toBe("122180"); }); - it("skips entries without external IDs", () => { - const index = buildIndex([tracked("uuid-a"), tracked("uuid-b", {})]); - expect(index.size).toBe(0); + it("skips entries without external ids and ignores empty values", () => { + const ctx = buildMatchContext([tracked("uuid-a"), tracked("uuid-b", { mangabaka: "" })]); + expect(ctx.series.size).toBe(0); + expect(ctx.byKey.size).toBe(0); }); +}); - it("ignores empty external-id values", () => { - const index = buildIndex([tracked("uuid-a", { mangabaka: "" })]); - expect(index.size).toBe(0); +describe("externalIdFilter", () => { + it("returns every provider:id key (the POST feed filter set)", () => { + const ctx = buildMatchContext([tracked("uuid-a", { mangabaka: "9741", mal: "5" })]); + expect(new Set(externalIdFilter(ctx))).toEqual(new Set(["mangabaka:9741", "mal:5"])); }); }); // ----------------------------------------------------------------------------- -// matchItem +// matchItem — weighted voting // ----------------------------------------------------------------------------- describe("matchItem", () => { - it("matches on a single provider id", () => { - const index = buildIndex([tracked("uuid-a", { mangabaka: "9741" })]); - const result = matchItem(feedItem([ext("mangabaka", "9741")]), index); - expect(result).toEqual({ codexSeriesId: "uuid-a", provider: "mangabaka", externalId: "9741" }); + it("matches when a single shared id agrees (no mangabaka required)", () => { + const ctx = buildMatchContext([tracked("uuid-a", { mal: "128555" })]); + const res = matchItem(feedItem([ext("mal", "128555")]), ctx); + expect(res?.codexSeriesId).toBe("uuid-a"); + expect(res?.agreeingProviders).toEqual(["mal"]); + expect(res?.score).toBe(1); }); - it("returns null when no provider id is tracked", () => { - const index = buildIndex([tracked("uuid-a", { mangabaka: "9741" })]); - expect(matchItem(feedItem([ext("mangabaka", "0000")]), index)).toBeNull(); - expect(matchItem(feedItem([ext("anilist", "9741")]), index)).toBeNull(); + it("returns null when nothing is shared", () => { + const ctx = buildMatchContext([tracked("uuid-a", { mangabaka: "9741" })]); + expect(matchItem(feedItem([ext("mangabaka", "0000")]), ctx)).toBeNull(); + expect(matchItem(feedItem([]), ctx)).toBeNull(); }); - it("prefers the highest-priority provider when several would match", () => { - // The same series is tracked under both mangabaka and mal; mangabaka leads - // the priority order, so it should win regardless of array order on the item. - const index = buildIndex([tracked("uuid-a", { mangabaka: "9741", mal: "128555" })]); - const result = matchItem(feedItem([ext("mal", "128555"), ext("mangabaka", "9741")]), index); - expect(result?.provider).toBe("mangabaka"); + it("lets a trusted disagreement veto a sloppy agreement", () => { + // ABC has mangabaka:X + mal:Y. A different series shares mal:Y but its + // mangabaka differs — the mangabaka conflict (weight 3) outvotes the mal + // agreement (weight 1), so it must NOT match ABC. + const ctx = buildMatchContext([tracked("ABC", { mangabaka: "X", mal: "Y" })]); + const res = matchItem(feedItem([ext("mangabaka", "W"), ext("mal", "Y")]), ctx); + expect(res).toBeNull(); }); - it("falls through to a lower-priority provider when the leader misses", () => { - const index = buildIndex([tracked("uuid-a", { mal: "128555" })]); - const result = matchItem( - feedItem([ext("mangabaka", "not-tracked"), ext("mal", "128555")]), - index, + it("accepts a true match that disagrees on one low-trust id", () => { + // Same series, but its MAL id was remapped upstream: mangabaka+anilist + // agree (3+2), mal disagrees (1) → net +4 → still a match. + const ctx = buildMatchContext([tracked("ABC", { mangabaka: "X", anilist: "A", mal: "Y" })]); + const res = matchItem( + feedItem([ext("mangabaka", "X"), ext("anilist", "A"), ext("mal", "Z")]), + ctx, ); - expect(result).toEqual({ codexSeriesId: "uuid-a", provider: "mal", externalId: "128555" }); + expect(res?.codexSeriesId).toBe("ABC"); + expect(res?.score).toBe(4); // 3 + 2 - 1 + expect(res?.agreeingProviders).toEqual(["mangabaka", "anilist"]); + }); + + it("rejects a net-zero tally (equal agree/disagree weight)", () => { + const ctx = buildMatchContext([tracked("ABC", { mangabaka: "X", anilist: "A" })]); + // mangabaka agrees (3), anilist... make weights cancel: agree mal(1) vs disagree mangabaka(3) handled above; + // here anilist disagrees (2) and mal agrees — but ABC has no mal, so only anilist shared (disagree) → score<0. + const res = matchItem(feedItem([ext("anilist", "ZZ")]), ctx); + expect(res).toBeNull(); }); - it("ignores providers outside the supported set", () => { - const index = new Map([["someunknownprovider:1", "uuid-a"]]); - expect(matchItem(feedItem([ext("someunknownprovider", "1")]), index)).toBeNull(); + it("weights mangabaka above anilist when scoring confidence", () => { + const ctxMb = buildMatchContext([tracked("a", { mangabaka: "1" })]); + const ctxAl = buildMatchContext([tracked("b", { anilist: "1" })]); + const mb = matchItem(feedItem([ext("mangabaka", "1")]), ctxMb); + const al = matchItem(feedItem([ext("anilist", "1")]), ctxAl); + expect(mb?.confidence).toBe(1.0); // 0.7 + 0.1*3 + expect(al?.confidence).toBeCloseTo(0.9); // 0.7 + 0.1*2 + expect((mb?.score ?? 0) > (al?.score ?? 0)).toBe(true); }); - it("returns null for an item with no external ids", () => { - const index = buildIndex([tracked("uuid-a", { mangabaka: "9741" })]); - expect(matchItem(feedItem([]), index)).toBeNull(); + it("returns null when two series match the item equally well (ambiguous)", () => { + // Two tracked series each share only mal:Y with the item (no higher-trust + // discriminator) → equal score → can't safely pick. + const ctx = buildMatchContext([ + tracked("uuid-a", { mal: "Y" }), + tracked("uuid-b", { mal: "Y" }), + ]); + expect(matchItem(feedItem([ext("mal", "Y")]), ctx)).toBeNull(); + }); + + it("picks the higher-scoring series when candidates differ", () => { + // The item shares mangabaka with A (score 3) and mal with B (score 1). + const ctx = buildMatchContext([ + tracked("uuid-a", { mangabaka: "X" }), + tracked("uuid-b", { mal: "Y" }), + ]); + const res = matchItem(feedItem([ext("mangabaka", "X"), ext("mal", "Y")]), ctx); + expect(res?.codexSeriesId).toBe("uuid-a"); + expect(res?.score).toBe(3); }); }); diff --git a/plugins/release-tsundoku/src/matcher.ts b/plugins/release-tsundoku/src/matcher.ts index 91e7242c..203c45c8 100644 --- a/plugins/release-tsundoku/src/matcher.ts +++ b/plugins/release-tsundoku/src/matcher.ts @@ -1,82 +1,171 @@ /** - * Exact external-ID matching between the Tsundoku feed and Codex's tracked - * series. + * Match Tsundoku feed items to tracked Codex series by external ID — using + * *weighted voting* across providers rather than trusting a single ID. * - * Codex's `releases/record` path keys on a `codexSeriesId` (UUID), not on - * external IDs — so matching happens here, plugin-side, with zero fuzzy - * logic. The host returns each tracked series' provider IDs via - * `releases/list_tracked` (scoped by the manifest's `requiresExternalIds`, - * prefix-stripped to bare provider names). We index those into - * `"provider:id" -> codexSeriesId`, then resolve each feed item by looking - * up its own provider IDs in priority order. The first provider that hits - * wins; the match is exact, so the candidate's confidence is always 1.0. + * Why voting: provider IDs vary in quality. MangaBaka is an aggregation hub + * with reliably 1:1 IDs; others (MAL, MangaUpdates, …) occasionally share or + * merge IDs across distinct series, so a lone matching ID can be a false + * positive. So for each candidate we tally the providers the feed item and the + * Codex series *both* carry: a shared ID that agrees adds its weight, one that + * disagrees subtracts it. A series matches only when agreement outweighs + * disagreement — a trusted disagreement (e.g. different MangaBaka IDs) vetoes a + * sloppy agreement (e.g. a shared MAL ID). + * + * Codex's `releases/record` keys on a `codexSeriesId`, so matching is done + * here, plugin-side, over the full ID sets both the host and the feed expose. */ import type { TrackedSeriesEntry } from "@ashdev/codex-plugin-sdk"; import type { FeedItem } from "./fetcher.js"; -import { TSUNDOKU_EXTERNAL_ID_SOURCES } from "./manifest.js"; + +/** + * Vote weight per provider — higher means more trusted as a match signal. + * MangaBaka leads (its IDs are reliably 1:1), AniList next; the rest default + * to 1. Tune here if real data shows a source is noisier than assumed. + */ +export const PROVIDER_WEIGHTS: Record = { + mangabaka: 3, + anilist: 2, +}; +const DEFAULT_WEIGHT = 1; + +function weightOf(provider: string): number { + return PROVIDER_WEIGHTS[provider] ?? DEFAULT_WEIGHT; +} /** Result of resolving a feed item to a tracked Codex series. */ export interface MatchResult { /** The Codex series UUID the candidate should be recorded against. */ codexSeriesId: string; - /** The provider whose ID produced the match (for the candidate `reason`). */ - provider: string; - /** The external ID value that matched. */ - externalId: string; + /** Net vote score (agreeing weights minus disagreeing). Always `> 0`. */ + score: number; + /** Host confidence in `[0.8, 1.0]`, derived from the score. */ + confidence: number; + /** Providers that agreed, highest-weight first — used for the candidate `reason`. */ + agreeingProviders: string[]; +} + +/** Pre-computed lookup over the tracked series for matching. */ +export interface MatchContext { + /** `provider:id` -> codex series ids carrying it (usually one). */ + byKey: Map; + /** codex series id -> its `provider -> id` map (for the conflict tally). */ + series: Map>; } -/** Compose the index key for a `(provider, externalId)` pair. */ +/** Compose the lookup key for a `(provider, externalId)` pair. */ function indexKey(provider: string, externalId: string): string { return `${provider}:${externalId}`; } /** - * Build a reverse index `"provider:id" -> codexSeriesId` from the host's - * tracked-series rows. Entries without external IDs contribute nothing. - * - * If two tracked series somehow share the same `(provider, id)` (shouldn't - * happen — provider IDs are unique per series), the later entry wins. That's - * an arbitrary-but-deterministic tie-break for a degenerate input. + * Build the match context from the host's tracked-series rows. Entries without + * external IDs contribute nothing. */ -export function buildIndex(entries: TrackedSeriesEntry[]): Map { - const index = new Map(); +export function buildMatchContext(entries: TrackedSeriesEntry[]): MatchContext { + const byKey = new Map(); + const series = new Map>(); + for (const entry of entries) { const ids = entry.externalIds; if (!ids) continue; + const map = new Map(); for (const [provider, externalId] of Object.entries(ids)) { if (!externalId) continue; - index.set(indexKey(provider, externalId), entry.seriesId); + map.set(provider, externalId); + const key = indexKey(provider, externalId); + const arr = byKey.get(key); + if (arr) { + arr.push(entry.seriesId); + } else { + byKey.set(key, [entry.seriesId]); + } + } + if (map.size > 0) { + series.set(entry.seriesId, map); } } - return index; + + return { byKey, series }; } /** - * Resolve a feed item against the reverse index. Providers are tried in the - * manifest's declared priority order (`TSUNDOKU_EXTERNAL_ID_SOURCES`, most - * canonical first) so the `reason` is stable when an item carries several - * matchable IDs. Returns `null` when no provider ID hits the index — the - * common case, since the feed spans the whole Tsundoku catalog and the user - * only tracks a slice of it. + * The full set of `provider:id` keys across all tracked series. This is the + * filter set posted to Tsundoku's `POST /series/feed` so the feed is narrowed + * to the consumer's catalog. */ -export function matchItem(item: FeedItem, index: Map): MatchResult | null { - // Collapse the item's external-ID array into a provider -> id lookup so the - // priority sweep below is O(providers) rather than O(providers * ids). - const byProvider = new Map(); +export function externalIdFilter(ctx: MatchContext): string[] { + return [...ctx.byKey.keys()]; +} + +/** Map a net score to a host confidence in `[0.8, 1.0]` (gate is 0.7). */ +function confidenceForScore(score: number): number { + return Math.min(1, Math.max(0.7, 0.7 + 0.1 * score)); +} + +/** + * Resolve a feed item to the single best-matching tracked series, or `null` + * when nothing matches net-positive or the top two candidates tie (ambiguous — + * the item's IDs point at two series equally well, so we can't safely pick). + */ +export function matchItem(item: FeedItem, ctx: MatchContext): MatchResult | null { + const itemMap = new Map(); for (const ext of item.externalIds) { if (ext.externalId) { - byProvider.set(ext.provider, ext.externalId); + itemMap.set(ext.provider, ext.externalId); } } - for (const provider of TSUNDOKU_EXTERNAL_ID_SOURCES) { - const externalId = byProvider.get(provider); - if (externalId === undefined) continue; - const codexSeriesId = index.get(indexKey(provider, externalId)); - if (codexSeriesId !== undefined) { - return { codexSeriesId, provider, externalId }; + // Candidate Codex series: any that shares at least one id with the item. + const candidates = new Set(); + for (const [provider, id] of itemMap) { + const arr = ctx.byKey.get(indexKey(provider, id)); + if (arr) { + for (const sid of arr) candidates.add(sid); } } - return null; + if (candidates.size === 0) return null; + + let best: MatchResult | null = null; + let tiedAtBest = false; + + for (const cid of candidates) { + const cSeries = ctx.series.get(cid); + if (!cSeries) continue; + + let agree = 0; + let disagree = 0; + const agreeing: Array<{ provider: string; weight: number }> = []; + for (const [provider, idVal] of itemMap) { + const cVal = cSeries.get(provider); + if (cVal === undefined) continue; // provider not shared by both + const w = weightOf(provider); + if (cVal === idVal) { + agree += w; + agreeing.push({ provider, weight: w }); + } else { + disagree += w; + } + } + + const score = agree - disagree; + if (score <= 0) continue; // disagreement outweighs (or ties) agreement + + if (!best || score > best.score) { + agreeing.sort((a, b) => b.weight - a.weight || a.provider.localeCompare(b.provider)); + best = { + codexSeriesId: cid, + score, + confidence: confidenceForScore(score), + agreeingProviders: agreeing.map((a) => a.provider), + }; + tiedAtBest = false; + } else if (score === best.score) { + tiedAtBest = true; + } + } + + // No net-positive candidate, or two series matched equally well → don't guess. + if (!best || tiedAtBest) return null; + return best; } From 431a089426a8d87e191ab606c74177f8908ebc7a Mon Sep 17 00:00:00 2001 From: Sylvain Cau Date: Mon, 8 Jun 2026 19:36:17 -0700 Subject: [PATCH 09/10] feat(plugins): fetch tsundoku via filtered POST, drop the cursor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the catalog-wide GET + persisted cursor with Tsundoku's filtered POST /series/feed, posting the tracked series' provider:externalId set so the response contains only those series. There's no persisted cursor: each poll re-walks the tracked set's current coverage and relies on host-side dedup to suppress unchanged releases. This makes newly tracked series backfill automatically and untracked ones drop out, with no cursor bookkeeping — closing the new-series backfill gap by construction. When the tracked set is empty the poll skips entirely (an empty filter would mean "whole catalog" upstream). The system KV store and its cursor helpers are no longer needed and are removed from the plugin. Also fix a provider-name mismatch: Codex stores some sources under different names than the feed uses (e.g. myanimelist vs mal, animeplanet vs anime_planet, animenewsnetwork vs anime_news_network). Add a Codex → Tsundoku name map; requiresExternalIds now declares the Codex names the host filters on, and the matcher translates to the feed's names so the filter and the vote line up across all eight providers. Tests and the README are updated for the POST flow and the name mapping. --- plugins/release-tsundoku/README.md | 44 ++--- plugins/release-tsundoku/src/fetcher.test.ts | 83 +++++----- plugins/release-tsundoku/src/fetcher.ts | 67 +++++--- plugins/release-tsundoku/src/index.test.ts | 154 +++++------------ plugins/release-tsundoku/src/index.ts | 164 +++++++------------ plugins/release-tsundoku/src/manifest.ts | 45 ++--- plugins/release-tsundoku/src/matcher.test.ts | 8 + plugins/release-tsundoku/src/matcher.ts | 7 +- 8 files changed, 246 insertions(+), 326 deletions(-) diff --git a/plugins/release-tsundoku/README.md b/plugins/release-tsundoku/README.md index 9c86175c..7311d745 100644 --- a/plugins/release-tsundoku/README.md +++ b/plugins/release-tsundoku/README.md @@ -15,10 +15,12 @@ incremental series feed. **Notification-only** — Codex does not download anyth agreement wins. So a trusted disagreement (different MangaBaka IDs) overrides a sloppy agreement (a shared MAL ID), and genuinely ambiguous ties are skipped rather than mis-attributed. -- **Incremental, cursor-based.** Walks Tsundoku's keyset-paginated - `/api/v1/series/feed`, persisting its position in the plugin's - (system-scoped) KV store so each poll only processes activity since the last - run. +- **Filtered feed, no stored cursor.** Each poll `POST`s your tracked series' + `provider:externalId` set to Tsundoku's filtered `/api/v1/series/feed`, so the + response contains only your series (not the whole catalog). There's no + persisted cursor — every poll re-evaluates your tracked set's current + coverage and lets Codex dedup unchanged releases. Newly tracked series are + picked up automatically and untracked ones drop out, with no bookkeeping. - **Volume- and chapter-aware.** The feed's merged, gap-preserving coverage spans map directly onto Codex's release model. @@ -69,24 +71,25 @@ providers, in match-priority order: On each poll the plugin: -1. Loads its stored feed cursor from the plugin's system-scoped KV store - (`feed_cursor`). -2. Builds a reverse index (`provider:id → Codex series`) from your tracked - series via the host's `releases/list_tracked`. -3. Walks the feed from the cursor. Each item is matched to a tracked series by - weighted external-ID voting (see Features); on a confident match it records a - release candidate whose `volumes`/`chapters` mirror the item's coverage and - whose confidence reflects the vote. When several feed entries map to the same - Codex series, only the best-scoring one is recorded (ties are skipped). The - cursor is persisted after each processed page, so an interrupted walk resumes - cleanly. +1. Builds a match context from your tracked series via the host's + `releases/list_tracked`, and derives the `provider:externalId` filter set. +2. `POST`s that filter to `/api/v1/series/feed`, paginating through the response + (cursor used only within the poll; nothing is persisted). The response is + narrowed to your tracked series. +3. Matches each returned item to a tracked series by weighted external-ID voting + (see Features); on a confident match it records a release candidate whose + `volumes`/`chapters` mirror the item's coverage and whose confidence reflects + the vote. When several feed entries map to the same Codex series, only the + best-scoring one is recorded (ambiguous ties are skipped). 4. Reports counters back to the host; the host applies its own threshold, auto-ignore (for coverage you already own), and dedup. The candidate's dedup key is the coverage high-water mark (`tsundoku:{seriesId}:v{highestVolume}:c{highestChapter}`), so a new announcement fires only when the frontier advances; re-delivery of the same -coverage dedups host-side. +coverage dedups host-side. Because each poll re-evaluates the full tracked set, +**newly tracked series are backfilled on the next poll** and untracked ones stop +without any cursor bookkeeping. If the very first feed page can't be fetched (e.g. `baseUrl` is wrong or the instance is unreachable), the poll fails and the source shows `last_error` in @@ -100,10 +103,11 @@ container can resolve (e.g. `http://host.docker.internal:`), not - **Default language.** Tsundoku tracks official release coverage and carries no language, so every candidate uses `defaultLanguage` (`en` unless overridden). Per-series language preferences still gate the high-water mark host-side. -- **Incremental backfill gap.** Because the walk is cursor-based, a series you - start tracking *after* its last Tsundoku coverage change won't get a catch-up - announcement until it changes again. This is correct for "new releases going - forward"; a full backfill would require resetting the cursor. +- **Full re-walk each poll.** Each poll re-fetches the current coverage of your + whole tracked set (filtered server-side, so only your series). Cheap at + typical sizes and polled a few times a day; if it ever needs to scale, an + incremental cursor could be reintroduced (with explicit invalidation on + track/untrack). - **High-water dedup.** A filled interior gap that doesn't move the highest volume/chapter won't re-announce. diff --git a/plugins/release-tsundoku/src/fetcher.test.ts b/plugins/release-tsundoku/src/fetcher.test.ts index e7b0d730..5e2a5b05 100644 --- a/plugins/release-tsundoku/src/fetcher.test.ts +++ b/plugins/release-tsundoku/src/fetcher.test.ts @@ -1,38 +1,17 @@ import { describe, expect, it, vi } from "vitest"; -import { type FeedResponse, feedUrl, fetchFeedPage } from "./fetcher.js"; +import { type FeedRequest, type FeedResponse, feedUrl, fetchFeedPage } from "./fetcher.js"; // ----------------------------------------------------------------------------- // feedUrl // ----------------------------------------------------------------------------- describe("feedUrl", () => { - it("appends the feed path and limit", () => { - const url = feedUrl("https://t.example.com", null, 100); - expect(url).toBe("https://t.example.com/api/v1/series/feed?limit=100"); - }); - - it("includes the cursor when provided", () => { - const url = feedUrl("https://t.example.com", "abc123", 50); - const parsed = new URL(url); - expect(parsed.pathname).toBe("/api/v1/series/feed"); - expect(parsed.searchParams.get("limit")).toBe("50"); - expect(parsed.searchParams.get("cursor")).toBe("abc123"); - }); - - it("omits the cursor param when null or empty", () => { - expect(feedUrl("https://t.example.com", "", 10)).not.toContain("cursor="); - expect(feedUrl("https://t.example.com", null, 10)).not.toContain("cursor="); + it("appends the feed path", () => { + expect(feedUrl("https://t.example.com")).toBe("https://t.example.com/api/v1/series/feed"); }); it("strips trailing slashes from the base URL", () => { - expect(feedUrl("https://t.example.com///", null, 10)).toBe( - "https://t.example.com/api/v1/series/feed?limit=10", - ); - }); - - it("url-encodes an opaque cursor", () => { - const url = feedUrl("https://t.example.com", "a b/c+d", 10); - expect(new URL(url).searchParams.get("cursor")).toBe("a b/c+d"); + expect(feedUrl("https://t.example.com///")).toBe("https://t.example.com/api/v1/series/feed"); }); }); @@ -64,48 +43,70 @@ function jsonResponse(body: unknown, status = 200): Response { }); } +function req(overrides: Partial = {}): FeedRequest { + return { externalIds: ["mangabaka:9741"], cursor: null, limit: 100, ...overrides }; +} + +/** Read the JSON body of the first call to a mocked fetch. */ +function calledBody(fetchImpl: typeof fetch): Record { + const init = (fetchImpl as unknown as ReturnType).mock.calls[0][1] as RequestInit; + return JSON.parse(init.body as string); +} + +function calledInit(fetchImpl: typeof fetch): RequestInit { + return (fetchImpl as unknown as ReturnType).mock.calls[0][1] as RequestInit; +} + describe("fetchFeedPage", () => { it("returns ok with the parsed page on 200", async () => { const fetchImpl = vi .fn() .mockResolvedValue(jsonResponse(samplePage)) as unknown as typeof fetch; - const result = await fetchFeedPage("https://t.example.com", null, 100, { fetchImpl }); + const result = await fetchFeedPage("https://t.example.com", req(), { fetchImpl }); expect(result.kind).toBe("ok"); if (result.kind !== "ok") throw new Error("expected ok"); expect(result.data.hasMore).toBe(true); expect(result.data.nextCursor).toBe("next-cursor-token"); - expect(result.data.items).toHaveLength(1); expect(result.data.items[0].seriesId).toBe(87); }); - it("sends the cursor and limit in the request URL", async () => { + it("POSTs the external-id filter, cursor and limit in the body", async () => { const fetchImpl = vi .fn() .mockResolvedValue(jsonResponse(samplePage)) as unknown as typeof fetch; - await fetchFeedPage("https://t.example.com", "cur-1", 250, { fetchImpl }); + await fetchFeedPage( + "https://t.example.com", + { externalIds: ["mangabaka:9741", "mal:5"], cursor: "cur-1", limit: 250 }, + { fetchImpl }, + ); const calledUrl = (fetchImpl as unknown as ReturnType).mock.calls[0][0] as string; - const parsed = new URL(calledUrl); - expect(parsed.searchParams.get("cursor")).toBe("cur-1"); - expect(parsed.searchParams.get("limit")).toBe("250"); + expect(calledUrl).toBe("https://t.example.com/api/v1/series/feed"); + const init = calledInit(fetchImpl); + expect(init.method).toBe("POST"); + expect((init.headers as Record).Accept).toBe("application/json"); + expect((init.headers as Record)["Content-Type"]).toBe("application/json"); + + const body = calledBody(fetchImpl); + expect(body.externalIds).toEqual(["mangabaka:9741", "mal:5"]); + expect(body.cursor).toBe("cur-1"); + expect(body.limit).toBe(250); }); - it("requests JSON via the Accept header", async () => { + it("sends a null cursor for the first page", async () => { const fetchImpl = vi .fn() .mockResolvedValue(jsonResponse(samplePage)) as unknown as typeof fetch; - await fetchFeedPage("https://t.example.com", null, 100, { fetchImpl }); - - const init = (fetchImpl as unknown as ReturnType).mock.calls[0][1] as RequestInit; - expect((init.headers as Record).Accept).toBe("application/json"); + await fetchFeedPage("https://t.example.com", req({ cursor: null }), { fetchImpl }); + expect(calledBody(fetchImpl).cursor).toBeNull(); }); it("maps a non-200 status to an error result", async () => { const fetchImpl = vi .fn() .mockResolvedValue(new Response("nope", { status: 503 })) as unknown as typeof fetch; - const result = await fetchFeedPage("https://t.example.com", null, 100, { fetchImpl }); + const result = await fetchFeedPage("https://t.example.com", req(), { fetchImpl }); expect(result.kind).toBe("error"); if (result.kind !== "error") throw new Error("expected error"); @@ -116,7 +117,7 @@ describe("fetchFeedPage", () => { const fetchImpl = vi .fn() .mockRejectedValue(new Error("ECONNREFUSED")) as unknown as typeof fetch; - const result = await fetchFeedPage("https://t.example.com", null, 100, { fetchImpl }); + const result = await fetchFeedPage("https://t.example.com", req(), { fetchImpl }); expect(result.kind).toBe("error"); if (result.kind !== "error") throw new Error("expected error"); @@ -130,7 +131,7 @@ describe("fetchFeedPage", () => { .mockResolvedValue( new Response("not json", { status: 200, headers: { "content-type": "application/json" } }), ) as unknown as typeof fetch; - const result = await fetchFeedPage("https://t.example.com", null, 100, { fetchImpl }); + const result = await fetchFeedPage("https://t.example.com", req(), { fetchImpl }); expect(result.kind).toBe("error"); if (result.kind !== "error") throw new Error("expected error"); @@ -142,7 +143,7 @@ describe("fetchFeedPage", () => { const fetchImpl = vi .fn() .mockResolvedValue(jsonResponse({ hasMore: false })) as unknown as typeof fetch; - const result = await fetchFeedPage("https://t.example.com", null, 100, { fetchImpl }); + const result = await fetchFeedPage("https://t.example.com", req(), { fetchImpl }); expect(result.kind).toBe("error"); if (result.kind !== "error") throw new Error("expected error"); diff --git a/plugins/release-tsundoku/src/fetcher.ts b/plugins/release-tsundoku/src/fetcher.ts index 54a0ac65..34084555 100644 --- a/plugins/release-tsundoku/src/fetcher.ts +++ b/plugins/release-tsundoku/src/fetcher.ts @@ -1,15 +1,18 @@ /** * Tsundoku series-feed fetcher. * - * Wraps `fetch` against `GET {baseUrl}/api/v1/series/feed` with a hard + * Wraps `fetch` against `POST {baseUrl}/api/v1/series/feed` with a hard * timeout and JSON parsing, returning a discriminated result so the caller * can act on a parsed page (`ok`) or surface the upstream status back to the * host's per-host backoff layer (`error`). * - * The feed is keyset-paginated: pass the previous response's `nextCursor` - * back as `cursor` and walk while `hasMore` is true. Network and parsing are - * the only side effects; nothing here touches storage, the host, or process - * state, which keeps it trivially testable with a mocked `fetch`. + * We use the filtered `POST` variant — the body carries the consumer's + * `provider:externalId` set so the feed returns only the tracked series, not + * the whole catalog. The response is keyset-paginated: walk while `hasMore` is + * true, passing `nextCursor` back as `cursor`. That cursor paginates *within a + * single poll* and is not persisted — each poll re-walks the tracked set's + * current coverage and relies on host-side dedup. Network and parsing are the + * only side effects, which keeps it trivially testable with a mocked `fetch`. */ // ============================================================================= @@ -78,50 +81,64 @@ export const FEED_PATH = "/api/v1/series/feed"; const DEFAULT_TIMEOUT_MS = 10_000; -/** - * Build the feed URL for a page. Defensively strips trailing slashes off - * `baseUrl` so callers don't have to. `limit` is always sent; `cursor` is - * sent only when non-empty (its absence starts the walk from the beginning). - */ -export function feedUrl(baseUrl: string, cursor: string | null, limit: number): string { - const base = baseUrl.replace(/\/+$/, ""); - const params = new URLSearchParams(); - params.set("limit", String(limit)); - if (cursor) { - params.set("cursor", cursor); - } - return `${base}${FEED_PATH}?${params.toString()}`; +/** Body for one `POST /series/feed` page. */ +export interface FeedRequest { + /** + * `provider:externalId` filter — the feed is narrowed to series carrying one + * of these. Must be non-empty (an empty list means "no filter" upstream, + * i.e. the whole catalog — callers guard against that). + */ + externalIds: string[]; + /** Pagination cursor within this poll. `null` starts at the beginning. */ + cursor: string | null; + /** Page size (the caller clamps to 1..=500). */ + limit: number; +} + +/** Build the feed endpoint URL (trailing slashes on `baseUrl` tolerated). */ +export function feedUrl(baseUrl: string): string { + return `${baseUrl.replace(/\/+$/, "")}${FEED_PATH}`; } /** - * Fetch one page of the Tsundoku series feed. + * Fetch one page of the filtered Tsundoku series feed via `POST`. + * + * We post the tracked `externalIds` set so the feed returns only the + * consumer's series (not the whole catalog). The `cursor` is for pagination + * *within a single poll* — it is not persisted across polls; each poll walks + * the current coverage of the tracked set and relies on host-side dedup to + * suppress unchanged releases. * * @param baseUrl - Tsundoku instance base URL (trailing slash tolerated). - * @param cursor - Cursor from the previous page, or null to start over. - * @param limit - Page size (the caller is responsible for clamping to 1..=500). + * @param req - Filter set + pagination cursor + page size. * @param opts - Fetcher options (custom fetch, timeout). */ export async function fetchFeedPage( baseUrl: string, - cursor: string | null, - limit: number, + req: FeedRequest, opts: FeedFetcherOptions = {}, ): Promise { const fetchImpl = opts.fetchImpl ?? globalThis.fetch; const timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS; - const url = feedUrl(baseUrl, cursor, limit); + const url = feedUrl(baseUrl); const headers: Record = { Accept: "application/json", + "Content-Type": "application/json", "User-Agent": "Codex-ReleaseTracker/1.0 (+https://github.com/AshDevFr/codex)", }; + const body = JSON.stringify({ + externalIds: req.externalIds, + cursor: req.cursor, + limit: req.limit, + }); // AbortSignal.timeout is the cleanest path; we already require Node 22+. const signal = AbortSignal.timeout(timeoutMs); let resp: Response; try { - resp = await fetchImpl(url, { method: "GET", headers, signal }); + resp = await fetchImpl(url, { method: "POST", headers, body, signal }); } catch (err) { const msg = err instanceof Error ? err.message : "Unknown fetch error"; // Aborts and transport-level failures map to 0/unavailable so the host's diff --git a/plugins/release-tsundoku/src/index.test.ts b/plugins/release-tsundoku/src/index.test.ts index d3113f79..88c49b5b 100644 --- a/plugins/release-tsundoku/src/index.test.ts +++ b/plugins/release-tsundoku/src/index.test.ts @@ -1,14 +1,7 @@ -import { HostRpcClient, type PluginStorage } from "@ashdev/codex-plugin-sdk"; +import { HostRpcClient } from "@ashdev/codex-plugin-sdk"; import { describe, expect, it, vi } from "vitest"; import type { FeedItem, FeedResponse } from "./fetcher.js"; -import { - CURSOR_STORAGE_KEY, - loadCursor, - normalizeBaseUrl, - poll, - registerSources, - saveCursor, -} from "./index.js"; +import { normalizeBaseUrl, poll, registerSources } from "./index.js"; // ----------------------------------------------------------------------------- // Mock host RPC @@ -82,68 +75,6 @@ describe("normalizeBaseUrl", () => { }); }); -// ----------------------------------------------------------------------------- -// Cursor persistence (system-scoped KV store) -// ----------------------------------------------------------------------------- - -/** A stateful in-memory `PluginStorage` double recording every `set`. */ -function makeStorage(initialCursor: string | null = null): { - storage: PluginStorage; - sets: string[]; -} { - let value = initialCursor; - const sets: string[] = []; - const storage = { - get: vi.fn(async () => ({ data: value })), - set: vi.fn(async (_key: string, data: unknown) => { - value = data as string; - sets.push(value); - return { success: true }; - }), - } as unknown as PluginStorage; - return { storage, sets }; -} - -describe("loadCursor", () => { - it("returns the stored cursor string", async () => { - const { storage } = makeStorage("cursor-42"); - expect(await loadCursor(storage)).toBe("cursor-42"); - }); - - it("returns null when nothing is stored", async () => { - expect(await loadCursor(makeStorage(null).storage)).toBeNull(); - }); - - it("returns null and does not throw when the read fails", async () => { - const storage = { - get: vi.fn(async () => { - throw new Error("kv down"); - }), - set: vi.fn(), - } as unknown as PluginStorage; - expect(await loadCursor(storage)).toBeNull(); - }); -}); - -describe("saveCursor", () => { - it("writes the cursor under the feed-cursor key", async () => { - const { storage, sets } = makeStorage(); - await saveCursor(storage, "cursor-99"); - expect(sets).toEqual(["cursor-99"]); - expect((storage.set as ReturnType).mock.calls[0][0]).toBe(CURSOR_STORAGE_KEY); - }); - - it("swallows a write failure without throwing", async () => { - const storage = { - get: vi.fn(), - set: vi.fn(async () => { - throw new Error("kv full"); - }), - } as unknown as PluginStorage; - await expect(saveCursor(storage, "cursor-99")).resolves.toBeUndefined(); - }); -}); - // ----------------------------------------------------------------------------- // registerSources // ----------------------------------------------------------------------------- @@ -184,15 +115,19 @@ describe("registerSources", () => { type PageOrError = FeedResponse | { errorStatus: number }; -/** A `fetch` impl that returns the given pages/errors in order, then empties. */ +/** + * A `fetch` impl that returns the given pages/errors in order, then empties. + * Captures each POST request body so tests can assert the posted filter set + * and the in-poll pagination cursor. + */ function makeFetchSequence(pages: PageOrError[]): { fetchImpl: typeof fetch; - urls: string[]; + bodies: Array>; } { - const urls: string[] = []; + const bodies: Array> = []; let i = 0; - const fetchImpl = vi.fn(async (url: string) => { - urls.push(url); + const fetchImpl = vi.fn(async (_url: string, init?: RequestInit) => { + bodies.push(init?.body ? JSON.parse(init.body as string) : {}); const page = pages[i++] ?? { items: [], hasMore: false, nextCursor: null }; if ("errorStatus" in page) { return new Response("err", { status: page.errorStatus }); @@ -202,7 +137,7 @@ function makeFetchSequence(pages: PageOrError[]): { headers: { "content-type": "application/json" }, }); }) as unknown as typeof fetch; - return { fetchImpl, urls }; + return { fetchImpl, bodies }; } function item( @@ -250,8 +185,7 @@ function makePollRpc(opts: { }); } -const pollDeps = (fetchImpl: typeof fetch, storage: PluginStorage) => ({ - storage, +const pollDeps = (fetchImpl: typeof fetch) => ({ baseUrl: "https://t.example.com", language: "en", pageLimit: 100, @@ -260,15 +194,14 @@ const pollDeps = (fetchImpl: typeof fetch, storage: PluginStorage) => ({ }); describe("poll", () => { - it("walks pages, matches by external id, records, and persists the cursor per page", async () => { + it("posts the tracked external-id filter, walks pages, and records matches", async () => { const { rpc } = makePollRpc({ tracked: [ { seriesId: "uuid-a", externalIds: { mangabaka: "9741" } }, { seriesId: "uuid-b", externalIds: { mangabaka: "5555" } }, ], }); - const { storage, sets } = makeStorage(); - const { fetchImpl } = makeFetchSequence([ + const { fetchImpl, bodies } = makeFetchSequence([ { items: [item(87, "mangabaka", "9741", 16), item(99, "anilist", "999")], hasMore: true, @@ -277,7 +210,7 @@ describe("poll", () => { { items: [item(88, "mangabaka", "5555", 3)], hasMore: false, nextCursor: "c2" }, ]); - const res = await poll({ sourceId: "src-1" }, rpc, pollDeps(fetchImpl, storage)); + const res = await poll({ sourceId: "src-1" }, rpc, pollDeps(fetchImpl)); expect(res).toMatchObject({ parsed: 3, // 87, 99, 88 @@ -286,7 +219,23 @@ describe("poll", () => { deduped: 0, upstreamStatus: 200, }); - expect(sets).toEqual(["c1", "c2"]); + // Posted the tracked external-id filter, and paginated within the poll + // (first page cursor null, second page the prior response's nextCursor). + expect(new Set(bodies[0].externalIds as string[])).toEqual( + new Set(["mangabaka:9741", "mangabaka:5555"]), + ); + expect(bodies[0].cursor).toBeNull(); + expect(bodies[1].cursor).toBe("c1"); + }); + + it("skips the fetch entirely when no tracked series carry a known id", async () => { + const { rpc } = makePollRpc({ tracked: [] }); + const { fetchImpl, bodies } = makeFetchSequence([{ items: [], hasMore: false }]); + + const res = await poll({ sourceId: "src-1" }, rpc, pollDeps(fetchImpl)); + expect(res).toMatchObject({ parsed: 0, matched: 0, recorded: 0 }); + // Never POSTed — an empty filter would mean "whole catalog" upstream. + expect(bodies).toHaveLength(0); }); it("counts host dedup separately from inserts", async () => { @@ -294,12 +243,11 @@ describe("poll", () => { tracked: [{ seriesId: "uuid-a", externalIds: { mangabaka: "9741" } }], onRecord: () => ({ ledgerId: "l1", deduped: true }), }); - const { storage } = makeStorage(); const { fetchImpl } = makeFetchSequence([ { items: [item(87, "mangabaka", "9741", 16)], hasMore: false, nextCursor: "c1" }, ]); - const res = await poll({ sourceId: "src-1" }, rpc, pollDeps(fetchImpl, storage)); + const res = await poll({ sourceId: "src-1" }, rpc, pollDeps(fetchImpl)); expect(res).toMatchObject({ matched: 1, recorded: 0, deduped: 1 }); }); @@ -310,7 +258,6 @@ describe("poll", () => { const { rpc, calls } = makePollRpc({ tracked: [{ seriesId: "uuid-a", externalIds: { mangabaka: "9741", mal: "555" } }], }); - const { storage } = makeStorage(); const { fetchImpl } = makeFetchSequence([ { items: [item(87, "mangabaka", "9741", 16), item(88, "mal", "555", 9)], @@ -319,7 +266,7 @@ describe("poll", () => { }, ]); - const res = await poll({ sourceId: "src-1" }, rpc, pollDeps(fetchImpl, storage)); + const res = await poll({ sourceId: "src-1" }, rpc, pollDeps(fetchImpl)); expect(res).toMatchObject({ parsed: 2, matched: 1, recorded: 1 }); const recordCalls = calls.filter((c) => c.method === "releases/record"); @@ -335,7 +282,6 @@ describe("poll", () => { const { rpc, calls } = makePollRpc({ tracked: [{ seriesId: "uuid-a", externalIds: { mal: "555" } }], }); - const { storage } = makeStorage(); const { fetchImpl } = makeFetchSequence([ { items: [item(87, "mal", "555", 16), item(88, "mal", "555", 9)], @@ -344,7 +290,7 @@ describe("poll", () => { }, ]); - const res = await poll({ sourceId: "src-1" }, rpc, pollDeps(fetchImpl, storage)); + const res = await poll({ sourceId: "src-1" }, rpc, pollDeps(fetchImpl)); expect(res).toMatchObject({ parsed: 2, matched: 0, recorded: 0 }); expect(calls.some((c) => c.method === "releases/record")).toBe(false); }); @@ -353,12 +299,11 @@ describe("poll", () => { const { rpc, calls } = makePollRpc({ tracked: [{ seriesId: "uuid-a", externalIds: { mangabaka: "9741" } }], }); - const { storage } = makeStorage(); const { fetchImpl } = makeFetchSequence([ { items: [item(99, "anilist", "999")], hasMore: false, nextCursor: "c1" }, ]); - const res = await poll({ sourceId: "src-1" }, rpc, pollDeps(fetchImpl, storage)); + const res = await poll({ sourceId: "src-1" }, rpc, pollDeps(fetchImpl)); expect(res).toMatchObject({ parsed: 1, matched: 0, recorded: 0 }); expect(calls.some((c) => c.method === "releases/record")).toBe(false); }); @@ -368,57 +313,38 @@ describe("poll", () => { tracked: [{ seriesId: "uuid-a", externalIds: { mangabaka: "9741" } }], onRecord: () => ({ __error: { code: -32000, message: "ledger down" } }), }); - const { storage, sets } = makeStorage(); const { fetchImpl } = makeFetchSequence([ { items: [item(87, "mangabaka", "9741", 16)], hasMore: false, nextCursor: "c1" }, ]); - const res = await poll({ sourceId: "src-1" }, rpc, pollDeps(fetchImpl, storage)); + const res = await poll({ sourceId: "src-1" }, rpc, pollDeps(fetchImpl)); expect(res).toMatchObject({ parsed: 1, matched: 1, recorded: 0, deduped: 0 }); - expect(sets).toEqual(["c1"]); // walk still completed and advanced the cursor - }); - - it("resumes from the cursor stored in the KV store", async () => { - const { rpc } = makePollRpc({ tracked: [] }); - const { storage } = makeStorage("resume-here"); - const { fetchImpl, urls } = makeFetchSequence([ - { items: [], hasMore: false, nextCursor: null }, - ]); - - await poll({ sourceId: "src-1" }, rpc, pollDeps(fetchImpl, storage)); - expect(new URL(urls[0]).searchParams.get("cursor")).toBe("resume-here"); }); it("throws when even the first page can't be fetched (so the source shows last_error)", async () => { const { rpc } = makePollRpc({ tracked: [{ seriesId: "uuid-a", externalIds: { mangabaka: "9741" } }], }); - const { storage, sets } = makeStorage(); const { fetchImpl } = makeFetchSequence([{ errorStatus: 503 }]); - await expect(poll({ sourceId: "src-1" }, rpc, pollDeps(fetchImpl, storage))).rejects.toThrow( - /503/, - ); - expect(sets).toEqual([]); // nothing persisted on a hard failure + await expect(poll({ sourceId: "src-1" }, rpc, pollDeps(fetchImpl))).rejects.toThrow(/503/); }); it("stops without throwing on a mid-walk fetch error, keeping prior progress", async () => { const { rpc } = makePollRpc({ tracked: [{ seriesId: "uuid-a", externalIds: { mangabaka: "9741" } }], }); - const { storage, sets } = makeStorage(); const { fetchImpl } = makeFetchSequence([ { items: [item(87, "mangabaka", "9741", 16)], hasMore: true, nextCursor: "c1" }, { errorStatus: 503 }, ]); - const res = await poll({ sourceId: "src-1" }, rpc, pollDeps(fetchImpl, storage)); + const res = await poll({ sourceId: "src-1" }, rpc, pollDeps(fetchImpl)); expect(res).toMatchObject({ parsed: 1, matched: 1, recorded: 1, upstreamStatus: 503, }); - expect(sets).toEqual(["c1"]); }); }); diff --git a/plugins/release-tsundoku/src/index.ts b/plugins/release-tsundoku/src/index.ts index aba60b70..052ad054 100644 --- a/plugins/release-tsundoku/src/index.ts +++ b/plugins/release-tsundoku/src/index.ts @@ -1,26 +1,27 @@ /** * Tsundoku API-feed release-source plugin for Codex. * - * Tsundoku exposes a single, catalog-wide incremental feed - * (`GET /api/v1/series/feed`) ordered by `(updatedAt, id)` and walked with an - * opaque keyset cursor. Each item carries the provider external IDs Codex - * matches on plus the merged volume/chapter coverage for the series. This - * plugin polls that feed, matches each item to a tracked Codex series by - * *exact* external ID (no fuzzy matching), and records release candidates. + * Tsundoku exposes a series feed at `/api/v1/series/feed` carrying, per series, + * the provider external IDs Codex matches on plus the merged volume/chapter + * coverage. This plugin polls the **filtered** `POST` variant, matches each + * returned series to a tracked Codex series by weighted external-ID voting, and + * records release candidates. * - * Unlike the per-series RSS plugins (MangaUpdates, Nyaa), the feed is not - * scoped to the user's tracked series — it's the whole Tsundoku catalog's - * recent activity. So each poll: - * 1. Loads the stored cursor from the plugin KV store. - * 2. Builds a reverse index `"provider:id" -> codexSeriesId` from the - * host's `releases/list_tracked` rows (scoped by `requiresExternalIds`). - * 3. Walks the feed from the cursor, matching each item against the index - * and streaming matches via `releases/record`. - * 4. Persists the advancing cursor back to the KV store. + * Each poll: + * 1. Builds a match context from the host's `releases/list_tracked` rows + * (scoped by `requiresExternalIds`) and derives the `provider:externalId` + * filter set. + * 2. `POST`s that filter to `/series/feed`, so the response contains only the + * tracked series — not the whole catalog. There is no persisted cursor: + * each poll re-walks the tracked set's current coverage and relies on + * host-side dedup to suppress unchanged releases. This keeps newly + * tracked series backfilled and untracked ones dropped, automatically. + * 3. Matches each item (weighted voting), resolves cross-item (one feed entry + * per Codex series), and records via `releases/record`. * - * The feed walk and matching land in dedicated modules (`fetcher`, - * `matcher`, `candidate`); this entry point owns plugin lifecycle, config, - * source registration, and the poll orchestration that ties them together. + * The fetch, matching, and candidate mapping live in dedicated modules + * (`fetcher`, `matcher`, `candidate`); this entry point owns plugin lifecycle, + * config, source registration, and the poll orchestration. */ import { @@ -29,7 +30,6 @@ import { type HostRpcClient, HostRpcError, type InitializeParams, - type PluginStorage, RELEASES_METHODS, type ReleaseCandidate, type ReleasePollRequest, @@ -39,13 +39,10 @@ import { import { feedItemToCandidate } from "./candidate.js"; import { type FeedItem, fetchFeedPage } from "./fetcher.js"; import { manifest } from "./manifest.js"; -import { buildMatchContext, type MatchResult, matchItem } from "./matcher.js"; +import { buildMatchContext, externalIdFilter, type MatchResult, matchItem } from "./matcher.js"; const logger = createLogger({ name: manifest.name, level: "info" }); -/** KV-store key under which the feed cursor bookmark is persisted. */ -export const CURSOR_STORAGE_KEY = "feed_cursor"; - /** Default feed page size when config omits / mis-types `pageLimit`. */ const DEFAULT_PAGE_LIMIT = 100; /** Tsundoku caps the feed page size at 500. */ @@ -62,8 +59,6 @@ const DEFAULT_LANGUAGE = "en"; interface PluginState { hostRpc: HostRpcClient | null; - /** Per-plugin (system-scoped) KV store for the feed cursor bookmark. */ - storage: PluginStorage | null; /** Tsundoku instance base URL (no trailing slash), e.g. `https://t.example.com`. */ baseUrl: string; /** ISO 639-1 tag stamped on every candidate (the feed carries none). */ @@ -76,7 +71,6 @@ interface PluginState { const state: PluginState = { hostRpc: null, - storage: null, baseUrl: "", defaultLanguage: DEFAULT_LANGUAGE, pageLimit: DEFAULT_PAGE_LIMIT, @@ -86,7 +80,6 @@ const state: PluginState = { /** Reset state. Exported for tests; not part of the plugin contract. */ export function _resetState(): void { state.hostRpc = null; - state.storage = null; state.baseUrl = ""; state.defaultLanguage = DEFAULT_LANGUAGE; state.pageLimit = DEFAULT_PAGE_LIMIT; @@ -98,47 +91,6 @@ export function normalizeBaseUrl(raw: string): string { return raw.trim().replace(/\/+$/, ""); } -// ============================================================================= -// Cursor persistence (system-scoped KV store) -// ============================================================================= -// -// The feed cursor lives in the plugin's KV store under `feed_cursor`. Release -// sources are *system* plugins (no user context), so this is the per-plugin -// (system) bucket — the host resolves the scope from the connection. Persisted -// after each processed page so a long or interrupted walk resumes mid-feed. - -/** - * Load the feed cursor bookmark from the KV store. Returns `null` when no - * cursor has been stored yet (first run) or when the read fails — a missing - * cursor simply restarts the walk from the beginning, which is safe given - * keyset pagination is gap-free and the host dedups re-delivered items. - */ -export async function loadCursor(storage: PluginStorage): Promise { - try { - const res = await storage.get(CURSOR_STORAGE_KEY); - const data = res?.data; - return typeof data === "string" && data.length > 0 ? data : null; - } catch (err) { - const reason = err instanceof Error ? err.message : String(err); - logger.warn(`failed to load cursor; restarting from the beginning: ${reason}`); - return null; - } -} - -/** - * Persist the feed cursor bookmark. Best-effort: a failed write is logged but - * never aborts a poll — the worst case is re-walking already-seen pages on the - * next poll, which dedups host-side. - */ -export async function saveCursor(storage: PluginStorage, cursor: string): Promise { - try { - await storage.set(CURSOR_STORAGE_KEY, cursor); - } catch (err) { - const reason = err instanceof Error ? err.message : String(err); - logger.warn(`failed to persist cursor "${cursor}": ${reason}`); - } -} - // ============================================================================= // Source registration // ============================================================================= @@ -260,8 +212,6 @@ async function reportProgress( /** Dependencies a poll needs, defaulted from plugin state at the call site. */ export interface PollDeps { - /** System-scoped KV store holding the feed cursor bookmark. */ - storage: PluginStorage; /** Tsundoku base URL (no trailing slash). */ baseUrl: string; /** Language stamped on every candidate. */ @@ -277,12 +227,13 @@ export interface PollDeps { /** * Top-level poll handler. * - * Builds the exact-match index from the host's tracked series, then walks the - * Tsundoku feed from the stored cursor: each item is matched by external ID - * and, on a hit, recorded as a candidate. The cursor is persisted after every - * processed page so an interrupted walk resumes from the last completed page - * (keyset pagination is gap-free, and host-side dedup makes re-processing - * safe). Exported for tests. + * Builds the match context from the host's tracked series and posts their + * `provider:externalId` set to Tsundoku's filtered feed, so the response + * contains only the tracked series (not the whole catalog). It walks every + * page of that filtered feed each poll — there is no persisted cursor; the + * in-poll cursor only paginates the current response, and host-side dedup + * suppresses unchanged releases. Matched items are resolved cross-item (one + * feed entry per Codex series) and recorded. Exported for tests. */ export async function poll( params: ReleasePollRequest, @@ -291,47 +242,58 @@ export async function poll( ): Promise { const sourceId = params.sourceId; - // 1. Build the match context from the user's tracked series. The feed spans - // the whole Tsundoku catalog, so this is what scopes it to the user. + // 1. Build the match context from the user's tracked series, and derive the + // `provider:externalId` filter we post to Tsundoku. const trackedEntries: TrackedSeriesEntry[] = []; for await (const entry of iterateTrackedSeries(rpc, sourceId)) { trackedEntries.push(entry); } const ctx = buildMatchContext(trackedEntries); - if (ctx.series.size === 0) { + const externalIds = externalIdFilter(ctx); + if (externalIds.length === 0) { + // Nothing to query. Posting an empty filter would mean "no filter" upstream + // (the whole catalog), so skip entirely instead. logger.info( - `poll: no tracked series carry a Tsundoku-known external ID (source=${sourceId}); nothing to match`, + `poll: no tracked series carry a Tsundoku-known external ID (source=${sourceId}); nothing to fetch`, ); + return { + notModified: false, + upstreamStatus: 200, + parsed: 0, + matched: 0, + recorded: 0, + deduped: 0, + }; } - // 2. Walk the feed from the stored cursor, collecting per-item matches. We - // resolve them after the walk (cross-item) rather than recording inline, - // so that when several feed entries map to the same Codex series we keep - // only the best one instead of polluting the ledger. - let cursor = await loadCursor(deps.storage); + // 2. Walk the filtered feed, collecting per-item matches. We resolve them + // after the walk (cross-item) rather than recording inline, so that when + // several feed entries map to the same Codex series we keep only the best + // one instead of polluting the ledger. The cursor here is ephemeral — it + // paginates this poll's response and is never persisted. + let cursor: string | null = null; let parsed = 0; let worstStatus = 200; let pagesFetched = 0; const hits: Array<{ item: FeedItem; match: MatchResult }> = []; while (true) { - const result = await fetchFeedPage(deps.baseUrl, cursor, deps.pageLimit, { - timeoutMs: deps.timeoutMs, - fetchImpl: deps.fetchImpl, - }); + const result = await fetchFeedPage( + deps.baseUrl, + { externalIds, cursor, limit: deps.pageLimit }, + { timeoutMs: deps.timeoutMs, fetchImpl: deps.fetchImpl }, + ); if (result.kind === "error") { worstStatus = Math.max(worstStatus, result.status); // Couldn't fetch even the first page: surface a hard failure so the host // records `last_error` and the source shows it (e.g. an unreachable or // misconfigured `baseUrl`). A mid-walk failure, by contrast, keeps the - // pages already processed and just stops — the cursor is preserved. + // pages already processed and just stops. if (pagesFetched === 0) { throw new Error(`feed fetch failed (status ${result.status}): ${result.message}`); } - logger.warn( - `feed fetch failed (status ${result.status}): ${result.message}; stopping walk, cursor preserved`, - ); + logger.warn(`feed fetch failed (status ${result.status}): ${result.message}; stopping walk`); break; } @@ -345,16 +307,9 @@ export async function poll( } } - // Advance + persist the cursor before deciding whether to continue, so an - // error or crash on the next page resumes from this point. - const next = page.nextCursor ?? null; - if (next) { - cursor = next; - await saveCursor(deps.storage, next); - } - await reportProgress(rpc, parsed, parsed, `Processed ${parsed} feed items`); + const next = page.nextCursor ?? null; if (!page.hasMore) break; if (!next) { // hasMore with no advancing cursor would loop forever; stop defensively. @@ -362,6 +317,7 @@ export async function poll( break; } if (page.items.length === 0) break; + cursor = next; } // 3. Cross-item resolution: a Codex series should map to at most one feed @@ -439,14 +395,13 @@ createReleaseSourcePlugin({ manifest, provider: { async poll(params: ReleasePollRequest): Promise { - if (!state.hostRpc || !state.storage) { - throw new Error("Plugin not initialized: host RPC / storage client missing"); + if (!state.hostRpc) { + throw new Error("Plugin not initialized: host RPC client missing"); } if (!state.baseUrl) { throw new Error("Plugin not configured: baseUrl is required"); } return poll(params, state.hostRpc, { - storage: state.storage, baseUrl: state.baseUrl, language: state.defaultLanguage, pageLimit: state.pageLimit, @@ -457,7 +412,6 @@ createReleaseSourcePlugin({ logLevel: "info", async onInitialize(params: InitializeParams) { state.hostRpc = params.hostRpc; - state.storage = params.storage; const ac = params.adminConfig ?? {}; if (typeof ac.baseUrl === "string") { diff --git a/plugins/release-tsundoku/src/manifest.ts b/plugins/release-tsundoku/src/manifest.ts index 0f3ba88c..d9973977 100644 --- a/plugins/release-tsundoku/src/manifest.ts +++ b/plugins/release-tsundoku/src/manifest.ts @@ -2,27 +2,32 @@ import type { PluginManifest } from "@ashdev/codex-plugin-sdk"; import packageJson from "../package.json" with { type: "json" }; /** - * External-ID source names Tsundoku exposes on each feed item, in match - * priority order (most canonical first). These are the *bare* service names - * (no `api:` / `plugin:` prefix): the host strips the stored prefix before - * filtering, so a manifest `requiresExternalIds` entry of `"mangabaka"` - * matches a series whose ID was stored as `api:mangabaka` or - * `plugin:mangabaka`. + * Maps a Codex external-ID source name to the provider name the Tsundoku feed + * uses. Codex stores some sources under different names than Tsundoku emits + * (e.g. Codex `myanimelist` ↔ Tsundoku `mal`), so we translate when building + * the match index and the feed filter. Identity for names that already agree. * - * The order matters only for the candidate's `reason` string — the first - * provider that resolves a tracked series wins. MangaBaka is the dominant - * cross-reference hub in Codex, so it leads. + * The keys are the *bare* Codex source names — the host strips the stored + * `api:` / `plugin:` prefix before matching `requiresExternalIds`, so a series + * stored as `api:myanimelist` is delivered to us as `myanimelist`. */ -export const TSUNDOKU_EXTERNAL_ID_SOURCES = [ - "mangabaka", - "anilist", - "mal", - "mangaupdates", - "kitsu", - "shikimori", - "anime_planet", - "anime_news_network", -] as const; +export const CODEX_TO_TSUNDOKU_PROVIDER: Record = { + mangabaka: "mangabaka", + anilist: "anilist", + myanimelist: "mal", + mangaupdates: "mangaupdates", + kitsu: "kitsu", + shikimori: "shikimori", + animeplanet: "anime_planet", + animenewsnetwork: "anime_news_network", +}; + +/** + * The Codex source names the plugin asks the host for via + * `requiresExternalIds`. These must be the names Codex *stores* (the map keys), + * not Tsundoku's — the host filters `series_external_ids.source` against them. + */ +export const CODEX_EXTERNAL_ID_SOURCES = Object.keys(CODEX_TO_TSUNDOKU_PROVIDER); export const manifest = { name: "release-tsundoku", @@ -37,7 +42,7 @@ export const manifest = { releaseSource: { kinds: ["api-feed"], requiresAliases: false, - requiresExternalIds: [...TSUNDOKU_EXTERNAL_ID_SOURCES], + requiresExternalIds: [...CODEX_EXTERNAL_ID_SOURCES], canAnnounceChapters: true, canAnnounceVolumes: true, }, diff --git a/plugins/release-tsundoku/src/matcher.test.ts b/plugins/release-tsundoku/src/matcher.test.ts index 60bedf96..9f15e126 100644 --- a/plugins/release-tsundoku/src/matcher.test.ts +++ b/plugins/release-tsundoku/src/matcher.test.ts @@ -70,6 +70,14 @@ describe("matchItem", () => { expect(res?.score).toBe(1); }); + it("translates Codex provider names to Tsundoku's (myanimelist -> mal)", () => { + // Codex stores `myanimelist`; the feed uses `mal`. They must still match. + const ctx = buildMatchContext([tracked("uuid-a", { myanimelist: "128555" })]); + const res = matchItem(feedItem([ext("mal", "128555")]), ctx); + expect(res?.codexSeriesId).toBe("uuid-a"); + expect(res?.agreeingProviders).toEqual(["mal"]); + }); + it("returns null when nothing is shared", () => { const ctx = buildMatchContext([tracked("uuid-a", { mangabaka: "9741" })]); expect(matchItem(feedItem([ext("mangabaka", "0000")]), ctx)).toBeNull(); diff --git a/plugins/release-tsundoku/src/matcher.ts b/plugins/release-tsundoku/src/matcher.ts index 203c45c8..3937d1ba 100644 --- a/plugins/release-tsundoku/src/matcher.ts +++ b/plugins/release-tsundoku/src/matcher.ts @@ -17,6 +17,7 @@ import type { TrackedSeriesEntry } from "@ashdev/codex-plugin-sdk"; import type { FeedItem } from "./fetcher.js"; +import { CODEX_TO_TSUNDOKU_PROVIDER } from "./manifest.js"; /** * Vote weight per provider — higher means more trusted as a match signal. @@ -70,8 +71,12 @@ export function buildMatchContext(entries: TrackedSeriesEntry[]): MatchContext { const ids = entry.externalIds; if (!ids) continue; const map = new Map(); - for (const [provider, externalId] of Object.entries(ids)) { + for (const [codexProvider, externalId] of Object.entries(ids)) { if (!externalId) continue; + // Translate the Codex source name to Tsundoku's provider name so both + // the index keys and the feed filter line up with what the feed emits + // (e.g. Codex `myanimelist` -> Tsundoku `mal`). + const provider = CODEX_TO_TSUNDOKU_PROVIDER[codexProvider] ?? codexProvider; map.set(provider, externalId); const key = indexKey(provider, externalId); const arr = byKey.get(key); From 5cb915278d4910a0dc7e28084600c31440ca31dd Mon Sep 17 00:00:00 2001 From: Sylvain Cau Date: Mon, 8 Jun 2026 19:59:22 -0700 Subject: [PATCH 10/10] test(tasks): pass result_data arg to mark_failed in task queue tests The `mark_failed` signature gained a fourth `result_data: Option` parameter when failed-task event replay was added, but the task queue tests were not updated and failed to compile. Pass `None` at each call site since these tests don't exercise event replay. --- tests/task_queue/mod.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/task_queue/mod.rs b/tests/task_queue/mod.rs index 2b931258..a5c381fd 100644 --- a/tests/task_queue/mod.rs +++ b/tests/task_queue/mod.rs @@ -231,7 +231,7 @@ async fn test_mark_failed_retry() { .expect("Failed to claim task"); // Mark as failed (should retry) - TaskRepository::mark_failed(&db, task_id, "Test error".to_string()) + TaskRepository::mark_failed(&db, task_id, "Test error".to_string(), None) .await .expect("Failed to mark failed"); @@ -266,7 +266,7 @@ async fn test_max_attempts_reached() { TaskRepository::claim_next(&db, "worker-1", 300) .await .expect("Failed to claim"); - TaskRepository::mark_failed(&db, task_id, "Error 1".to_string()) + TaskRepository::mark_failed(&db, task_id, "Error 1".to_string(), None) .await .expect("Failed to mark failed"); // Reset scheduled_for to now so we can claim immediately @@ -284,7 +284,7 @@ async fn test_max_attempts_reached() { TaskRepository::claim_next(&db, "worker-1", 300) .await .expect("Failed to claim"); - TaskRepository::mark_failed(&db, task_id, "Error 2".to_string()) + TaskRepository::mark_failed(&db, task_id, "Error 2".to_string(), None) .await .expect("Failed to mark failed"); // Reset scheduled_for @@ -302,7 +302,7 @@ async fn test_max_attempts_reached() { TaskRepository::claim_next(&db, "worker-1", 300) .await .expect("Failed to claim"); - TaskRepository::mark_failed(&db, task_id, "Error 3".to_string()) + TaskRepository::mark_failed(&db, task_id, "Error 3".to_string(), None) .await .expect("Failed to mark failed");