diff --git a/.github/dependabot.yml b/.github/dependabot.yml deleted file mode 100644 index 02087b4c..00000000 --- a/.github/dependabot.yml +++ /dev/null @@ -1,36 +0,0 @@ -version: 2 -updates: - # Rust backend (workspace). Replaces hand-bumping deps. - - package-ecosystem: cargo - directory: / - schedule: - interval: weekly - open-pull-requests-limit: 10 - groups: - cargo-minor-patch: - update-types: - - minor - - patch - - # Frontend + web (bun reads package.json; npm ecosystem covers the workspace). - - package-ecosystem: npm - directory: / - schedule: - interval: weekly - open-pull-requests-limit: 10 - groups: - npm-minor-patch: - update-types: - - minor - - patch - - # Keep the SHA-pinned GitHub Actions current so pinning doesn't rot. - - package-ecosystem: github-actions - directory: / - schedule: - interval: weekly - open-pull-requests-limit: 10 - groups: - actions-all: - patterns: - - "*" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f9ef5192..563d25ca 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -42,6 +42,7 @@ jobs: filters: | rust: - 'apps/backend/**' + - 'xtask/**' - 'libs/proto/**' - 'Cargo.toml' - 'Cargo.lock' @@ -276,6 +277,30 @@ jobs: - name: Build run: bunx nx run dashboard:build + packaging: + name: Packaging checks + needs: changes + if: needs.changes.outputs.rust == 'true' || needs.changes.outputs.other == 'true' + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: dtolnay/rust-toolchain@67ef31d5b988238dd797d409d6f9574278e20537 # master + with: + toolchain: "1.94" + - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 # v2 + with: + workspaces: ". -> target" + shared-key: linux-packaging + cache-on-failure: true + - name: xtask tests + run: cargo test -p xtask --locked + - name: package dry-run + run: | + cargo run -p xtask --locked -- package --kind all --version 0.0.0-ci --dry-run --skip-build --target-os linux --arch amd64 + cargo run -p xtask --locked -- package --kind host --version 0.0.0-ci --dry-run --skip-build --target-os macos --arch arm64 + cargo run -p xtask --locked -- package --kind host --version 0.0.0-ci --dry-run --skip-build --target-os windows --arch amd64 + web: name: Web (landing + docs) needs: changes @@ -307,7 +332,7 @@ jobs: # "skipped required check stays pending forever" trap of naive path filtering. ci-status: name: CI status - needs: [changes, lint, security, test, test-os, frontend, web] + needs: [changes, lint, security, test, test-os, frontend, packaging, web] if: always() runs-on: ubuntu-latest timeout-minutes: 5 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0aaeaf7f..7ecb54c7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -12,9 +12,6 @@ on: tags: - "v*" -permissions: - contents: write - jobs: # Build the static dashboard once. Every build job consumes this same dist. dashboard: @@ -45,7 +42,7 @@ jobs: if-no-files-found: error build: - name: Build (${{ matrix.os }}-${{ matrix.arch }}) + name: Build/package (${{ matrix.os }}-${{ matrix.arch }}) needs: dashboard runs-on: ${{ matrix.runner }} strategy: @@ -55,33 +52,21 @@ jobs: - os: linux arch: amd64 runner: ubuntu-latest - bin: vcms - asset: vcms-${{ github.ref_name }}-linux-amd64 - os: linux arch: arm64 runner: ubuntu-24.04-arm - bin: vcms - asset: vcms-${{ github.ref_name }}-linux-arm64 - os: macos arch: amd64 runner: macos-26-intel - bin: vcms - asset: vcms-${{ github.ref_name }}-macos-amd64 - os: macos arch: arm64 runner: macos-26 - bin: vcms - asset: vcms-${{ github.ref_name }}-macos-arm64 - os: windows arch: amd64 runner: windows-latest - bin: vcms.exe - asset: vcms-${{ github.ref_name }}-windows-amd64.exe - os: windows arch: arm64 runner: windows-11-arm - bin: vcms.exe - asset: vcms-${{ github.ref_name }}-windows-arm64.exe steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: @@ -107,20 +92,54 @@ jobs: workspaces: ". -> target" key: release-${{ matrix.os }}-${{ matrix.arch }} + - name: Install Linux packaging tools + if: matrix.os == 'linux' + run: sudo apt-get update && sudo apt-get install -y rpm + + - name: Install WiX + if: matrix.os == 'windows' + shell: pwsh + run: | + dotnet tool install --global wix --version 7.* + wix eula accept wix7 + wix extension add --global WixToolset.UI.wixext/7.0.0 + # Build cargo directly (dist is already present from the artifact); keep the # default `embed-dashboard` feature so rust-embed bakes in apps/dashboard/dist. - name: Build release binary working-directory: apps/backend run: cargo build --release --locked - - name: Stage artifact - shell: bash - run: | - mkdir -p dist - cp "target/release/${{ matrix.bin }}" "dist/${{ matrix.asset }}" + - name: Package artifacts + run: cargo run -p xtask -- package --kind host --skip-build --target-os ${{ matrix.os }} --arch ${{ matrix.arch }} - - name: Attach to release + - name: Upload release artifacts + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0 # v7.0.1 + with: + name: release-${{ matrix.os }}-${{ matrix.arch }} + path: dist/packages/* + if-no-files-found: error + + publish: + name: Attest and publish release + needs: build + runs-on: ubuntu-latest + permissions: + contents: write + id-token: write + attestations: write + steps: + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: release-* + path: dist/packages + merge-multiple: true + - name: Attest build provenance + uses: actions/attest-build-provenance@v3 + with: + subject-path: dist/packages/* + - name: Attach all artifacts after matrix passes uses: softprops/action-gh-release@718ea10b132b3b2eba29c1007bb80653f286566b # v3.0.1 with: - files: dist/${{ matrix.asset }} + files: dist/packages/* generate_release_notes: true diff --git a/AGENTS.md b/AGENTS.md index ab222c07..97c16426 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,12 +1,18 @@ # AI Agent Instructions for CMS (Rust + React) +## Product Identity + +- Human-facing product and service display name: **Velopulent CMS**. +- Canonical documentation: . +- Keep the executable, package name, native service identifier, environment prefix, and internal identifiers as `vcms` / `VCMS_*` for backward compatibility. Do not rename persisted paths or service keys when updating branding. + ## Architecture - **Backend**: Rust + Axum HTTP server, `SQLx` + (SQLite | PostgreSQL | MySQL), `rust-embed` (static assets) - **Frontend**: React in `apps/dashboard/` with Tanstack Router, Tanstack Query, shadcn/ui - **gRPC**: Separate server on port 50051 (compiled from `libs/proto/*.proto` via `tonic-build`) - **Build**: Nx orchestrates `dashboard:build` → `backend:build`. `build.rs` compiles proto files only. -- **Runtime**: Single binary serves REST API (`/api/*`), gRPC, GraphQL (`/api/graphql`), MCP (Streamable HTTP at `/mcp`), and static SPA fallback +- **Runtime**: Single binary serves REST API (`/api/*`), gRPC, GraphQL (`/api/graphql`), MCP (Streamable HTTP at `/mcp`), health probes (`/health/live`, `/health/ready`), and static SPA fallback ## Key Directories @@ -25,6 +31,8 @@ - `libs/proto/` - Protocol Buffer definitions (`cms.proto`) - `apps/dashboard/` - React frontend app - `apps/web/` - Landing Page and Documentation (NextJS + Fumadocs) +- `packaging/` - Native Linux, macOS, Windows, Debian, RPM, and Arch definitions and lifecycle scripts +- `xtask/` - Typed release/package orchestration; platform builders live in separate modules ## Developer Commands @@ -75,16 +83,35 @@ vcms admin reset-password --email U --password P vcms backup create [--scope instance|site] [--site ID] [--out FILE] [--no-files] [--encrypt] vcms backup list # list recorded backups vcms restore --file PATH [--scope instance|site] [--site ID] [--import-as-new] --yes +vcms mcp stdio # thin HTTP proxy to a running server's /mcp (for MCP clients) +vcms service status # normalized native-service status and manager details +vcms doctor # validate config, storage, database, bind ports, and service identity ``` `backup`/`restore` run offline (no HTTP server) against the configured database — the disaster-recovery path when the instance won't boot. `restore` is destructive and requires `--yes`. +`mcp stdio` opens no database, secrets, or search index of its own: it forwards +JSON-RPC between stdin/stdout and the server's `/mcp` Streamable-HTTP endpoint, +reading only `VCMS_MCP_TOKEN` (bearer) and `VCMS_MCP_URL` (default +`http://127.0.0.1:3000`). So it works even when the data is owned by the OS-service +account. The installed service pins `VCMS_HOME` to a system dir so the daemon stores +everything under one owned root. + Global flags (highest precedence): `--config `, `--bind `, `--database-url `, `--log-level `. The server auto-migrates the database on every startup; there is no separate migrate command. +## Packaging and Releases + +- Keep native package definitions and service files in `packaging/`; do not embed them in Rust source or workflow YAML. +- `xtask` orchestrates deterministic staging and packaging. Keep `xtask/src/main.rs` limited to CLI parsing and dispatch, with shared and platform-specific implementation in modules. +- Keep the release workflow thin: build the dashboard, run native build/package jobs, assemble artifacts, attest, and publish. +- Ordinary CI validates packaging templates, `xtask` tests, and deterministic dry-runs. It must not install or mutate host services. +- Release artifacts include portable archives, Debian, RPM, MSI, and macOS PKG packages. Arch is maintained as a package recipe consuming published Linux archives; render it with `xtask arch-render`, not as a fake release archive. +- The stable native service identifier is `vcms`; its human-facing display name is **Velopulent CMS**. Fresh Linux and macOS package installs register/enable the service but do not auto-start it; the Windows MSI installs the service for automatic startup and starts it immediately. + ## Configuration Non-secret settings live in a TOML config file; secrets stay in the environment (or `.env`). @@ -93,29 +120,53 @@ Layers merge with precedence: **CLI flag > env var > config file > built-in defa Config file search order (first existing wins; missing is fine): 1. `--config` flag / `VCMS_CONFIG` env 2. `./vcms.toml` (current dir) -3. `~/.vcms/config.toml` (CMS home; `$VCMS_HOME/config.toml` if set) — where `vcms config init` writes +3. the platform config dir (`config.toml`; `$VCMS_HOME/config.toml` in single-dir mode) — where `vcms config init` writes 4. `/etc/vcms/config.toml` -## Data directory (CMS home) +## Data directory + +Resolution lives in `apps/backend/src/paths.rs` and has two layouts: -All runtime files live under one home directory: `$VCMS_HOME` if set, else `~/.vcms` -(same layout on Windows, macOS, Linux via the `directories` crate). `vcms serve` -creates it on first run. Resolution lives in `apps/backend/src/paths.rs`. +**Split (default, interactive installs)** — files land in the platform-conventional +per-type directories via the `directories` crate (`ProjectDirs`): + +| File(s) | Dir | Linux | macOS | Windows | +|---------|-----|-------|-------|---------| +| `config.toml`, `secrets.toml`, `.env` | config | `~/.config/vcms` | `~/Library/Application Support/vcms` | `%APPDATA%\vcms\config` | +| `vcms.db`, `storage/`, `backups/` | data | `~/.local/share/vcms` | `~/Library/Application Support/vcms` | `%APPDATA%\vcms\data` | +| `search/` (derived, rebuildable) | cache | `~/.cache/vcms` | `~/Library/Caches/vcms` | `%LOCALAPPDATA%\vcms\cache` | +| `logs/` | state | `~/.local/state/vcms` | `~/Library/Application Support/vcms` | `%LOCALAPPDATA%\vcms\data` | + +(`logs/` uses the state dir where the platform has one — Linux — else the local data dir.) + +**Single** — everything nests under one root. Chosen (in precedence order) when: +1. **`$VCMS_HOME` is set** — forces the root explicitly. +2. **the system service home dir exists** — Linux `/var/lib/vcms`, macOS + `/Library/Application Support/vcms`, Windows `C:\ProgramData\vcms`. The + platform installer creates it (and leaves it behind on uninstall), so a plain + `vcms serve`/`admin`/`backup` **follows the service's data instead of forking to a + per-user split store**. This path is defined once in `paths::system_home()` and + imported by the Windows SCM host. +3. **a legacy `~/.vcms` exists** — an existing install keeps working untouched. + +Otherwise (dev/eval boxes with no service) files use the platform split dirs. ```text -~/.vcms/ - config.toml # non-secret config (vcms config init target) - secrets.toml # auto-generated HMAC_SECRET + backup key (0600 on unix) - vcms.db # default SQLite database (+ -wal / -shm) - logs/ # rolling logs when [log] output = "file" - storage/ # default filesystem storage for uploads +$VCMS_HOME/ # or system home, or ~/.vcms (legacy) + config.toml secrets.toml .env + vcms.db (+ -wal / -shm) logs/ storage/ backups/ search/ ``` -Secrets: on first `serve`/`admin`, a random `HMAC_SECRET` is generated -and persisted to `secrets.toml` (`apps/backend/src/secrets.rs`), then loaded by -every process — including `vcms mcp stdio`, which is launched from an arbitrary cwd -and so cannot rely on a cwd `.env`. Env vars still override the file. `mcp stdio` -is read-only: it never creates the home dir, database, or secrets file. +When the active home is the system service home (owned by SYSTEM/root), the +data-touching commands (`serve`/`admin`/`backup`/`restore`) require elevation — a +non-elevated invocation **fails fast** with an "Administrator/root" hint (in +`paths::ensure`'s preflight) rather than silently forking to a second store. + +Secrets: on first `serve`/`admin`, a random `HMAC_SECRET` is generated and persisted to +`secrets.toml` (`apps/backend/src/secrets.rs`), then loaded by the server processes. +`vcms mcp stdio` does **not** load secrets, the database, or any data-dir file: it is a +thin HTTP proxy to the running server's `/mcp` (see CLI), forwarding `VCMS_MCP_TOKEN` as +the bearer; the server owns all disk I/O. Env-only secrets (never read from `config.toml` by convention, omitted from `config init`): `DATABASE_URL`, `HMAC_SECRET`, `S3_ACCESS_KEY_ID`, @@ -153,8 +204,10 @@ Logging keys map to the `[log]` table: `RUST_LOG`→`log.level`, `LOG_OUTPUT`→ | Variable | Default | Description | |----------|---------|-------------| | `VCMS_CONFIG` | - | Explicit config file path (same as `--config`) | -| `VCMS_HOME` | `~/.vcms` | CMS home directory (db, config, secrets, logs, storage) | -| `DATABASE_URL` | `sqlite://~/.vcms/vcms.db` | Database URL: `sqlite:path`, `postgres://...`, `mysql://...` | +| `VCMS_HOME` | - | If set, forces single-dir mode: db/config/secrets/logs/storage all nest under this root (else files use the platform split dirs) | +| `VCMS_MCP_TOKEN` | - | `vcms_site_*` access token forwarded by `vcms mcp stdio` as the bearer credential (required for stdio) | +| `VCMS_MCP_URL` | `http://127.0.0.1:3000` | Running server's base URL that `vcms mcp stdio` proxies to (`{url}/mcp`) | +| `DATABASE_URL` | `sqlite:///vcms.db` | Database URL: `sqlite:path`, `postgres://...`, `mysql://...` | | `HMAC_SECRET` | auto | HMAC key for token lookup (required; auto-generated to `secrets.toml`, env overrides) | | `BIND_ADDRESS` | `0.0.0.0:3000` | REST API listen address | | `GRPC_BIND_ADDRESS` | `0.0.0.0:50051` | gRPC server listen address | @@ -167,7 +220,7 @@ Logging keys map to the `[log]` table: `RUST_LOG`→`log.level`, `LOG_OUTPUT`→ | `S3_PUBLIC_URL` | - | Public URL for S3 assets | | `BACKUP_ENABLED` | `true` | Run the scheduled-backup poller / allow backups | | `BACKUP_DESTINATION` | `filesystem` | Backup destination: `filesystem` or `s3` | -| `BACKUP_LOCAL_PATH` | `~/.vcms/backups` | Local backup dir (when destination is filesystem) | +| `BACKUP_LOCAL_PATH` | `/backups` | Local backup dir (when destination is filesystem) | | `BACKUP_ZSTD_LEVEL` | `12` | zstd compression level for backups | | `BACKUP_DEFAULT_RETENTION` | `7` | Default "keep last N" for new schedules | | `BACKUP_S3_BUCKET` / `_REGION` / `_ENDPOINT` / `_PUBLIC_URL` | - | S3 backup destination (non-secret parts) | @@ -175,6 +228,7 @@ Logging keys map to the `[log]` table: `RUST_LOG`→`log.level`, `LOG_OUTPUT`→ | `BACKUP_S3_SECRET_ACCESS_KEY` | - | S3 backup secret key (secret, env-only) | | `BACKUP_ENCRYPTION_KEY` | auto | AES-256 backup key (hex); auto-generated to `secrets.toml` | | `MAX_UPLOAD_SIZE_MB` | `50` | Max upload size in MB | +| `UPLOAD_TOKEN_EXPIRY_SECS` | `900` | Signed upload URL lifetime (seconds) | | `COOKIE_SECURE` | `false` | Require HTTPS cookies | | `DB_MAX_CONNECTIONS` | `10` | Max DB connections | | `DB_MIN_CONNECTIONS` | `2` | Min DB connections | @@ -185,10 +239,10 @@ Logging keys map to the `[log]` table: `RUST_LOG`→`log.level`, `LOG_OUTPUT`→ | `LOG_OUTPUT` | `stdout` | `stdout` or `file` (`[log] output`) | | `LOG_FORMAT` | `pretty` | `pretty` or `json` (`[log] format`) | | `LOG_ANNOTATIONS` | `false` | Include file + line numbers (`[log] annotations`) | -| `LOG_DIR` | `~/.vcms/logs` | Log directory when `output = file` (`[log] dir`) | +| `LOG_DIR` | `/logs` | Log directory when `output = file` (`[log] dir`) | -**Note**: `HMAC_SECRET` is auto-generated and persisted to -`~/.vcms/secrets.toml` on first run. Set it explicitly via env to override. +**Note**: `HMAC_SECRET` is auto-generated and persisted to the config dir's +`secrets.toml` on first run. Set it explicitly via env to override. ## Proto Compilation diff --git a/CLAUDE.md b/CLAUDE.md index 7fa991bf..93eebca1 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -94,30 +94,54 @@ Layers merge with precedence: **CLI flag > env var > config file > built-in defa Config file search order (first existing wins; missing is fine): 1. `--config` flag / `VCMS_CONFIG` env 2. `./vcms.toml` (current dir) -3. `~/.vcms/config.toml` (CMS home; `$VCMS_HOME/config.toml` if set) — where `vcms config init` writes +3. the platform config dir (`config.toml`; `$VCMS_HOME/config.toml` in single-dir mode) — where `vcms config init` writes 4. `/etc/vcms/config.toml` -## Data directory (CMS home) +## Data directory -All runtime files live under one home directory: `$VCMS_HOME` if set, else `~/.vcms` -(same layout on Windows, macOS, Linux via the `directories` crate). `vcms serve` -creates it on first run. Resolution lives in `apps/backend/src/paths.rs`. +Resolution lives in `apps/backend/src/paths.rs` and has two layouts: + +**Split (default, interactive installs)** — files land in the platform-conventional +per-type directories via the `directories` crate (`ProjectDirs`): + +| File(s) | Dir | Linux | macOS | Windows | +|---------|-----|-------|-------|---------| +| `config.toml`, `secrets.toml`, `.env` | config | `~/.config/vcms` | `~/Library/Application Support/vcms` | `%APPDATA%\vcms\config` | +| `vcms.db`, `storage/`, `backups/` | data | `~/.local/share/vcms` | `~/Library/Application Support/vcms` | `%APPDATA%\vcms\data` | +| `search/` (derived, rebuildable) | cache | `~/.cache/vcms` | `~/Library/Caches/vcms` | `%LOCALAPPDATA%\vcms\cache` | +| `logs/` | state | `~/.local/state/vcms` | `~/Library/Application Support/vcms` | `%LOCALAPPDATA%\vcms\data` | + +(`logs/` uses the state dir where the platform has one — Linux — else the local data dir.) + +**Single** — everything nests under one root. Chosen (in precedence order) when: +1. **`$VCMS_HOME` is set** — forces the root explicitly. +2. **the system service home dir exists** — Linux `/var/lib/vcms`, macOS + `/Library/Application Support/vcms`, Windows `C:\ProgramData\vcms`. The + platform installer creates it (and leaves it behind on uninstall), so a plain + `vcms serve`/`admin`/`backup` **follows the service's data instead of forking to a + per-user split store**. This path is defined once in `paths::system_home()` and + imported by the Windows SCM host. +3. **a legacy `~/.vcms` exists** — an existing install keeps working untouched. + +Otherwise (dev/eval boxes with no service) files use the platform split dirs. ```text -~/.vcms/ - config.toml # non-secret config (vcms config init target) - secrets.toml # auto-generated HMAC_SECRET + backup key (0600 on unix) - vcms.db # default SQLite database (+ -wal / -shm) - logs/ # rolling logs when [log] output = "file" - storage/ # default filesystem storage for uploads - search/ # Tantivy full-text search index (derived; rebuildable) +$VCMS_HOME/ # or system home, or ~/.vcms (legacy) + config.toml secrets.toml .env + vcms.db (+ -wal / -shm) logs/ storage/ backups/ search/ ``` +`vcms serve`/`admin` create the dirs they need on first run; `vcms mcp stdio` creates +nothing. When the active home is the system service home (owned by SYSTEM/root), the +data-touching commands (`serve`/`admin`/`backup`/`restore`) require elevation — a +non-elevated invocation **fails fast** with an "Administrator/root" hint (in +`paths::ensure`'s preflight) rather than silently forking to a second store. + Secrets: on first `serve`/`admin`, a random `HMAC_SECRET` is generated and persisted to `secrets.toml` (`apps/backend/src/secrets.rs`), then loaded by -every process — including `vcms mcp stdio`, which is launched from an arbitrary cwd -and so cannot rely on a cwd `.env`. Env vars still override the file. `mcp stdio` -is read-only: it never creates the home dir, database, or secrets file. +the server processes. `vcms mcp stdio` does **not** load secrets, the database, or +any home-dir file: it is a thin HTTP proxy (see below) and the server it forwards to +owns all of those. Env-only secrets (never read from `config.toml` by convention, omitted from `config init`): `DATABASE_URL`, `HMAC_SECRET`, `S3_ACCESS_KEY_ID`, @@ -156,8 +180,10 @@ Logging keys map to the `[log]` table: `RUST_LOG`→`log.level`, `LOG_OUTPUT`→ | Variable | Default | Description | |----------|---------|-------------| | `VCMS_CONFIG` | - | Explicit config file path (same as `--config`) | -| `VCMS_HOME` | `~/.vcms` | CMS home directory (db, config, secrets, logs, storage) | -| `DATABASE_URL` | `sqlite://~/.vcms/vcms.db` | Database URL: `sqlite:path`, `postgres://...`, `mysql://...` | +| `VCMS_HOME` | - | If set, forces single-dir mode: db/config/secrets/logs/storage all nest under this root (else files use the platform split dirs) | +| `VCMS_MCP_TOKEN` | - | `vcms_site_*` access token forwarded by `vcms mcp stdio` as the bearer credential (required for stdio) | +| `VCMS_MCP_URL` | `http://127.0.0.1:3000` | Running server's base URL that `vcms mcp stdio` proxies to (`{url}/mcp`) | +| `DATABASE_URL` | `sqlite:///vcms.db` | Database URL: `sqlite:path`, `postgres://...`, `mysql://...` | | `HMAC_SECRET` | auto | HMAC key for token lookup (required; auto-generated to `secrets.toml`, env overrides) | | `BIND_ADDRESS` | `0.0.0.0:3000` | REST API listen address | | `GRPC_BIND_ADDRESS` | `0.0.0.0:50051` | gRPC server listen address | @@ -170,7 +196,7 @@ Logging keys map to the `[log]` table: `RUST_LOG`→`log.level`, `LOG_OUTPUT`→ | `S3_PUBLIC_URL` | - | Public URL for S3 assets | | `BACKUP_ENABLED` | `true` | Run the scheduled-backup poller / allow backups | | `BACKUP_DESTINATION` | `filesystem` | Backup destination: `filesystem` or `s3` | -| `BACKUP_LOCAL_PATH` | `~/.vcms/backups` | Local backup dir (when destination is filesystem) | +| `BACKUP_LOCAL_PATH` | `/backups` | Local backup dir (when destination is filesystem) | | `BACKUP_ZSTD_LEVEL` | `12` | zstd compression level for backups | | `BACKUP_DEFAULT_RETENTION` | `7` | Default "keep last N" for new schedules | | `BACKUP_S3_BUCKET` / `_REGION` / `_ENDPOINT` / `_PUBLIC_URL` | - | S3 backup destination (non-secret parts) | @@ -178,8 +204,9 @@ Logging keys map to the `[log]` table: `RUST_LOG`→`log.level`, `LOG_OUTPUT`→ | `BACKUP_S3_SECRET_ACCESS_KEY` | - | S3 backup secret key (secret, env-only) | | `BACKUP_ENCRYPTION_KEY` | auto | AES-256 backup key (hex); auto-generated to `secrets.toml` | | `SEARCH_ENABLED` | `true` | Build/use the Tantivy full-text index for entry search (else SQL `LIKE`) | -| `SEARCH_INDEX_PATH` | `~/.vcms/search` | Directory for the Tantivy search index | +| `SEARCH_INDEX_PATH` | `/search` | Directory for the Tantivy search index | | `MAX_UPLOAD_SIZE_MB` | `50` | Max upload size in MB | +| `UPLOAD_TOKEN_EXPIRY_SECS` | `900` | Signed upload URL lifetime (seconds) | | `COOKIE_SECURE` | `false` | Require HTTPS cookies | | `DB_MAX_CONNECTIONS` | `10` | Max DB connections | | `DB_MIN_CONNECTIONS` | `2` | Min DB connections | @@ -190,10 +217,10 @@ Logging keys map to the `[log]` table: `RUST_LOG`→`log.level`, `LOG_OUTPUT`→ | `LOG_OUTPUT` | `stdout` | `stdout` or `file` (`[log] output`) | | `LOG_FORMAT` | `pretty` | `pretty` or `json` (`[log] format`) | | `LOG_ANNOTATIONS` | `false` | Include file + line numbers (`[log] annotations`) | -| `LOG_DIR` | `~/.vcms/logs` | Log directory when `output = file` (`[log] dir`) | +| `LOG_DIR` | `/logs` | Log directory when `output = file` (`[log] dir`) | -**Note**: `HMAC_SECRET` is auto-generated and persisted to -`~/.vcms/secrets.toml` on first run. Set it explicitly via env to override. +**Note**: `HMAC_SECRET` is auto-generated and persisted to the config dir's +`secrets.toml` on first run. Set it explicitly via env to override. ## Proto Compilation @@ -298,23 +325,28 @@ is **derived** and fully rebuildable. scalar text of `data`, English-stemmed, BM25-ranked). `fields_from` re-resolves the schema when opening an existing index read-only. - `mod.rs` — `SearchService`. **Reading** needs no lock: `open_read_only` (reader - only) lets *any* process search the index, including a separate `vcms mcp stdio` - running next to the server. **Writing** requires the directory lock: `open` + only) lets *any* auxiliary process search the index alongside the server (this is a + general capability; `vcms mcp stdio` itself no longer opens the index — it proxies + over HTTP). **Writing** requires the directory lock: `open` (reader + writer) is held only by the running server. `index_doc`/`delete_doc` stage uncommitted ops; `commit` flushes; `rebuild_all`/`rebuild_site` reindex from the DB by reusing `EntryRepository::list` (covers singletons too). Write/commit on a read-only instance returns `SearchError::ReadOnly`. - `queue.rs` — `SearchQueue` over the `search_index_queue` table (migration - `20260618000000`). Content writes from *any* process enqueue here + `20260618000000`). Content writes enqueue here (`enqueue`/`dequeue_batch`/`delete_ids`); UUIDv7 ids order the queue chronologically. + The table exists purely for **durability** (crash recovery), not cross-process + signaling — all producers run in the server process (`vcms mcp stdio` is an HTTP proxy). - `indexer.rs` — the **single consumer**, spawned once by the server (it owns the writer). Drains the queue in batches (present in DB ⇒ upsert doc, absent ⇒ delete - doc — `op` is advisory), commits per batch, deletes processed rows. Wakes instantly - on a local enqueue (`Notify`) and polls every 2s to catch other processes' enqueues. + doc — `op` is advisory), commits per batch, deletes processed rows. **Purely + event-driven**: one startup drain (rows left by a crash), then sleeps until an + enqueue rings the in-process `Notify` — no polling. Wiring: `Services` holds `search: Option>` (reads) and `search_queue: Option>` (writes). `Services::new` opens the index -read-write (server); `Services::new_read_only` opens it read-only (`vcms mcp stdio`). +read-write (server); `Services::new_read_only` opens it read-only for an auxiliary +process that searches without taking the writer lock. `EntryService`/`SingletonService` **enqueue** on write; `EntryService::list_entries` queries the index — so REST, GraphQL, gRPC, and MCP all get ranked search via the existing `search` param with **no handler changes**. Indexing is asynchronous @@ -324,10 +356,9 @@ index builds on startup when empty, rebuilds after a restore, and exposes owner/operator reindex routes: `POST /api/dashboard/instance/search/reindex` and `POST /api/dashboard/sites/{site_id}/search/reindex`. -This is the cross-process model: writes from `vcms mcp stdio` (or any process) land in -the durable queue and the running server indexes them; if the server is down they -drain on its next start. The remaining hard limit is *concurrent writers* — only one -process indexes at a time. +Writes land in the durable queue and the running server indexes them; rows enqueued +before a crash drain on the next start. The remaining hard limit is *concurrent +writers* — only one process indexes at a time. ### No typo tolerance (deliberate) — and how to add it diff --git a/Cargo.lock b/Cargo.lock index ac72d5f2..2690fab2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -16,34 +16,34 @@ checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" [[package]] name = "aead" -version = "0.5.2" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" +checksum = "1973cfbc1a2daf9cf550e74e1f088c28e7f7d8c1e1418fb6c9dc5184b7e84c99" dependencies = [ - "crypto-common 0.1.7", - "generic-array", + "crypto-common 0.2.2", + "inout", ] [[package]] name = "aes" -version = "0.8.4" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +checksum = "f1fc76eaeac4c9164506c466d4ffdd8ec9d0c5bf57ee97177c4d8eceb3a0e138" dependencies = [ - "cfg-if", - "cipher 0.4.4", - "cpufeatures 0.2.17", + "cipher", + "cpubits", + "cpufeatures 0.3.0", ] [[package]] name = "aes-gcm" -version = "0.10.3" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "831010a0f742e1209b3bcea8fab6a8e149051ba6099432c8cb2cc117dec3ead1" +checksum = "fdf011db2e21ce0d575593d749db5554b47fed37aff429e4dc50bc91ac93a028" dependencies = [ "aead", "aes", - "cipher 0.4.4", + "cipher", "ctr", "ghash", "subtle", @@ -143,9 +143,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.102" +version = "1.0.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3" [[package]] name = "arbitrary" @@ -155,9 +155,9 @@ checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" [[package]] name = "arc-swap" -version = "1.9.1" +version = "1.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a3a1fd6f75306b68087b831f025c712524bcb19aad54e557b1129cfa0a2b207" +checksum = "c049c0be4daef0b145cb3555416b3b8ef5b7888a38aea1a3a155801fe7b0810b" dependencies = [ "rustversion", ] @@ -413,9 +413,9 @@ dependencies = [ [[package]] name = "aws-lc-rs" -version = "1.17.0" +version = "1.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ec2f1fc3ec205783a5da9a7e6c1509cc69dedf09a1949e412c1e18469326d00" +checksum = "4342d8937fc7e5dd9b1c60292261c0670c882a2cd1719cfc11b1af41731e32ad" dependencies = [ "aws-lc-sys", "zeroize", @@ -423,14 +423,15 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.41.0" +version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a2f9779ce85b93ab6170dd940ad0169b5766ff848247aff13bb788b832fe3f4" +checksum = "6d9ceb1da931507a12f4fccea479dccd00da1943e1b4ae72d8e502d707361444" dependencies = [ "cc", "cmake", "dunce", "fs_extra", + "pkg-config", ] [[package]] @@ -583,7 +584,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "62ce3946557b35e71d1bbe07ec385073ce9eda05043f95de134eb578fcf1a298" dependencies = [ "byteorder", - "cipher 0.5.2", + "cipher", ] [[package]] @@ -668,6 +669,17 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4f4c707c6a209cbe82d10abd08e1ea8995e9ea937d2550646e02798948992be0" +[[package]] +name = "cfb" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d38f2da7a0a2c4ccf0065be06397cc26a81f4e528be095826eee9d4adbb8c60f" +dependencies = [ + "byteorder", + "fnv", + "uuid", +] + [[package]] name = "cfg-if" version = "1.0.4" @@ -705,24 +717,15 @@ dependencies = [ "windows-link", ] -[[package]] -name = "cipher" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" -dependencies = [ - "crypto-common 0.1.7", - "inout 0.1.4", -] - [[package]] name = "cipher" version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8cf2a2c93cd704877c0858356ed03480ff301ee950b43f1cbe4573b088bfa6c" dependencies = [ + "block-buffer 0.12.1", "crypto-common 0.2.2", - "inout 0.2.2", + "inout", ] [[package]] @@ -802,18 +805,20 @@ dependencies = [ "dotenvy", "email_address", "figment", + "futures-util", "hex", "hmac", "http", "http-body-util", "image", + "infer", "json-patch", "mime_guess", "object_store", "prost", "rand 0.10.1", "regex", - "reqwest 0.13.4", + "reqwest", "rmcp", "rust-embed", "schemars", @@ -846,6 +851,7 @@ dependencies = [ "utoipa", "utoipa-scalar", "uuid", + "windows-service", "wiremock", "zstd", ] @@ -942,6 +948,12 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "cpubits" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15b85f9c39137c3a891689859392b1bd49812121d0d61c9caf00d46ed5ce06ae" + [[package]] name = "cpufeatures" version = "0.2.17" @@ -975,6 +987,16 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" +[[package]] +name = "crc-fast" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e75b2483e97a5a7da73ac68a05b629f9c53cff58d8ed1c77866079e18b00dba5" +dependencies = [ + "digest 0.10.7", + "spin 0.10.0", +] + [[package]] name = "crc32fast" version = "1.5.0" @@ -1016,9 +1038,9 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.18" +version = "0.9.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" dependencies = [ "crossbeam-utils", ] @@ -1046,12 +1068,11 @@ checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" [[package]] name = "crypto-common" -version = "0.1.7" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" dependencies = [ "generic-array", - "rand_core 0.6.4", "typenum", ] @@ -1061,16 +1082,18 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" dependencies = [ + "getrandom 0.4.3", "hybrid-array", + "rand_core 0.10.1", ] [[package]] name = "ctr" -version = "0.9.2" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835" +checksum = "baaca1c4b237092596f64d571e9db6ce4109c4ef9742e27590f1709594461f21" dependencies = [ - "cipher 0.4.4", + "cipher", ] [[package]] @@ -1242,7 +1265,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer 0.10.4", - "crypto-common 0.1.7", + "crypto-common 0.1.6", ] [[package]] @@ -1501,7 +1524,7 @@ checksum = "5e139bc46ca777eb5efaf62df0ab8cc5fd400866427e56c68b22e414e53bd3be" dependencies = [ "futures-core", "futures-sink", - "spin", + "spin 0.9.8", ] [[package]] @@ -1658,9 +1681,9 @@ dependencies = [ [[package]] name = "generic-array" -version = "0.14.7" +version = "0.14.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2" dependencies = [ "typenum", "version_check", @@ -1707,11 +1730,10 @@ dependencies = [ [[package]] name = "ghash" -version = "0.5.1" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0d8a4362ccb29cb0b265253fb0a2728f592895ee6854fd9bc13f2ffda266ff1" +checksum = "2eecf2d5dc9b66b732b97707a0210906b1d30523eb773193ab777c0c84b3e8d5" dependencies = [ - "opaque-debug", "polyval", ] @@ -1906,9 +1928,9 @@ checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" [[package]] name = "hybrid-array" -version = "0.4.12" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9155a582abd142abc056962c29e3ce5ff2ad5469f4246b537ed42c5deba857da" +checksum = "818356c5132c1fede50f837ca96afbe78ff42413047f4abb886217845e1b6c8c" dependencies = [ "typenum", ] @@ -1945,7 +1967,6 @@ dependencies = [ "hyper", "hyper-util", "rustls", - "rustls-native-certs", "tokio", "tokio-rustls", "tower-service", @@ -2171,19 +2192,19 @@ dependencies = [ ] [[package]] -name = "inlinable_string" -version = "0.1.15" +name = "infer" +version = "0.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8fae54786f62fb2918dcfae3d568594e50eb9b5c25bf04371af6fe7516452fb" +checksum = "a588916bfdfd92e71cacef98a63d9b1f0d74d6599980d11894290e7ddefffcf7" +dependencies = [ + "cfb", +] [[package]] -name = "inout" -version = "0.1.4" +name = "inlinable_string" +version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" -dependencies = [ - "generic-array", -] +checksum = "c8fae54786f62fb2918dcfae3d568594e50eb9b5c25bf04371af6fe7516452fb" [[package]] name = "inout" @@ -2235,6 +2256,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.18" @@ -2468,16 +2498,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "md-5" -version = "0.10.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" -dependencies = [ - "cfg-if", - "digest 0.10.7", -] - [[package]] name = "md-5" version = "0.11.0" @@ -2578,7 +2598,7 @@ dependencies = [ "httparse", "memchr", "mime", - "spin", + "spin 0.9.8", "version_check", ] @@ -2600,6 +2620,18 @@ version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" +[[package]] +name = "nix" +version = "0.31.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" +dependencies = [ + "bitflags", + "cfg-if", + "cfg_aliases", + "libc", +] + [[package]] name = "no_std_io2" version = "0.9.4" @@ -2726,14 +2758,16 @@ dependencies = [ [[package]] name = "object_store" -version = "0.13.2" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "622acbc9100d3c10e2ee15804b0caa40e55c933d5aa53814cd520805b7958a49" +checksum = "765784b4390c6bcf80316e5a22f4e3661b639c9d8c83246856643c27d8ce9dbe" dependencies = [ "async-trait", + "aws-lc-rs", "base64", "bytes", "chrono", + "crc-fast", "form_urlencoded", "futures-channel", "futures-core", @@ -2742,14 +2776,15 @@ dependencies = [ "http-body-util", "humantime", "hyper", - "itertools", - "md-5 0.10.6", + "itertools 0.15.0", + "md-5", + "nix", "parking_lot", "percent-encoding", "quick-xml", "rand 0.10.1", - "reqwest 0.12.28", - "ring", + "reqwest", + "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", @@ -2760,6 +2795,7 @@ dependencies = [ "walkdir", "wasm-bindgen-futures", "web-time", + "windows-sys 0.61.2", ] [[package]] @@ -2780,12 +2816,6 @@ version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "269bca4c2591a28585d6bf10d9ed0332b7d76900a1b02bec41bdc3a2cdcda107" -[[package]] -name = "opaque-debug" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" - [[package]] name = "openssl-probe" version = "0.2.1" @@ -3007,13 +3037,12 @@ dependencies = [ [[package]] name = "polyval" -version = "0.6.2" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" +checksum = "7dfc63250416fea14f5749b90725916a6c903f599d51cb635aa7a52bfd03eede" dependencies = [ - "cfg-if", - "cpufeatures 0.2.17", - "opaque-debug", + "cpubits", + "cpufeatures 0.3.0", "universal-hash", ] @@ -3118,7 +3147,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03da047801ff44bb6a4d407d4860c05fd70bb81714e6b2f3812603d5b145b042" dependencies = [ "heck", - "itertools", + "itertools 0.14.0", "log", "multimap", "petgraph", @@ -3139,7 +3168,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", - "itertools", + "itertools 0.14.0", "proc-macro2", "quote", "syn", @@ -3204,9 +3233,9 @@ checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" [[package]] name = "quick-xml" -version = "0.39.4" +version = "0.40.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdcc8dd4e2f670d309a5f0e83fe36dfdc05af317008fea29144da1a2ac858e5e" +checksum = "2474bd2e5029e7ccb6abb2ba48cf2383a333851dedf495901544281590c7da7f" dependencies = [ "memchr", "serde", @@ -3320,15 +3349,6 @@ dependencies = [ "rand_core 0.9.5", ] -[[package]] -name = "rand_core" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" -dependencies = [ - "getrandom 0.2.17", -] - [[package]] name = "rand_core" version = "0.9.5" @@ -3360,7 +3380,7 @@ dependencies = [ "built", "cfg-if", "interpolate_name", - "itertools", + "itertools 0.14.0", "libc", "libfuzzer-sys", "log", @@ -3482,48 +3502,6 @@ version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" -[[package]] -name = "reqwest" -version = "0.12.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" -dependencies = [ - "base64", - "bytes", - "futures-core", - "futures-util", - "h2", - "http", - "http-body", - "http-body-util", - "hyper", - "hyper-rustls", - "hyper-util", - "js-sys", - "log", - "percent-encoding", - "pin-project-lite", - "quinn", - "rustls", - "rustls-native-certs", - "rustls-pki-types", - "serde", - "serde_json", - "serde_urlencoded", - "sync_wrapper", - "tokio", - "tokio-rustls", - "tokio-util", - "tower", - "tower-http 0.6.11", - "tower-service", - "url", - "wasm-bindgen", - "wasm-bindgen-futures", - "wasm-streams", - "web-sys", -] - [[package]] name = "reqwest" version = "0.13.4" @@ -3559,12 +3537,14 @@ dependencies = [ "sync_wrapper", "tokio", "tokio-rustls", + "tokio-util", "tower", "tower-http 0.6.11", "tower-service", "url", "wasm-bindgen", "wasm-bindgen-futures", + "wasm-streams", "web-sys", ] @@ -3590,9 +3570,9 @@ dependencies = [ [[package]] name = "rmcp" -version = "1.8.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d1f571c72940a19d9532fe52dbea8bc9912bf1d766c2970bb824056b86f3f59" +checksum = "d52d21e5b342699bc4de690e6104fc4e43255e4e8420ff0f2cbb963aac09da6f" dependencies = [ "async-trait", "base64", @@ -3621,9 +3601,9 @@ dependencies = [ [[package]] name = "rmcp-macros" -version = "1.8.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1aad0035b69380782d78ea95b508327e6deaa2235909053e596eea8f27b5e1d5" +checksum = "6c68cec74c5b3ac73ff46375ae49e161637bda80bba70f0d5db641583bb308ee" dependencies = [ "darling 0.23.0", "proc-macro2", @@ -3712,7 +3692,6 @@ checksum = "6b92b125634d9b795e7beca796cc790df15a7fb38323bf3196fda83292d06b1f" dependencies = [ "aws-lc-rs", "once_cell", - "ring", "rustls-pki-types", "rustls-webpki", "subtle", @@ -3733,9 +3712,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.14.1" +version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" +checksum = "764899a24af3980067ee14bc143654f297b22eaebfe3c7b6b211920a5a59b046" dependencies = [ "web-time", "zeroize", @@ -3989,6 +3968,12 @@ dependencies = [ "digest 0.11.3", ] +[[package]] +name = "sha1_smol" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbfa15b3dddfee50a0fff136974b3e1bde555604ba463834a7eb7deb6417705d" + [[package]] name = "sha2" version = "0.10.9" @@ -4110,6 +4095,12 @@ dependencies = [ "lock_api", ] +[[package]] +name = "spin" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5fe4ccb98d9c292d56fec89a5e07da7fc4cf0dc11e156b41793132775d3e591" + [[package]] name = "sqlx" version = "0.9.0" @@ -4248,7 +4239,7 @@ dependencies = [ "hmac", "itoa", "log", - "md-5 0.11.0", + "md-5", "memchr", "rand 0.10.1", "serde", @@ -4437,7 +4428,7 @@ dependencies = [ "fnv", "fs4", "htmlescape", - "itertools", + "itertools 0.14.0", "levenshtein_automata", "log", "lru", @@ -4486,7 +4477,7 @@ checksum = "c57166f5bcfd478f370ab8445afb4678dce44801fa5ce5c451aaf8595583c5dc" dependencies = [ "downcast-rs", "fastdivide", - "itertools", + "itertools 0.14.0", "serde", "tantivy-bitpacker", "tantivy-common", @@ -4538,7 +4529,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a2cfc3ac5164cbadc28965ffb145a8f47582a60ae5897859ad8d4316596c606" dependencies = [ "futures-util", - "itertools", + "itertools 0.14.0", "tantivy-bitpacker", "tantivy-common", "tantivy-fst", @@ -4619,9 +4610,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.51" +version = "0.3.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85c17d80feb7334b40c484e45ed1a5273dfd8bfda537c3be2e74a06a6686f327" +checksum = "18dfaaeddcb932337b5e7866ee7d0ce9b76d2fd092997146f187ec09b4558a50" dependencies = [ "deranged", "num-conv", @@ -4639,9 +4630,9 @@ checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" [[package]] name = "time-macros" -version = "0.2.30" +version = "0.2.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dcef1a61bdb119096e153208ec5cbec23944ce8bca13be5c7f60c634f7403935" +checksum = "c431b87111666e491a90baa837f914fb45cd5dc3c268591b0220ff5057f2085f" dependencies = [ "num-conv", "time-core", @@ -5211,12 +5202,12 @@ checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" [[package]] name = "universal-hash" -version = "0.5.1" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" +checksum = "f4987bdc12753382e0bec4a65c50738ffaabc998b9cdd1f952fb5f39b0048a96" dependencies = [ - "crypto-common 0.1.7", - "subtle", + "crypto-common 0.2.2", + "ctutils", ] [[package]] @@ -5300,6 +5291,7 @@ dependencies = [ "getrandom 0.4.3", "js-sys", "serde_core", + "sha1_smol", "wasm-bindgen", ] @@ -5423,9 +5415,9 @@ dependencies = [ [[package]] name = "wasm-streams" -version = "0.4.2" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +checksum = "9d1ec4f6517c9e11ae630e200b2b65d193279042e28edd4a2cda233e46670bbb" dependencies = [ "futures-util", "js-sys", @@ -5475,6 +5467,12 @@ version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "998767ef88740d1f5b0682a9c53c24431453923962269c2db68ee43788c5a40d" +[[package]] +name = "widestring" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72069c3113ab32ab29e5584db3c6ec55d416895e60715417b5b883a357c3e471" + [[package]] name = "winapi" version = "0.3.9" @@ -5567,6 +5565,17 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-service" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "857224b3b211c6f3616921f081ee54721ee3ad2ace2fac6a6337e032f7b4dcf2" +dependencies = [ + "bitflags", + "widestring", + "windows-sys 0.61.2", +] + [[package]] name = "windows-strings" version = "0.5.1" @@ -5804,6 +5813,14 @@ dependencies = [ "rustix", ] +[[package]] +name = "xtask" +version = "0.1.0" +dependencies = [ + "sha2 0.11.0", + "uuid", +] + [[package]] name = "y4m" version = "0.8.0" diff --git a/Cargo.toml b/Cargo.toml index 2843d0fe..9ab00709 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace] -members = ["apps/backend"] +members = ["apps/backend", "xtask"] resolver = "2" [profile.release] diff --git a/README.md b/README.md index 7919d30b..ec28f7a3 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@

- CMS Logo + Velopulent CMS Logo

-

The CMS That Ships As a Single Binary

+

Velopulent CMS

Open-source headless CMS focused on user experience and content flexibility. @@ -10,6 +10,7 @@

Website • + DocumentationAboutFeaturesGetting Started • @@ -85,57 +86,83 @@ password after your first login.* | `/api/v1/docs` | Interactive API documentation | | `port 50051` | gRPC endpoint | | `/mcp` | MCP Streamable HTTP endpoint | +| `/health/live` | Unauthenticated process liveness probe | +| `/health/ready` | Unauthenticated database-backed readiness probe | + --- ### MCP over stdio -Run a standalone MCP process for clients that launch local stdio servers: +For clients that launch a local stdio MCP server, run `vcms mcp stdio`. It is a +**thin proxy** to a running server's `/mcp` endpoint — it opens no database, secrets, +or search index of its own. It forwards JSON-RPC between stdin/stdout and the server +over HTTP, so it keeps working even when the data is owned by a system-service account +that the client process can't read. ```bash -VCMS_MCP_TOKEN=vcms_site_... vcms mcp stdio +VCMS_MCP_TOKEN=vcms_site_... VCMS_MCP_URL=http://127.0.0.1:3000 vcms mcp stdio ``` -The command connects to an existing CMS database and never starts HTTP/gRPC -listeners, seeds users, creates a SQLite database, or runs migrations. Run -`vcms serve` first when the database schema needs initialization or migration. -MCP protocol messages use stdout; structured process logs use stderr. - -Because the process is launched by the MCP client from an arbitrary working -directory, it does **not** rely on a `.env` in the current directory. The -database path and the `HMAC_SECRET` it needs to verify the token are -read from the CMS home directory (`~/.vcms`, see [Data directory](#data-directory)) -that `vcms serve` initialized. The client only needs to supply `VCMS_MCP_TOKEN` -(and `VCMS_HOME` if you moved the home directory): +It needs only two env vars: `VCMS_MCP_TOKEN` (a `vcms_site_*` access token, forwarded +as the `Authorization: Bearer` credential) and `VCMS_MCP_URL` (the running server's +base URL, default `http://127.0.0.1:3000`; the proxy posts to `{url}/mcp`). A `vcms +serve` instance must be running. MCP protocol messages use stdout; logs use stderr. ```jsonc // Example MCP client config { "command": "vcms", "args": ["mcp", "stdio"], - "env": { "VCMS_MCP_TOKEN": "vcms_site_..." } + "env": { "VCMS_MCP_TOKEN": "vcms_site_...", "VCMS_MCP_URL": "http://127.0.0.1:3000" } } ``` ### Data directory -All runtime files live under a single home directory so a fresh install works -from any working directory. The location is `$VCMS_HOME` when set, otherwise -`~/.vcms` (resolved cross-platform — same layout on Windows, macOS, and Linux): - -```text -~/.vcms/ - config.toml # non-secret configuration (vcms config init writes here) - secrets.toml # auto-generated HMAC_SECRET + backup key (0600 on unix) - vcms.db # default SQLite database (+ -wal / -shm) - logs/ # rolling logs when [log] output = "file" - storage/ # default filesystem storage for uploads -``` +By default, runtime files go to the platform-conventional per-type directories +(resolved cross-platform via the `directories` crate): + +| File(s) | Linux | macOS | Windows | +|---------|-------|-------|---------| +| `config.toml`, `secrets.toml`, `.env` | `~/.config/vcms` | `~/Library/Application Support/vcms` | `%APPDATA%\vcms\config` | +| `vcms.db`, `storage/`, `backups/` | `~/.local/share/vcms` | `~/Library/Application Support/vcms` | `%APPDATA%\vcms\data` | +| `search/` (rebuildable index) | `~/.cache/vcms` | `~/Library/Caches/vcms` | `%LOCALAPPDATA%\vcms\cache` | +| `logs/` | `~/.local/state/vcms` | `~/Library/Application Support/vcms` | `%LOCALAPPDATA%\vcms\data` | + +Set **`$VCMS_HOME`** to keep everything under a single root instead. When several +locations exist, the first match wins: **`$VCMS_HOME`** → a legacy `~/.vcms` +(honored automatically, so upgrades don't move your data) → the split per-type +defaults above. + +The platform service installers store daemon data under one system dir — +`/var/lib/vcms` (Linux), `/Library/Application Support/vcms` (macOS), or +`C:\ProgramData\vcms` (Windows). When this directory exists, the binary uses it +automatically. Data-touching commands require an elevated shell when service-owned +permissions restrict access. + +`vcms serve` creates what it needs on first run and generates `secrets.toml` if +absent. Environment variables (`DATABASE_URL`, `HMAC_SECRET`, `STORAGE_FS_PATH`, +S3 settings, …) still override these defaults. + + + +### Installed service operations -`vcms serve` creates this directory on first run and generates `secrets.toml` if -absent. Environment variables (`DATABASE_URL`, `HMAC_SECRET`, -`STORAGE_FS_PATH`, S3 settings, …) still override these defaults. +Native packages register `vcms` with systemd, launchd, or Windows SCM, but never +start a fresh installation automatically. Validate configuration and inspect state +with `vcms doctor` and `vcms service status`, then start it through the native service +manager. +`vcms doctor` checks resolved configuration, directory access, current database +schema, listener availability, and execution identity without creating or migrating +the database. Normal package removal preserves configuration, secrets, databases, +uploads, backups, and search state. Delete the documented system data directory only +when an irreversible purge is intended. +Upgrades use package-manager semantics and keep paths/configuration stable. Back up +before upgrading: database migrations are forward-only, so downgrading requires an +explicit restore. On first start, immediately change the temporary +`admin@cms.local` / `admin` credentials. ## Why This CMS? diff --git a/apps/backend/Cargo.toml b/apps/backend/Cargo.toml index 137c47a0..6f1a095f 100644 --- a/apps/backend/Cargo.toml +++ b/apps/backend/Cargo.toml @@ -38,8 +38,10 @@ utoipa = { version = "5", features = ["axum_extras"] } utoipa-scalar = { version = "0.3", features = ["axum"] } dotenvy = "0.15" image = { version = "0.25", default-features = false, features = ["png", "jpeg", "webp", "gif", "avif"] } -object_store = { version = "0.13", features = ["aws"] } +object_store = { version = "0.14", features = ["aws"] } bytes = "1" +futures-util = "0.3" +infer = "0.19" http = "1" regex = "1" json-patch = "4" @@ -53,7 +55,7 @@ hmac = "0.13" hex = "0.4" cookie = "0.18" base64 = "0.22" -aes-gcm = "0.10" +aes-gcm = "0.11" rand = "0.10" dashmap = "6" url = "2" @@ -65,8 +67,9 @@ tonic-prost = "0.14.5" tonic-reflection = "0.14" tonic-health = "0.14" http-body-util = "0.1.3" -rmcp = { version = "1.6", features = ["server", "transport-io", "transport-streamable-http-server", "macros", "schemars"] } +rmcp = { version = "2.0", features = ["server", "transport-io", "transport-streamable-http-server", "macros", "schemars"] } tokio-util = { version = "0.7", features = ["rt"] } +tokio-stream = { version = "0.1", features = ["net"] } schemars = "1" clap = { version = "4.6.1", features = ["derive", "env"] } figment = { version = "0.10.19", features = ["toml", "env"] } @@ -77,6 +80,10 @@ tar = "0.4" croner = "3" tantivy = "0.26" +# Windows: native Service Control Manager host for `vcms service run`. +[target.'cfg(windows)'.dependencies] +windows-service = "0.8" + [build-dependencies] tonic-build = "0.14" tonic-prost-build = "0.14" @@ -84,5 +91,4 @@ tonic-prost-build = "0.14" [dev-dependencies] tempfile = "3" tokio-test = "0.4" -tokio-stream = { version = "0.1", features = ["net"] } wiremock = "0.6" diff --git a/apps/backend/src/cli.rs b/apps/backend/src/cli.rs index 5a0eff60..5eedb86f 100644 --- a/apps/backend/src/cli.rs +++ b/apps/backend/src/cli.rs @@ -7,21 +7,23 @@ use clap::{Parser, Subcommand}; #[command( name = "vcms", version, - about = "Headless CMS server", - long_about = "Headless CMS server.\n\n\ - All runtime files live under one home directory ($VCMS_HOME, default ~/.vcms): \ - config.toml, secrets.toml, the SQLite database, logs/, and storage/. \ - `vcms serve` creates it on first run and generates secrets if absent.", - after_help = "DATA DIRECTORY ($VCMS_HOME, default ~/.vcms):\n \ - config.toml non-secret config (written by `vcms config init`)\n \ - secrets.toml auto-generated HMAC + backup secrets (0600 on unix)\n \ - vcms.db default SQLite database (+ -wal / -shm)\n \ - logs/ rolling logs when [log] output = \"file\"\n \ - storage/ default filesystem storage for uploads\n\n\ + about = "Velopulent CMS", + long_about = "Velopulent CMS — self-hosted headless content management system.\n\n\ + Runtime files default to the platform's per-type directories (config, data, \ + cache, state) via the `directories` crate. Set $VCMS_HOME to keep everything \ + under one root instead. OS packages set it from their service definitions. \ + `vcms serve` creates what it needs on first run and generates secrets if absent.", + after_help = "DATA DIRECTORIES (defaults; set $VCMS_HOME for a single root):\n \ + config dir config.toml, secrets.toml (0600 on unix), .env\n \ + data dir vcms.db (+ -wal / -shm), storage/, backups/\n \ + cache dir search/ (derived Tantivy index, rebuildable)\n \ + state dir logs/ (when [log] output = \"file\")\n\n\ KEY ENVIRONMENT (env overrides config; CLI flags override env):\n \ - VCMS_HOME home directory [default: ~/.vcms]\n \ - DATABASE_URL sqlite/postgres/mysql URL [default: sqlite://~/.vcms/vcms.db]\n \ - HMAC_SECRET token-lookup HMAC key [auto-generated to secrets.toml]" + VCMS_HOME force single-root layout [default: platform split dirs]\n \ + DATABASE_URL sqlite/postgres/mysql URL [default: sqlite:///vcms.db]\n \ + HMAC_SECRET token-lookup HMAC key [auto-generated to secrets.toml]\n\n\ + DOCUMENTATION:\n \ + https://cms.velopulent.com/docs" )] pub struct Cli { /// Path to a config file (overrides the search path). @@ -33,7 +35,7 @@ pub struct Cli { pub bind: Option, /// Database URL, e.g. sqlite:path / postgres://… (overrides config + env) - /// [default: sqlite://~/.vcms/vcms.db]. + /// [default: sqlite:///vcms.db]. #[arg(long, global = true, value_name = "URL")] pub database_url: Option, @@ -69,6 +71,13 @@ pub enum Command { #[command(subcommand)] action: BackupAction, }, + /// Inspect the native vcms service. + Service { + #[command(subcommand)] + action: ServiceAction, + }, + /// Validate configuration, storage, database, ports, and service context. + Doctor, /// Restore a backup artifact (runs offline; destructive — replaces data in scope). Restore { /// Path to the backup artifact (`.cmsbak`). @@ -114,6 +123,19 @@ pub enum BackupAction { List, } +#[derive(Subcommand, Debug)] +pub enum ServiceAction { + /// Print normalized native service state and manager details. + Status, + /// Internal entry point invoked by the Windows Service Control Manager. + /// + /// Not for direct use — the SCM launches this to host the server inside a + /// Windows service. Hidden from `--help`. + #[command(hide = true)] + #[cfg(windows)] + Run, +} + #[derive(Subcommand, Debug)] pub enum McpTransport { /// Run MCP over stdin/stdout. @@ -150,7 +172,7 @@ pub enum AdminAction { #[cfg(test)] mod tests { - use super::{Cli, Command, McpTransport}; + use super::{Cli, Command, McpTransport, ServiceAction}; use clap::Parser; #[test] @@ -163,4 +185,18 @@ mod tests { }) )); } + + #[test] + fn parses_operational_commands() { + let doctor = Cli::try_parse_from(["vcms", "doctor"]).unwrap(); + assert!(matches!(doctor.command, Some(Command::Doctor))); + + let status = Cli::try_parse_from(["vcms", "service", "status"]).unwrap(); + assert!(matches!( + status.command, + Some(Command::Service { + action: ServiceAction::Status + }) + )); + } } diff --git a/apps/backend/src/config.rs b/apps/backend/src/config.rs index 60df4d41..b5865f0c 100644 --- a/apps/backend/src/config.rs +++ b/apps/backend/src/config.rs @@ -44,6 +44,8 @@ pub struct Config { // Upload limits pub max_upload_size_bytes: usize, + /// Lifetime of signed upload URLs (seconds). + pub upload_token_expiry_secs: i64, // Cookie security pub cookie_secure: bool, @@ -129,6 +131,7 @@ struct RawConfig { backup_encryption_key: Option, max_upload_size_mb: Option, + upload_token_expiry_secs: Option, #[serde(default, deserialize_with = "de_opt_lenient_bool")] cookie_secure: Option, @@ -185,8 +188,8 @@ impl Config { pub fn load(cli: &Cli) -> Result> { let mut figment = Figment::new(); - // Lowest-precedence secret layer: persisted HMAC secret from - // `~/.vcms/secrets.toml`. Read-only here (generation happens in + // Lowest-precedence secret layer: persisted HMAC secret from the config dir's + // `secrets.toml`. Read-only here (generation happens in // `secrets::ensure()` during serve/admin). Best-effort: a missing or // unreadable file just leaves the built-in defaults in play. Env vars // and CLI flags still override these. @@ -285,7 +288,8 @@ impl Config { s3_endpoint = {}\n\ s3_public_url = {}\n\n\ # Uploads\n\ - max_upload_size_mb = {}\n\n\ + max_upload_size_mb = {}\n\ + upload_token_expiry_secs = {}\n\n\ # Security / sessions\n\ cookie_secure = {}\n\ session_lifetime_hours = {}\n\ @@ -326,6 +330,7 @@ impl Config { opt_str(&self.s3_endpoint), opt_str(&self.s3_public_url), self.max_upload_size_bytes / (1024 * 1024), + self.upload_token_expiry_secs, self.cookie_secure, self.session_lifetime_hours, self.public_registration_enabled, @@ -357,15 +362,23 @@ impl RawConfig { // `secrets::ensure()` and `mcp stdio` guards on its presence, so this only // fires on a genuinely uninitialized instance. let hmac_secret = self.hmac_secret.ok_or_else(|| { - Box::new(figment::Error::from( - "No HMAC secret resolved; run `vcms serve` once to generate ~/.vcms/secrets.toml, \ - or set HMAC_SECRET" - .to_string(), - )) + Box::new(figment::Error::from(format!( + "No HMAC secret resolved; run `vcms serve` once to generate {}, or set HMAC_SECRET", + paths::secrets_file().display(), + ))) })?; let log = self.log.unwrap_or_default(); + let upload_token_expiry_secs = self + .upload_token_expiry_secs + .unwrap_or(crate::signed_upload::DEFAULT_UPLOAD_TOKEN_EXPIRY_SECS); + if upload_token_expiry_secs <= 0 { + return Err(Box::new(figment::Error::from(format!( + "upload_token_expiry_secs must be positive (got {upload_token_expiry_secs})" + )))); + } + Ok(Config { database_url: self.database_url.unwrap_or_else(paths::default_database_url), hmac_secret, @@ -391,6 +404,7 @@ impl RawConfig { backup_s3_public_url: self.backup_s3_public_url, backup_encryption_key: self.backup_encryption_key, max_upload_size_bytes: self.max_upload_size_mb.unwrap_or(50) * 1024 * 1024, + upload_token_expiry_secs, cookie_secure: self.cookie_secure.unwrap_or(false), session_lifetime_hours: self.session_lifetime_hours.unwrap_or(24), public_registration_enabled: self.public_registration_enabled.unwrap_or(false), @@ -441,8 +455,8 @@ pub fn config_search_paths() -> Vec { paths } -/// The user-config location for the config file: `~/.vcms/config.toml` -/// (or `$VCMS_HOME/config.toml`). +/// The user-config location for the config file: the platform config dir +/// (`config.toml`), or `$VCMS_HOME/config.toml` in single-dir mode. pub fn user_config_path() -> Option { Some(paths::config_file()) } @@ -474,7 +488,9 @@ pub fn default_config_toml() -> String { # s3_endpoint = \"https://s3.example.com\"\n\ # s3_public_url = \"https://cdn.example.com\"\n\n\ # --- Uploads ---\n\ - max_upload_size_mb = 50\n\n\ + max_upload_size_mb = 50\n\ + # Signed upload URL lifetime (seconds)\n\ + upload_token_expiry_secs = {default_upload_expiry}\n\n\ # --- Security / sessions ---\n\ cookie_secure = false\n\ session_lifetime_hours = 24\n\ @@ -498,7 +514,7 @@ pub fn default_config_toml() -> String { # --- Backups ---\n\ # Run the scheduled-backup poller and allow on-demand backups.\n\ backup_enabled = true\n\ - # Destination for backup artifacts: \"filesystem\" (default, under ~/.vcms/backups)\n\ + # Destination for backup artifacts: \"filesystem\" (default, the data dir's backups/)\n\ # or \"s3\". S3 credentials are secrets — set them via env (see below).\n\ backup_destination = \"filesystem\"\n\ # backup_local_path = \"./backups\"\n\ @@ -515,7 +531,7 @@ pub fn default_config_toml() -> String { # Build a local inverted index so entry search is ranked + tokenized.\n\ # When false, search falls back to a basic SQL LIKE match.\n\ search_enabled = true\n\ - # search_index_path = \"./search\" # defaults to ~/.vcms/search\n\n\ + # search_index_path = \"./search\" # defaults to the cache dir's search/\n\n\ # --- MCP ---\n\ mcp_enabled = true\n\ mcp_allowed_hosts = [{hosts}]\n\ @@ -527,6 +543,7 @@ pub fn default_config_toml() -> String { format = \"pretty\" # pretty | json\n\ annotations = false # include file + line numbers\n\ dir = \"logs\" # used when output = \"file\"\n", + default_upload_expiry = crate::signed_upload::DEFAULT_UPLOAD_TOKEN_EXPIRY_SECS, ) } diff --git a/apps/backend/src/database/pool.rs b/apps/backend/src/database/pool.rs index 04c59e27..83e5588a 100644 --- a/apps/backend/src/database/pool.rs +++ b/apps/backend/src/database/pool.rs @@ -49,6 +49,14 @@ impl DbPool { .min_connections(config.db_min_connections) .acquire_timeout(Duration::from_secs(config.db_acquire_timeout_secs)) .idle_timeout(Duration::from_secs(config.db_idle_timeout_secs)) + // Pin the session to UTC so NOW()/CURRENT_TIMESTAMP agree with the + // chrono::Utc timestamps the app writes and compares against. + .after_connect(|conn, _meta| { + Box::pin(async move { + sqlx::query("SET time_zone = '+00:00'").execute(&mut *conn).await?; + Ok(()) + }) + }) .connect(&config.database_url) .await?; Ok(DbPool::MySql(pool)) @@ -104,6 +112,15 @@ impl DbPool { } } } + + /// Cheap, read-only connectivity check used by readiness and diagnostics. + pub async fn ping(&self) -> Result<(), sqlx::Error> { + match self { + DbPool::Postgres(pool) => sqlx::query("SELECT 1").execute(pool).await.map(|_| ()), + DbPool::MySql(pool) => sqlx::query("SELECT 1").execute(pool).await.map(|_| ()), + DbPool::Sqlite(pool) => sqlx::query("SELECT 1").execute(pool).await.map(|_| ()), + } + } } async fn validate_connection_migrations(conn: &mut C, migrator: &sqlx::migrate::Migrator) -> Result<(), MigrateError> diff --git a/apps/backend/src/diagnostics.rs b/apps/backend/src/diagnostics.rs new file mode 100644 index 00000000..c1b2c1ad --- /dev/null +++ b/apps/backend/src/diagnostics.rs @@ -0,0 +1,104 @@ +use std::net::SocketAddr; +use std::path::Path; + +use crate::cli::Cli; +use crate::config::{self, Config}; +use crate::database::connect_db_without_migrations; + +pub async fn run(cli: &Cli) -> Result<(), Box> { + let mut failed = false; + let config_path = config::resolve_config_path(cli); + let loaded = Config::load(cli).map_err(|error| format!("configuration invalid: {error}"))?; + report( + "config", + config_path.as_deref().is_none_or(Path::is_file), + config_path + .as_deref() + .map(|path| path.display().to_string()) + .unwrap_or_else(|| "built-in defaults (run `vcms config init`)".to_owned()), + &mut failed, + ); + + let directories = [ + ( + "config-dir", + crate::paths::config_file().parent().map(Path::to_path_buf), + ), + ( + "data-dir", + crate::paths::default_db_path().parent().map(Path::to_path_buf), + ), + ("storage-dir", Some(crate::paths::storage_dir())), + ]; + for (name, path) in directories { + if let Some(path) = path { + let result = std::fs::metadata(&path); + let ok = result + .as_ref() + .is_ok_and(|metadata| metadata.is_dir() && !metadata.permissions().readonly()); + report(name, ok, path.display().to_string(), &mut failed); + } + } + + match connect_db_without_migrations(&loaded).await { + Ok(pool) => { + report( + "database", + pool.ping().await.is_ok(), + "reachable; schema current".to_owned(), + &mut failed, + ); + } + Err(_) if loaded.database_url.starts_with("sqlite:") && !crate::paths::default_db_path().exists() => { + report( + "database", + true, + "not initialized; first service start will create and migrate it".to_owned(), + &mut failed, + ); + } + Err(error) => report("database", false, sanitize(&error.to_string()), &mut failed), + } + check_bind("rest-bind", &loaded.bind_address, &mut failed).await; + check_bind("grpc-bind", &loaded.grpc_bind_address, &mut failed).await; + report( + "service-identity", + true, + std::env::var("USER") + .or_else(|_| std::env::var("USERNAME")) + .unwrap_or_else(|_| "unknown".to_owned()), + &mut failed, + ); + if failed { + Err("doctor found one or more failures".into()) + } else { + println!("result=healthy"); + Ok(()) + } +} + +async fn check_bind(name: &str, raw: &str, failed: &mut bool) { + match raw.parse::() { + Ok(address) => match tokio::net::TcpListener::bind(address).await { + Ok(listener) => { + drop(listener); + report(name, true, "available".to_owned(), failed); + } + Err(error) => report(name, false, format!("unavailable: {}", error.kind()), failed), + }, + Err(_) => report(name, false, "invalid socket address".to_owned(), failed), + } +} + +fn report(name: &str, ok: bool, detail: String, failed: &mut bool) { + println!("{name}={} detail={detail}", if ok { "ok" } else { "fail" }); + *failed |= !ok; +} + +fn sanitize(message: &str) -> String { + if message.len() > 240 { + format!("{}…", &message[..240]) + } else { + message.to_owned() + } +} diff --git a/apps/backend/src/grpc/interceptor.rs b/apps/backend/src/grpc/interceptor.rs index 9ac4b810..de35e06d 100644 --- a/apps/backend/src/grpc/interceptor.rs +++ b/apps/backend/src/grpc/interceptor.rs @@ -89,7 +89,7 @@ async fn validate_auth(ctx: &AuthContext, repository: &Repository) -> Result Result, config: Arc, storage_registry: Arc, - grpc_addr: SocketAddr, + listener: tokio::net::TcpListener, + shutdown: impl Future + Send + 'static, ) -> Result<(), Box> { let collection_svc = CollectionServiceImpl::new(services.collection.clone(), repository.clone()); let entry_svc = EntryServiceImpl::new(services.entry.clone(), repository.clone()); @@ -90,7 +91,7 @@ pub async fn start_grpc_server( .add_service(file_svc) .add_service(site_svc) .add_service(webhook_svc) - .serve(grpc_addr) + .serve_with_incoming_shutdown(TcpListenerStream::new(listener), shutdown) .await?; Ok(()) @@ -101,13 +102,15 @@ pub fn spawn_grpc_server( repository: Arc, config: Arc, storage_registry: Arc, - grpc_addr: SocketAddr, + listener: tokio::net::TcpListener, + shutdown: Pin + Send>>, ) -> GrpcServerFuture { Box::pin(start_grpc_server( services, repository, config, storage_registry, - grpc_addr, + listener, + shutdown, )) } diff --git a/apps/backend/src/handlers/backup_handler.rs b/apps/backend/src/handlers/backup_handler.rs index 90c8cb65..dded2a7b 100644 --- a/apps/backend/src/handlers/backup_handler.rs +++ b/apps/backend/src/handlers/backup_handler.rs @@ -679,7 +679,7 @@ pub async fn inspect_instance_backup( Ok(b) => b, Err(r) => return r, }; - inspect_response(backup.inspect_sites(&bytes), None) + inspect_response(backup.inspect_sites(bytes).await, None) } /// Inspect an uploaded backup file, list its sites, and stage the bytes so the @@ -699,7 +699,7 @@ pub async fn inspect_instance_backup_upload( }; // Validate + read the site list before staging, so a bad file is rejected // without leaving an orphan temp object. - let (manifest, sites) = match backup.inspect_sites(&bytes) { + let (manifest, sites) = match backup.inspect_sites(bytes.clone()).await { Ok(v) => v, Err(e) => return err_response(e), }; diff --git a/apps/backend/src/handlers/dashboard_handler.rs b/apps/backend/src/handlers/dashboard_handler.rs index 1f546f94..a4014468 100644 --- a/apps/backend/src/handlers/dashboard_handler.rs +++ b/apps/backend/src/handlers/dashboard_handler.rs @@ -4,6 +4,8 @@ use axum::{ response::{IntoResponse, Response}, }; +#[cfg(feature = "embed-dashboard")] +use axum::http::HeaderMap; #[cfg(feature = "embed-dashboard")] use mime_guess::from_path; @@ -15,19 +17,61 @@ use rust_embed::RustEmbed; #[folder = "../dashboard/dist"] pub struct Assets; +/// Cache policy for an embedded asset path: Vite content-hashes everything +/// under `assets/`, so those are immutable; `index.html` (and the SPA +/// fallback) must revalidate every time; the rest (favicon etc.) get a short +/// TTL. ETags (embed-time sha256) let revalidations answer with a 304. #[cfg(feature = "embed-dashboard")] -pub async fn dashboard_handler(Path(path): Path) -> Response { +fn cache_control_for(path: &str) -> &'static str { + if path.starts_with("assets/") { + "public, max-age=31536000, immutable" + } else if path == "index.html" { + "no-cache" + } else { + "public, max-age=3600" + } +} + +#[cfg(feature = "embed-dashboard")] +fn serve_embedded(path: &str, content: rust_embed::EmbeddedFile, headers: &HeaderMap) -> Response { + let etag = format!("\"{}\"", hex::encode(content.metadata.sha256_hash())); + let cache_control = cache_control_for(path); + + if headers + .get(header::IF_NONE_MATCH) + .and_then(|v| v.to_str().ok()) + .is_some_and(|inm| inm == etag) + { + return ( + StatusCode::NOT_MODIFIED, + [(header::ETAG, etag), (header::CACHE_CONTROL, cache_control.to_string())], + ) + .into_response(); + } + + let mime = from_path(path).first_or_octet_stream(); + ( + StatusCode::OK, + [ + (header::CONTENT_TYPE, mime.as_ref().to_string()), + (header::ETAG, etag), + (header::CACHE_CONTROL, cache_control.to_string()), + ], + content.data, + ) + .into_response() +} + +#[cfg(feature = "embed-dashboard")] +pub async fn dashboard_handler(Path(path): Path, headers: HeaderMap) -> Response { let requested_path = if path.is_empty() { "index.html" } else { path.as_str() }; match Assets::get(requested_path) { - Some(content) => { - let mime = from_path(requested_path).first_or_octet_stream(); - (StatusCode::OK, [(header::CONTENT_TYPE, mime.as_ref())], content.data).into_response() - } + Some(content) => serve_embedded(requested_path, content, &headers), None => { // SPA fallback if let Some(index) = Assets::get("index.html") { - (StatusCode::OK, [(header::CONTENT_TYPE, "text/html")], index.data).into_response() + serve_embedded("index.html", index, &headers) } else { StatusCode::NOT_FOUND.into_response() } @@ -35,8 +79,43 @@ pub async fn dashboard_handler(Path(path): Path) -> Response { } } +#[cfg(all(test, feature = "embed-dashboard"))] +mod tests { + use super::*; + + #[test] + fn cache_control_hashed_assets_are_immutable() { + assert_eq!( + cache_control_for("assets/index-abc123.js"), + "public, max-age=31536000, immutable" + ); + } + + #[test] + fn cache_control_index_html_revalidates() { + assert_eq!(cache_control_for("index.html"), "no-cache"); + } + + #[test] + fn cache_control_other_files_get_short_ttl() { + assert_eq!(cache_control_for("favicon.ico"), "public, max-age=3600"); + } + + #[test] + fn serve_embedded_returns_304_on_matching_etag() { + let Some(content) = Assets::get("index.html") else { + return; // no dashboard build embedded in this test run + }; + let etag = format!("\"{}\"", hex::encode(content.metadata.sha256_hash())); + let mut headers = HeaderMap::new(); + headers.insert(header::IF_NONE_MATCH, etag.parse().unwrap()); + let response = serve_embedded("index.html", content, &headers); + assert_eq!(response.status(), StatusCode::NOT_MODIFIED); + } +} + #[cfg(not(feature = "embed-dashboard"))] -pub async fn dashboard_handler(Path(_path): Path) -> Response { +pub async fn dashboard_handler(Path(_path): Path, _headers: axum::http::HeaderMap) -> Response { ( StatusCode::SERVICE_UNAVAILABLE, [(header::CONTENT_TYPE, "text/plain")], diff --git a/apps/backend/src/handlers/file_handler.rs b/apps/backend/src/handlers/file_handler.rs index 70cb0a93..83533579 100644 --- a/apps/backend/src/handlers/file_handler.rs +++ b/apps/backend/src/handlers/file_handler.rs @@ -6,7 +6,7 @@ use axum::{ response::{IntoResponse, Response}, }; use axum_extra::extract::multipart::Multipart; -use bytes::Bytes; +use futures_util::StreamExt; use serde::Deserialize; use serde_json::json; use std::sync::Arc; @@ -24,7 +24,8 @@ use crate::models::file::{BatchFileIds, FileWithUrl}; use crate::repository::Repository; use crate::repository::traits::ListFilesParams; use crate::services::Services; -use crate::services::file::UploadFileRequest; +use crate::services::file::StreamingUploadRequest; +use crate::signed_upload::{SignedUploadError, SignedUploadToken}; use crate::storage::{StorageProvider, StorageRegistry}; #[derive(Deserialize, utoipa::IntoParams)] @@ -124,12 +125,11 @@ pub async fn list_files( security(("bearer" = []), ("access_token" = [])), tag = "files" )] -#[instrument(skip(repository, services, config, ctx, multipart, storage_registry))] +#[instrument(skip(repository, services, ctx, multipart, storage_registry))] pub async fn upload_file( ctx: RequestContext, Extension(repository): Extension, Extension(services): Extension, - Extension(config): Extension, Extension(storage_registry): Extension>, mut multipart: Multipart, ) -> Response { @@ -144,76 +144,151 @@ pub async fn upload_file( .await .unwrap_or_else(|_| "filesystem".into()); - let mut file_data: Option = None; - let mut file_name: Option = None; - let mut file_content_type: Option = None; + let storage = match get_storage_for_site(&storage_provider, &storage_registry) { + Ok(s) => s, + Err(status) => return (status, Json(json!({"error": "Storage not configured"}))).into_response(), + }; + let created_by = ctx.auth.actor.user_id().map(String::from); + + // Stream the "file" field straight into storage (constant memory); size cap + // and content sniffing are enforced by the service while streaming. + let mut uploaded: Option> = None; - while let Ok(Some(mut field)) = multipart.next_field().await { + while let Ok(Some(field)) = multipart.next_field().await { let name = field.name().unwrap_or("").to_string(); - if name == "file" { - file_name = field.file_name().map(String::from); - file_content_type = field.content_type().map(String::from); + if name == "file" && uploaded.is_none() { + let file_name = sanitize_filename(field.file_name().unwrap_or("upload")); + let content_type = field + .content_type() + .map(String::from) + .unwrap_or_else(|| "application/octet-stream".into()); - // Stream the field in chunks, enforcing the size cap incrementally so an - // oversized upload is rejected without first buffering it whole. - let max = config.max_upload_size_bytes; - let mut buf: Vec = Vec::new(); - loop { + let stream = Box::pin(futures_util::stream::unfold(field, |mut field| async move { match field.chunk().await { - Ok(Some(chunk)) => { - if buf.len() + chunk.len() > max { - return ( - StatusCode::PAYLOAD_TOO_LARGE, - Json(json!({ - "error": format!("File too large. Maximum size is {}MB", max / (1024 * 1024)) - })), - ) - .into_response(); - } - buf.extend_from_slice(&chunk); - } - Ok(None) => break, - Err(e) => { - return ( - StatusCode::BAD_REQUEST, - Json(json!({"error": format!("Failed to read file: {}", e)})), - ) - .into_response(); - } + Ok(Some(chunk)) => Some((Ok(chunk), field)), + Ok(None) => None, + Err(e) => Some((Err(Box::new(e) as Box), field)), } - } - file_data = Some(Bytes::from(buf)); + })); + + uploaded = Some( + services + .file + .upload_file_streaming( + StreamingUploadRequest { + site_id: &site_id, + file_id: None, + filename: &file_name, + content_type: &content_type, + created_by: created_by.as_deref(), + storage: storage.clone(), + storage_provider: &storage_provider, + }, + stream, + ) + .await, + ); } } - let file_data = match file_data { - Some(d) => d, - None => { - return (StatusCode::BAD_REQUEST, Json(json!({"error": "No file provided"}))).into_response(); + match uploaded { + Some(Ok(file)) => (StatusCode::CREATED, Json(file)).into_response(), + Some(Err(e)) => e.into_response(), + None => (StatusCode::BAD_REQUEST, Json(json!({"error": "No file provided"}))).into_response(), + } +} + +#[utoipa::path( + put, + path = "/api/v1/files/upload/{token}", + request_body(content = Vec, description = "Raw file bytes", content_type = "application/octet-stream"), + responses( + (status = 201, description = "File uploaded", body = FileWithUrl), + (status = 400, description = "Bad request (content mismatch or unreadable body)"), + (status = 401, description = "Invalid token"), + (status = 409, description = "Upload URL already used"), + (status = 410, description = "Upload URL expired"), + (status = 413, description = "File too large"), + ), + tag = "files" +)] +#[instrument(skip(services, config, storage_registry, headers, body))] +pub async fn upload_via_signed_url( + Path(token): Path, + headers: HeaderMap, + Extension(services): Extension, + Extension(config): Extension, + Extension(storage_registry): Extension>, + body: Body, +) -> Response { + // The token is the auth: HMAC-signed by the server, time-limited, single-use. + let token = match SignedUploadToken::verify(&token, &config.hmac_secret) { + Ok(t) => t, + Err(SignedUploadError::Expired) => { + return (StatusCode::GONE, Json(json!({"error": "Upload URL expired"}))).into_response(); + } + Err(_) => { + return (StatusCode::UNAUTHORIZED, Json(json!({"error": "Invalid upload token"}))).into_response(); } }; - let file_name = sanitize_filename(&file_name.unwrap_or_else(|| "upload".into())); - let content_type = file_content_type.unwrap_or_else(|| "application/octet-stream".into()); - let created_by = ctx.auth.actor.user_id(); + // Fast reject when the client announces an oversized body upfront. + if let Some(len) = headers + .get(header::CONTENT_LENGTH) + .and_then(|v| v.to_str().ok()) + .and_then(|v| v.parse::().ok()) + && len > config.max_upload_size_bytes as u64 + { + return ( + StatusCode::PAYLOAD_TOO_LARGE, + Json(json!({ + "error": format!("File too large. Maximum size is {}MB", config.max_upload_size_bytes / (1024 * 1024)) + })), + ) + .into_response(); + } - let storage = match get_storage_for_site(&storage_provider, &storage_registry) { + // The upload's content type was fixed when the URL was minted; a + // contradicting header is a client bug worth failing loudly on. + if let Some(ct) = headers.get(header::CONTENT_TYPE).and_then(|v| v.to_str().ok()) { + let declared = ct.split(';').next().unwrap_or(ct).trim(); + if !declared.eq_ignore_ascii_case(&token.content_type) { + return ( + StatusCode::BAD_REQUEST, + Json(json!({ + "error": format!("Content-Type '{}' does not match the upload URL's '{}'", declared, token.content_type) + })), + ) + .into_response(); + } + } + + let storage = match get_storage_for_site(&token.storage_provider, &storage_registry) { Ok(s) => s, Err(status) => return (status, Json(json!({"error": "Storage not configured"}))).into_response(), }; + let file_name = sanitize_filename(&token.filename); + let stream = Box::pin( + body.into_data_stream() + .map(|r| r.map_err(|e| Box::new(e) as Box)), + ); + match services .file - .upload_file(UploadFileRequest { - site_id: &site_id, - data: file_data, - filename: &file_name, - content_type: &content_type, - created_by, - storage, - storage_provider: &storage_provider, - }) + .upload_file_streaming( + StreamingUploadRequest { + site_id: &token.site_id, + file_id: Some(&token.file_id), + filename: &file_name, + content_type: &token.content_type, + created_by: None, + storage, + storage_provider: &token.storage_provider, + }, + stream, + ) .await { Ok(file) => (StatusCode::CREATED, Json(file)).into_response(), diff --git a/apps/backend/src/lib.rs b/apps/backend/src/lib.rs index c795463d..58333589 100644 --- a/apps/backend/src/lib.rs +++ b/apps/backend/src/lib.rs @@ -3,11 +3,14 @@ pub mod cli; pub mod config; pub mod database; +pub mod diagnostics; pub mod error; pub mod graphql; pub mod mcp; pub mod paths; pub mod secrets; +pub mod server; +pub mod service; pub mod signed_upload; pub mod test_helpers; pub mod tracing; diff --git a/apps/backend/src/main.rs b/apps/backend/src/main.rs index 653c364d..ea85f893 100644 --- a/apps/backend/src/main.rs +++ b/apps/backend/src/main.rs @@ -1,22 +1,33 @@ use clap::Parser; use cms::cli::{AdminAction, BackupAction, Cli, Command, ConfigAction, McpTransport}; use cms::config::{self, Config}; -use cms::database::{connect_db_without_migrations, init_db_with_config}; -use cms::grpc::server::spawn_grpc_server; -use cms::middleware::auth::Actor; +use cms::database::init_db_with_config; use cms::repository::Repository; -use cms::router::create_router; -use cms::services::Services; -use cms::storage::{self, STORAGE_KIND_FILESYSTEM, STORAGE_KIND_S3, StorageRegistry}; use std::error::Error; -use std::net::SocketAddr; -use std::sync::Arc; -use tracing::{debug, info, warn}; +use tracing::info; #[tokio::main] async fn main() { + // Load env from the cwd `.env` (dev) first, then `$VCMS_HOME/.env`. dotenvy is + // first-wins, so real env vars and the cwd file keep precedence over the home file. + // VCMS_HOME must be a real env var — resolved here before the `.env` loads — never + // set inside that file (chicken-and-egg). dotenvy::dotenv().ok(); + let env_file = cms::paths::env_file(); + match dotenvy::from_path(&env_file) { + Ok(()) => {} + Err(e) if e.not_found() => {} // missing file is fine + // The home `.env` is optional. If VCMS_HOME points at a service-owned directory, + // a non-elevated CLI may not read it — most notably `vcms mcp stdio`, a proxy + // that needs no home file. Treat "permission denied" like "absent" and move on: + // data-touching commands still hit `paths::ensure`'s preflight. + Err(dotenvy::Error::Io(io)) if io.kind() == std::io::ErrorKind::PermissionDenied => {} + Err(e) => { + eprintln!("Error: cannot load {}: {e}", env_file.display()); + std::process::exit(1); + } + } let cli = Cli::parse(); @@ -31,10 +42,12 @@ async fn main() { import_as_new, yes, }) => run_restore(file, scope, site, *import_as_new, *yes, &cli).await, + Some(Command::Service { action }) => cms::service::run_service(action, &cli).await, + Some(Command::Doctor) => cms::diagnostics::run(&cli).await, Some(Command::Mcp { transport: McpTransport::Stdio, - }) => run_mcp_stdio(&cli).await, - Some(Command::Serve) | None => run_serve(&cli).await, + }) => run_mcp_stdio().await, + Some(Command::Serve) | None => cms::server::run(&cli, cms::server::shutdown_signal(), || {}).await, }; if let Err(e) = result { @@ -43,231 +56,24 @@ async fn main() { } } -async fn run_serve(cli: &Cli) -> Result<(), Box> { - cms::paths::ensure()?; - cms::secrets::ensure()?; - let config = Config::load(cli)?; - - let _guard = cms::tracing::init_tracing(&config); - - if let Err(e) = config.validate_security() { - return Err(format!("Invalid production security configuration: {e}").into()); - } - - let pool = init_db_with_config(&config).await?; - - let repository = Repository::new(&pool); - - seed_admin(&repository).await; - - let storage_registry = initialize_storage(&config); - // Build these once and share them: the REST router and the gRPC server use the - // same `Services` so the single-writer search index is opened only once. - let repository_arc = Arc::new(repository.clone()); - let config_arc = Arc::new(config.clone()); - let services = Services::new(repository_arc.clone(), &pool, &config); - - let backup_destination = cms::services::backup::build_backup_destination(&config) - .map_err(|e| format!("Failed to initialize backup destination: {e}"))?; - let backup_service = Arc::new(cms::services::backup::BackupService::new( - pool.clone(), - storage_registry.clone(), - backup_destination, - &config, - )); - - let app = create_router( - repository.clone(), - config.clone(), - storage_registry.clone(), - services.clone(), - backup_service.clone(), - ); - - // Reconcile backups/restore jobs left mid-flight by a previous process: any - // running/pending row at startup is orphaned (backups only run in-process). - match cms::services::backup::meta::fail_orphaned(&pool, &cms::services::backup::now_iso()).await { - Ok(n) if n > 0 => info!("Reconciled {n} interrupted backup job(s) to failed"), - Ok(_) => {} - Err(e) => tracing::error!("Failed to reconcile interrupted backups: {e}"), - } - - if config.backup_enabled { - let scheduler_service = backup_service.clone(); - tokio::spawn(async move { - cms::services::backup::scheduler::run(scheduler_service).await; - }); - info!("Backup scheduler started"); - } - - // The search indexer is the single writer/consumer: it rebuilds the index when - // empty (first run / wiped), then drains the cross-process queue forever. - if let (Some(search), Some(queue)) = (services.search.clone(), services.search_queue.clone()) { - let repo = repository_arc.clone(); - tokio::spawn(async move { - if search.is_empty() { - info!("Search index is empty; building from database..."); - match search.rebuild_all(&repo).await { - Ok(n) => info!("Search index built: {} entries", n), - Err(e) => tracing::error!("Search index build failed: {}", e), - } - } - cms::services::search::indexer::run(search, queue, repo).await; - }); - } - - let addr: SocketAddr = config.bind_address.parse().expect("Invalid BIND_ADDRESS"); - info!("Dashboard UI available at http://{}/dashboard", addr); - info!("REST API server running on http://{}", addr); - info!("GraphQL endpoint at http://{}/api/graphql", addr); - if config.mcp_enabled { - info!("MCP HTTP endpoint at http://{}/mcp", addr); - } - - let grpc_addr: SocketAddr = config.grpc_bind_address.parse().expect("Invalid GRPC_BIND_ADDRESS"); - info!("gRPC server running on {}", grpc_addr); - - let rest_handle = tokio::spawn(async move { - let listener = tokio::net::TcpListener::bind(addr) - .await - .expect("Failed to bind address"); - - axum::serve( - listener, - app.into_make_service_with_connect_info::(), - ) - .with_graceful_shutdown(shutdown_signal()) - .await - .expect("Server error"); - }); - - let grpc_handle = tokio::spawn(spawn_grpc_server( - services.clone(), - repository_arc.clone(), - config_arc.clone(), - storage_registry.clone(), - grpc_addr, - )); - - tokio::select! { - result = rest_handle => { - if let Err(e) = result { - tracing::error!("REST server error: {}", e); - } - } - result = grpc_handle => { - if let Err(e) = result { - tracing::error!("gRPC server error: {}", e); - } - } - } - - Ok(()) -} - -async fn run_mcp_stdio(cli: &Cli) -> Result<(), Box> { - // Read-only: never create the home dir, database, or secrets file here. The - // persisted secrets written by `vcms serve` are what make this process verify - // the site token with the same HMAC secret the server signed it with. - if cms::secrets::load()?.is_none() && std::env::var("HMAC_SECRET").is_err() { - return Err("No instance secrets found. Run `vcms serve` once to initialize \ - ~/.vcms (or set HMAC_SECRET) before `vcms mcp stdio`." - .into()); - } - - let config = Config::load(cli)?; - cms::tracing::init_stdio_tracing(&config); - - info!("Starting standalone MCP stdio process"); - debug!( - database_backend = ?cms::database::backend::DatabaseBackend::from_url(&config.database_url), - "MCP stdio configuration loaded" - ); - - if let Err(error) = config.validate_security() { - return Err(format!("Invalid production security configuration: {error}").into()); - } +async fn run_mcp_stdio() -> Result<(), Box> { + // A thin proxy to the running server's `/mcp` endpoint — it touches no disk + // (no home dir, database, secrets, or search index), so it works even when those + // belong to the privileged service account. It needs only a server URL and a + // `vcms_site_*` access token, forwarded as the bearer credential. + cms::tracing::init_proxy_tracing(); let token = std::env::var("VCMS_MCP_TOKEN").map_err(|_| "VCMS_MCP_TOKEN is required for `vcms mcp stdio`")?; - if token.trim().is_empty() { + let token = token.trim().to_string(); + if token.is_empty() { return Err("VCMS_MCP_TOKEN must not be empty".into()); } - let pool = connect_db_without_migrations(&config).await?; - info!("Existing CMS database schema is compatible; no migrations were run"); - - let repository = Repository::new(&pool); - let actor = cms::mcp::auth::verify_stdio_token(&token, &repository, &config.hmac_secret) - .await - .map_err(|error| format!("MCP stdio authentication failed: {}", error.message))?; - match &actor { - Actor::ApiKey(api_key) => info!( - site_id = %api_key.site_id, - permission = %api_key.permission, - "MCP stdio site token authenticated" - ), - Actor::User(_) => return Err("MCP stdio requires a CMS site access token".into()), - } - - let storage_registry = initialize_storage(&config); - let repository = Arc::new(repository); - let config = Arc::new(config); - // Read-only search: stdio can run alongside the server without contending for - // the writer lock; its content writes enqueue for the server to index. - let services = Arc::new(Services::new_read_only(repository.clone(), &pool, &config)); - let server = cms::mcp::server::CmsServer::new_stdio(services, repository, storage_registry, config, token); - - let result = cms::mcp::transports::stdio::serve(server).await; - match &result { - Ok(()) => info!("Standalone MCP stdio process exited cleanly"), - Err(error) => tracing::error!(error = %error, "Standalone MCP stdio process exiting after failure"), - } - result -} - -fn initialize_storage(config: &Config) -> Arc { - let mut storage_registry = StorageRegistry::new(); - - // Use an explicit filesystem path if set; otherwise default to ~/.vcms/storage - // so uploads work out of the box — unless S3 is configured and takes over. - let fs_path = match (&config.storage_fs_path, config.has_s3()) { - (Some(path), _) => Some(path.clone()), - (None, false) => Some(cms::paths::storage_dir().to_string_lossy().into_owned()), - (None, true) => None, - }; - - if let Some(fs_path) = fs_path { - match storage::FileSystemStorage::new(&fs_path) { - Ok(fs) => { - storage_registry.register(STORAGE_KIND_FILESYSTEM, Arc::new(fs)); - info!("Filesystem storage initialized at {}", fs_path); - } - Err(error) => warn!("Failed to initialize filesystem storage: {}", error), - } - } - - if config.has_s3() { - match storage::S3Storage::new( - config.s3_access_key_id.as_deref().unwrap(), - config.s3_secret_access_key.as_deref().unwrap(), - config.s3_bucket.as_deref().unwrap(), - config.s3_region.as_deref().unwrap_or("us-east-1"), - config.s3_endpoint.as_deref(), - config.s3_public_url.as_deref(), - ) { - Ok(s3) => { - storage_registry.register(STORAGE_KIND_S3, Arc::new(s3)); - info!("S3 storage initialized"); - } - Err(error) => warn!("Failed to initialize S3 storage: {}", error), - } - } - - if storage_registry.get(STORAGE_KIND_FILESYSTEM).is_none() && storage_registry.get(STORAGE_KIND_S3).is_none() { - warn!("No storage providers configured. Set STORAGE_FS_PATH or S3_* env vars."); - } + let base = std::env::var("VCMS_MCP_URL").unwrap_or_else(|_| "http://127.0.0.1:3000".to_string()); + let endpoint = format!("{}/mcp", base.trim_end_matches('/')); - Arc::new(storage_registry) + info!(%endpoint, "Starting MCP stdio proxy"); + cms::mcp::transports::stdio::serve(endpoint, token).await } fn run_config(action: &ConfigAction, cli: &Cli) -> Result<(), Box> { @@ -337,7 +143,7 @@ async fn run_backup(action: &BackupAction, cli: &Cli) -> Result<(), Box) -> Result Err(format!("unknown scope '{other}' (use instance|site)").into()), } } - -/// Email identity of the default admin account. Login is by email, so the seed -/// gate keys on this (non-unique display name "admin" would be the wrong column). -const ADMIN_EMAIL: &str = "admin@cms.local"; - -async fn seed_admin(repository: &Repository) { - debug!("Checking if admin user needs to be seeded"); - if !repository.user.exists(ADMIN_EMAIL).await.unwrap_or(false) { - info!("Seeding default admin user"); - let id = uuid::Uuid::now_v7().to_string(); - let password_hash = bcrypt::hash("admin", bcrypt::DEFAULT_COST).expect("Failed to hash password"); - repository - .user - .create(&id, "admin", ADMIN_EMAIL, &password_hash) - .await - .expect("Failed to seed admin user"); - repository - .user - .set_instance_role(&id, Some("instance_owner")) - .await - .expect("Failed to assign instance owner"); - repository - .user - .update_password(&id, &password_hash, true) - .await - .expect("Failed to require password change"); - - warn!("Seeded default admin user (admin@cms.local / admin) — CHANGE THE PASSWORD IMMEDIATELY!"); - eprintln!( - "\n\ - ============================ SECURITY WARNING ============================\n\ - A default admin account was created: email 'admin@cms.local' password 'admin'\n\ - Sign in with the email and password. Anyone who can reach this server can log\n\ - in until you change it. Run:\n\ - \n vcms admin reset-password --email admin@cms.local --password \n\n\ - or change it from the dashboard now. Do NOT expose this server until done.\n\ - =========================================================================\n" - ); - } else { - debug!("Admin user already exists, skipping seeding"); - } -} - -async fn shutdown_signal() { - let ctrl_c = async { - tokio::signal::ctrl_c().await.expect("failed to install Ctrl+C handler"); - }; - - #[cfg(unix)] - let terminate = async { - tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()) - .expect("failed to install signal handler") - .recv() - .await; - }; - - #[cfg(not(unix))] - let terminate = std::future::pending::<()>(); - - tokio::select! { - _ = ctrl_c => info!("Received Ctrl+C, shutting down..."), - _ = terminate => info!("Received SIGTERM, shutting down..."), - } -} diff --git a/apps/backend/src/mcp/auth.rs b/apps/backend/src/mcp/auth.rs index 3e41e443..7a4edb46 100644 --- a/apps/backend/src/mcp/auth.rs +++ b/apps/backend/src/mcp/auth.rs @@ -5,7 +5,7 @@ use axum::{ middleware::Next, response::{IntoResponse, Response}, }; -use rmcp::model::{CallToolResult, Content, ErrorCode, ErrorData}; +use rmcp::model::{CallToolResult, ContentBlock, ErrorCode, ErrorData}; use rmcp::service::{RequestContext, RoleServer}; use serde_json::json; @@ -26,11 +26,11 @@ pub fn ok_result(data: &impl serde::Serialize) -> Result) -> CallToolResult { - CallToolResult::success(vec![Content::text(message.into())]) + CallToolResult::success(vec![ContentBlock::text(message.into())]) } pub fn map_err(e: impl Into) -> ErrorData { @@ -41,7 +41,7 @@ pub fn map_err(e: impl Into) -> ErrorData /// Use this for business logic errors that the LLM can act on (not found, validation, permission). pub fn tool_error(e: impl Into) -> CallToolResult { let error = e.into(); - CallToolResult::error(vec![Content::text(error.error_message())]) + CallToolResult::error(vec![ContentBlock::text(error.error_message())]) } pub fn resolve_actor(ctx: &RequestContext) -> Result { @@ -63,12 +63,6 @@ pub fn resolve_actor(ctx: &RequestContext) -> Result Result { - verify_access_token(token, repository, hmac_secret) - .await - .map_err(|(_, Json(error))| mcp_error(ErrorCode::INVALID_REQUEST, error.message)) -} - pub async fn authenticate_mcp_request(mut request: Request, next: Next) -> Response { let repository = match request.extensions().get::>() { Some(repository) => repository.clone(), diff --git a/apps/backend/src/mcp/resources/site_schema.rs b/apps/backend/src/mcp/resources/site_schema.rs index 34100cf2..893a3967 100644 --- a/apps/backend/src/mcp/resources/site_schema.rs +++ b/apps/backend/src/mcp/resources/site_schema.rs @@ -2,8 +2,7 @@ use std::sync::Arc; use chrono::Utc; use rmcp::ErrorData as McpError; -use rmcp::model::RawResource; -use rmcp::model::{Annotated, Annotations, ListResourcesResult, ReadResourceResult, Resource, ResourceContents}; +use rmcp::model::{Annotations, ListResourcesResult, ReadResourceResult, Resource, ResourceContents}; use crate::middleware::auth::Actor; use crate::models::authorization::Action; @@ -26,19 +25,11 @@ fn resource_uri(site_id: &str, path: &str) -> String { } fn make_resource(uri: &str, name: &str, title: &str, description: &str) -> Resource { - Annotated::new( - RawResource { - uri: uri.to_string(), - name: name.to_string(), - title: Some(title.to_string()), - description: Some(description.to_string()), - mime_type: Some("application/json".to_string()), - size: None, - icons: None, - meta: None, - }, - Some(Annotations::for_resource(0.5, Utc::now())), - ) + Resource::new(uri, name) + .with_title(title) + .with_description(description) + .with_mime_type("application/json") + .with_annotations(Annotations::for_resource(0.5, Utc::now())) } fn collection_to_schema_value(c: &Collection) -> serde_json::Value { diff --git a/apps/backend/src/mcp/server.rs b/apps/backend/src/mcp/server.rs index 359f6106..23264eae 100644 --- a/apps/backend/src/mcp/server.rs +++ b/apps/backend/src/mcp/server.rs @@ -28,7 +28,6 @@ pub struct CmsServer { pub storage_registry: Arc, pub config: Arc, pub authorizer: Arc, - stdio_token: Option>, } #[tool_router] @@ -46,22 +45,9 @@ impl CmsServer { storage_registry, config, authorizer, - stdio_token: None, } } - pub fn new_stdio( - services: Arc, - repository: Arc, - storage_registry: Arc, - config: Arc, - token: String, - ) -> Self { - let mut server = Self::new(services, repository, storage_registry, config); - server.stdio_token = Some(Arc::from(token)); - server - } - #[tool(description = "Get details of a specific site by ID")] async fn get_site( &self, @@ -252,7 +238,9 @@ impl CmsServer { file::get_file(&self.authorizer, &self.services, &actor, params).await } - #[tool(description = "Create a signed upload URL for uploading a file")] + #[tool( + description = "Create a single-use signed upload URL. To upload: send an HTTP PUT to upload_url with the raw file bytes as the request body and a Content-Type header equal to content_type. The URL expires at expires_at and can be used exactly once. The PUT response body is the created file record (JSON, includes the file id and url)." + )] async fn create_upload_url( &self, ctx: RequestContext, @@ -260,7 +248,15 @@ impl CmsServer { ) -> Result { let actor = self.resolve_actor(&ctx)?; let public_base_url = self.public_base_url(&ctx); - file::create_upload_url(&self.authorizer, &self.config, &actor, public_base_url, params).await + file::create_upload_url( + &self.authorizer, + &self.services, + &self.config, + &actor, + public_base_url, + params, + ) + .await } #[tool(description = "Delete a file (soft delete)")] @@ -488,12 +484,6 @@ impl ServerHandler for CmsServer { impl CmsServer { async fn authenticate_context(&self, ctx: &mut RequestContext) -> Result { - if let Some(token) = &self.stdio_token { - let actor = crate::mcp::auth::verify_stdio_token(token, &self.repository, &self.config.hmac_secret).await?; - ctx.extensions.insert(actor.clone()); - return Ok(actor); - } - self.resolve_actor(ctx) } diff --git a/apps/backend/src/mcp/tools/file.rs b/apps/backend/src/mcp/tools/file.rs index d43a9f8a..52394de5 100644 --- a/apps/backend/src/mcp/tools/file.rs +++ b/apps/backend/src/mcp/tools/file.rs @@ -113,6 +113,7 @@ fn default_content_type() -> String { pub async fn create_upload_url( authorization: &Arc, + services: &Arc, config: &Arc, actor: &Actor, public_base_url: Option, @@ -126,11 +127,28 @@ pub async fn create_upload_url( return Ok(tool_error(e)); } - let (token, upload_path) = SignedUploadToken::generate( + // Fail fast at mint time instead of returning a URL doomed to a 400 PUT. + if !crate::utils::content_types::is_allowed(¶ms.0.content_type) { + return Ok(tool_error(crate::services::error::ServiceError::BadRequest(format!( + "Content type '{}' is not allowed. Accepted types: images, videos, audio, documents, archives", + params.0.content_type + )))); + } + + // Mint against the site's actual storage provider, not a hardcoded default. + let storage_provider = services + .file + .get_storage_provider(&site_id) + .await + .unwrap_or_else(|_| "filesystem".into()); + + let (token, upload_path) = SignedUploadToken::generate_with_storage_provider( &site_id, ¶ms.0.filename, ¶ms.0.content_type, + &storage_provider, &config.hmac_secret, + config.upload_token_expiry_secs, ); let fallback_base_url = format!("http://{}", config.bind_address); diff --git a/apps/backend/src/mcp/transports/stdio.rs b/apps/backend/src/mcp/transports/stdio.rs index 5cf831f5..9ced9474 100644 --- a/apps/backend/src/mcp/transports/stdio.rs +++ b/apps/backend/src/mcp/transports/stdio.rs @@ -1,19 +1,274 @@ -use rmcp::{ServiceExt, transport}; +//! `vcms mcp stdio` — a thin proxy between an MCP client's stdin/stdout and the +//! running server's Streamable HTTP `/mcp` endpoint. +//! +//! The stdio process runs under the *invoking* user from an arbitrary cwd, while the +//! server runs as the (often privileged) service account that owns the database, +//! secrets, and search index. Rather than open those files directly — which fails +//! when they belong to the service account — this proxy forwards JSON-RPC over HTTP +//! and lets the server do all disk I/O. It needs only a URL and a `vcms_site_*` +//! access token, which it injects as the `Authorization: Bearer` header. +//! +//! The server runs the HTTP transport in stateless, JSON-response mode (no SSE, no +//! `Mcp-Session-Id`), so a flat per-message proxy is sufficient: read a +//! newline-delimited JSON-RPC message from stdin, POST it, write the JSON reply back. -use crate::mcp::server::CmsServer; +use serde_json::Value; +use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader}; -pub async fn serve(server: CmsServer) -> Result<(), Box> { - tracing::info!("MCP stdio transport active"); - let service = server.serve(transport::stdio()).await?; +/// Run the proxy loop until stdin closes (EOF → clean exit). +/// +/// `endpoint` is the fully-qualified MCP URL (e.g. `http://127.0.0.1:3000/mcp`) and +/// `token` is the `vcms_site_*` access token forwarded as the bearer credential. +pub async fn serve(endpoint: String, token: String) -> Result<(), Box> { + let client = reqwest::Client::new(); + let mut lines = BufReader::new(tokio::io::stdin()).lines(); + let mut stdout = tokio::io::stdout(); - match service.waiting().await { - Ok(reason) => { - tracing::info!(?reason, "MCP stdio transport stopped"); - Ok(()) + tracing::info!(%endpoint, "MCP stdio proxy active"); + + while let Some(line) = lines.next_line().await? { + let message = line.trim(); + if message.is_empty() { + continue; + } + if let Some(response) = forward(&client, &endpoint, &token, message).await { + stdout.write_all(response.as_bytes()).await?; + stdout.write_all(b"\n").await?; + stdout.flush().await?; + } + } + + tracing::info!("MCP stdio proxy stopped (stdin closed)"); + Ok(()) +} + +/// Forward one JSON-RPC message to the server and return the line to write to stdout, +/// or `None` when nothing should be written (a notification, which carries no `id` +/// and expects no reply). On a transport failure for a request we synthesize a +/// JSON-RPC error so the client sees a clean failure and the proxy keeps running. +async fn forward(client: &reqwest::Client, endpoint: &str, token: &str, message: &str) -> Option { + let parsed = serde_json::from_str::(message).ok(); + // Requests and batches expect a response; a lone notification object (no `id`) + // does not. Unparseable input is forwarded so the server returns a proper + // JSON-RPC parse error. + let (needs_response, id) = match &parsed { + Some(Value::Object(obj)) => (obj.contains_key("id"), obj.get("id").cloned()), + Some(Value::Array(_)) => (true, None), // batch + _ => (true, None), // unparseable → let the server reject it + }; + + let result = client + .post(endpoint) + .header(reqwest::header::AUTHORIZATION, format!("Bearer {token}")) + .header(reqwest::header::CONTENT_TYPE, "application/json") + .header(reqwest::header::ACCEPT, "application/json, text/event-stream") + .body(message.to_string()) + .send() + .await; + + let response = match result { + Ok(response) => response, + Err(error) => { + return error_line( + needs_response, + &id, + format!("cannot reach vcms server at {endpoint}: {error}"), + ); } + }; + + let status = response.status(); + let content_type = response + .headers() + .get(reqwest::header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .unwrap_or_default() + .to_string(); + + let body = match response.text().await { + Ok(body) => body, Err(error) => { - tracing::error!(error = %error, "MCP stdio transport failed"); - Err(error.into()) + return error_line( + needs_response, + &id, + format!("reading vcms server response failed: {error}"), + ); } + }; + + if !needs_response { + return None; // notification: server replies 202 with an empty body + } + + // A non-2xx is an auth/host/transport failure (e.g. 401 for a bad token), not a + // JSON-RPC reply. Wrap it so the client sees a proper JSON-RPC error envelope. + if !status.is_success() { + let detail = http_error_detail(&body); + return error_line(needs_response, &id, format!("vcms server returned {status}: {detail}")); + } + + // json-response mode returns `application/json`; defensively unwrap a single SSE + // frame if the server ever streams one back. + let payload = if content_type.contains("text/event-stream") { + extract_sse_data(&body).unwrap_or(body) + } else { + body + }; + let payload = payload.trim(); + if payload.is_empty() { + return error_line( + needs_response, + &id, + "vcms server returned an empty response".to_string(), + ); + } + + // Re-serialize compactly so the line carries no embedded newlines (MCP stdio + // framing is one message per line). Fall back to the raw payload if it isn't JSON. + Some( + serde_json::from_str::(payload) + .map(|value| value.to_string()) + .unwrap_or_else(|_| payload.replace('\n', " ")), + ) +} + +/// Build a JSON-RPC error line for a failed request, or `None` for a notification +/// (which has no `id` to respond to). +fn error_line(needs_response: bool, id: &Option, message: String) -> Option { + if !needs_response { + tracing::warn!(%message, "MCP notification forward failed"); + return None; + } + let error = serde_json::json!({ + "jsonrpc": "2.0", + "id": id.clone().unwrap_or(Value::Null), + "error": { "code": -32603, "message": message }, + }); + Some(error.to_string()) +} + +/// Concatenate the `data:` lines of an SSE payload into a single JSON string. +fn extract_sse_data(body: &str) -> Option { + let mut data = String::new(); + for line in body.lines() { + if let Some(rest) = line.strip_prefix("data:") { + data.push_str(rest.trim_start()); + } + } + (!data.is_empty()).then_some(data) +} + +/// Pull a human-readable detail out of an error response body. The server's auth +/// failures are `{"error": "..."}`; fall back to the raw trimmed body otherwise. +fn http_error_detail(body: &str) -> String { + serde_json::from_str::(body) + .ok() + .and_then(|value| value.get("error").and_then(Value::as_str).map(str::to_string)) + .unwrap_or_else(|| body.trim().to_string()) +} + +#[cfg(test)] +mod tests { + use super::*; + use axum::Router; + use axum::body::Bytes; + use axum::http::{HeaderMap, StatusCode}; + use axum::routing::post; + use std::net::SocketAddr; + + /// A stub `/mcp` that echoes the received Authorization/Accept headers back inside + /// a JSON-RPC result (for requests) and 202-accepts notifications. + async fn spawn_stub() -> SocketAddr { + let app = Router::new().route( + "/mcp", + post(|headers: HeaderMap, body: Bytes| async move { + let header = |name: &str| { + headers + .get(name) + .and_then(|value| value.to_str().ok()) + .unwrap_or_default() + .to_string() + }; + let auth = header("authorization"); + let accept = header("accept"); + let parsed: Value = serde_json::from_slice(&body).unwrap_or(Value::Null); + let response_headers = [("content-type", "application/json")]; + match parsed.get("id") { + None => (StatusCode::ACCEPTED, response_headers, String::new()), + Some(id) => { + let reply = serde_json::json!({ + "jsonrpc": "2.0", + "id": id, + "result": { "auth": auth, "accept": accept }, + }); + (StatusCode::OK, response_headers, reply.to_string()) + } + } + }), + ); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + addr + } + + #[tokio::test] + async fn forwards_bearer_and_relays_response() { + let addr = spawn_stub().await; + let endpoint = format!("http://{addr}/mcp"); + let out = forward( + &reqwest::Client::new(), + &endpoint, + "vcms_site_test", + r#"{"jsonrpc":"2.0","id":1,"method":"tools/list"}"#, + ) + .await + .expect("a request must produce a response line"); + let value: Value = serde_json::from_str(&out).unwrap(); + assert_eq!(value["id"], 1); + assert_eq!(value["result"]["auth"], "Bearer vcms_site_test"); + assert!(value["result"]["accept"].as_str().unwrap().contains("application/json")); + } + + #[tokio::test] + async fn notification_produces_no_output() { + let addr = spawn_stub().await; + let endpoint = format!("http://{addr}/mcp"); + let out = forward( + &reqwest::Client::new(), + &endpoint, + "t", + r#"{"jsonrpc":"2.0","method":"notifications/initialized"}"#, + ) + .await; + assert!(out.is_none()); + } + + #[tokio::test] + async fn unreachable_server_yields_jsonrpc_error_for_request() { + // Port 1 refuses immediately on all supported platforms. + let out = forward( + &reqwest::Client::new(), + "http://127.0.0.1:1/mcp", + "t", + r#"{"jsonrpc":"2.0","id":7,"method":"tools/list"}"#, + ) + .await + .expect("a request must produce an error line"); + let value: Value = serde_json::from_str(&out).unwrap(); + assert_eq!(value["id"], 7); + assert_eq!(value["error"]["code"], -32603); + } + + #[tokio::test] + async fn unreachable_server_swallows_notification() { + let out = forward( + &reqwest::Client::new(), + "http://127.0.0.1:1/mcp", + "t", + r#"{"jsonrpc":"2.0","method":"ping"}"#, + ) + .await; + assert!(out.is_none()); } } diff --git a/apps/backend/src/middleware/auth.rs b/apps/backend/src/middleware/auth.rs index 1dd04b9f..0d1e51dd 100644 --- a/apps/backend/src/middleware/auth.rs +++ b/apps/backend/src/middleware/auth.rs @@ -176,6 +176,34 @@ fn extract_csrf_token(parts: &Parts) -> Option { // ── HMAC / sessions ── +/// How stale a session/token "last seen" timestamp may get before we rewrite +/// it. Skipping the unconditional per-request UPDATE keeps the auth hot path +/// read-only for most requests. +pub(crate) const TOUCH_INTERVAL_SECS: i64 = 60; + +/// Lenient parse of the backend-specific timestamp texts: RFC3339, Postgres +/// `::text` (`YYYY-MM-DD HH:MM:SS[.fff]+00`), or naive UTC (SQLite/MySQL). +pub(crate) fn parse_db_timestamp(s: &str) -> Option> { + if let Ok(dt) = chrono::DateTime::parse_from_rfc3339(s) { + return Some(dt.with_timezone(&chrono::Utc)); + } + if let Ok(dt) = chrono::DateTime::parse_from_str(s, "%Y-%m-%d %H:%M:%S%.f%#z") { + return Some(dt.with_timezone(&chrono::Utc)); + } + chrono::NaiveDateTime::parse_from_str(s, "%Y-%m-%d %H:%M:%S%.f") + .ok() + .map(|naive| naive.and_utc()) +} + +/// True when a "last seen/used" timestamp is missing, unparseable, or older +/// than [`TOUCH_INTERVAL_SECS`] — i.e. it should be rewritten (safe fallback). +pub(crate) fn needs_touch(last: Option<&str>) -> bool { + match last.and_then(parse_db_timestamp) { + Some(ts) => (chrono::Utc::now() - ts).num_seconds() > TOUCH_INTERVAL_SECS, + None => true, + } +} + pub fn compute_key_hmac(key: &str, hmac_secret: &str) -> String { let mut mac = HmacSha256::new_from_slice(hmac_secret.as_bytes()).expect("HMAC can take key of any size"); mac.update(key.as_bytes()); @@ -193,7 +221,9 @@ pub async fn verify_session( .await .map_err(|_| AuthError::unauthorized("Authentication service unavailable"))? .ok_or_else(|| AuthError::unauthorized("Invalid or expired session"))?; - let _ = repository.session.touch(&session.id).await; + if needs_touch(Some(&session.last_seen_at)) { + let _ = repository.session.touch(&session.id).await; + } Ok(UserActor { user_id: session.user_id, session_id: session.id, @@ -221,7 +251,7 @@ pub(crate) async fn verify_access_token( let token_hmac = compute_key_hmac(token, hmac_secret); - for (token_id, site_id, stored_hash, stored_hmac, expires_at, revoked_at, permission) in keys { + for (token_id, site_id, stored_hash, stored_hmac, expires_at, revoked_at, permission, last_used_at) in keys { if let Some(ref stored) = stored_hmac { if stored != &token_hmac { continue; @@ -242,7 +272,9 @@ pub(crate) async fn verify_access_token( .parse::() .map_err(|_| AuthError::unauthorized("Invalid access token"))?; - let _ = repository.access_token.update_last_used(&token_id).await; + if needs_touch(last_used_at.as_deref()) { + let _ = repository.access_token.update_last_used(&token_id).await; + } Span::current().record("site_id", tracing::field::display(&site_id)); return Ok(Actor::ApiKey(ApiKeyActor { diff --git a/apps/backend/src/paths.rs b/apps/backend/src/paths.rs index 2466a24a..ec308682 100644 --- a/apps/backend/src/paths.rs +++ b/apps/backend/src/paths.rs @@ -1,116 +1,264 @@ -//! Resolution of the CMS home directory and the runtime files within it. +//! Resolution of where the CMS keeps its runtime files. //! -//! Everything the running instance owns lives under a single root so a fresh -//! install "just works" regardless of the current working directory: +//! There are two layouts: //! -//! ```text -//! ~/.vcms/ -//! config.toml # non-secret configuration -//! secrets.toml # auto-generated HMAC + backup encryption secrets (0600) -//! vcms.db # default SQLite database (+ -wal / -shm) -//! logs/ # rolling logs when log output = "file" -//! storage/ # default filesystem storage for uploads -//! ``` +//! * **Split** (the default for an interactive install) — files land in the +//! platform-conventional per-type directories via the `directories` crate: +//! config/secrets in the config dir, the database/uploads/backups in the data dir, +//! the (rebuildable) search index in the cache dir, and logs in the state dir. +//! On Linux this is XDG (`~/.config/vcms`, `~/.local/share/vcms`, `~/.cache/vcms`, +//! `~/.local/state/vcms`); on macOS `~/Library/Application Support/vcms` (+ Caches); +//! on Windows `%APPDATA%`/`%LOCALAPPDATA%` under `vcms`. //! -//! The root is `$VCMS_HOME` when set, otherwise `~/.vcms` resolved cross-platform -//! via the `directories` crate. This mirrors the convention used by tools like -//! `CARGO_HOME` / `PGDATA`: one predictable, overridable home. +//! * **Single** — everything nests under one root. Chosen when `$VCMS_HOME` is set, +//! the system service home exists, or a legacy `~/.vcms` already holds data. use std::path::PathBuf; -/// Environment variable that overrides the home directory location. +/// Environment variable that forces the single-directory layout at a chosen root. pub const CMS_HOME_ENV: &str = "VCMS_HOME"; -/// The CMS home directory root. -/// -/// `$VCMS_HOME` wins if set and non-empty. Otherwise `~/.vcms`. As a last resort -/// (no detectable home directory) falls back to `.vcms` in the current dir. -pub fn home() -> PathBuf { - if let Some(value) = std::env::var_os(CMS_HOME_ENV) - && !value.is_empty() +/// Where each class of file lives. Resolved fresh on each call (cheap) so tests and +/// the service can change `$VCMS_HOME` without a process restart. +enum Layout { + /// One root holding everything (`$VCMS_HOME`, a legacy `~/.vcms`, or a `.vcms` + /// fallback when no platform dirs are detectable). + Single(PathBuf), + /// Per-type platform directories. + Split { + config: PathBuf, + data: PathBuf, + cache: PathBuf, + state: PathBuf, + }, +} + +impl Layout { + fn config(&self) -> PathBuf { + match self { + Layout::Single(root) => root.clone(), + Layout::Split { config, .. } => config.clone(), + } + } + + fn data(&self) -> PathBuf { + match self { + Layout::Single(root) => root.clone(), + Layout::Split { data, .. } => data.clone(), + } + } + + fn cache(&self) -> PathBuf { + match self { + Layout::Single(root) => root.clone(), + Layout::Split { cache, .. } => cache.clone(), + } + } + + fn state(&self) -> PathBuf { + match self { + Layout::Single(root) => root.clone(), + Layout::Split { state, .. } => state.clone(), + } + } +} + +/// Conventional system directory an installed OS service owns, per platform. The app +/// does not auto-select this path; service packages pass it through `$VCMS_HOME`. +pub fn system_home() -> Option { + #[cfg(windows)] + { + Some(PathBuf::from(r"C:\ProgramData\vcms")) + } + #[cfg(target_os = "linux")] + { + Some(PathBuf::from("/var/lib/vcms")) + } + #[cfg(target_os = "macos")] + { + Some(PathBuf::from("/Library/Application Support/vcms")) + } + #[cfg(not(any(windows, target_os = "linux", target_os = "macos")))] { - return PathBuf::from(value); + None } +} + +/// Resolve the active layout: `$VCMS_HOME` → system home → legacy `~/.vcms` → platform split. +/// Gathers the real filesystem/env inputs and defers the precedence decision to the +/// pure [`resolve_layout`] (kept separate so it is testable without touching real +/// system dirs). +fn layout() -> Layout { + let env_home = std::env::var_os(CMS_HOME_ENV) + .filter(|v| !v.is_empty()) + .map(PathBuf::from); + let system = system_home().filter(|path| path.is_dir()); + + // Back-compat: a pre-existing single `~/.vcms` keeps owning everything so an upgrade + // never strands a user's database or secrets. + let legacy = directories::BaseDirs::new() + .map(|base| base.home_dir().join(".vcms")) + .filter(|p| p.is_dir()); + + let split = directories::ProjectDirs::from("", "", "vcms").map(|dirs| split_from(&dirs)); + + resolve_layout(env_home, system, legacy, split) +} + +/// Pure precedence policy. Each `Option` is a candidate the caller has already vetted +/// (env set & non-empty; legacy dir confirmed to exist; split from `ProjectDirs`). +/// Falls back to a local `.vcms` blob when no platform dirs are detectable (rare). +fn resolve_layout( + env_home: Option, + system_home: Option, + legacy_home: Option, + split: Option, +) -> Layout { + if let Some(root) = env_home { + return Layout::Single(root); + } + if let Some(root) = system_home { + return Layout::Single(root); + } + if let Some(root) = legacy_home { + return Layout::Single(root); + } + split.unwrap_or_else(|| Layout::Single(PathBuf::from(".vcms"))) +} - directories::BaseDirs::new() - .map(|dirs| dirs.home_dir().join(".vcms")) - .unwrap_or_else(|| PathBuf::from(".vcms")) +/// Map `ProjectDirs` onto our four directory classes. Logs use the state dir where +/// the platform has one (Linux), else the local data dir (macOS/Windows). +fn split_from(dirs: &directories::ProjectDirs) -> Layout { + Layout::Split { + config: dirs.config_dir().to_path_buf(), + data: dirs.data_dir().to_path_buf(), + cache: dirs.cache_dir().to_path_buf(), + state: dirs.state_dir().unwrap_or_else(|| dirs.data_local_dir()).to_path_buf(), + } } -/// `~/.vcms/config.toml` — the user-level config file. +/// `config.toml` — the non-secret config file (config dir). pub fn config_file() -> PathBuf { - home().join("config.toml") + layout().config().join("config.toml") } -/// `~/.vcms/secrets.toml` — auto-generated secrets file. +/// `secrets.toml` — auto-generated secrets file (config dir, 0600 on unix). pub fn secrets_file() -> PathBuf { - home().join("secrets.toml") + layout().config().join("secrets.toml") } -/// `~/.vcms/vcms.db` — the default SQLite database file. -pub fn default_db_path() -> PathBuf { - home().join("vcms.db") +/// `.env` — optional environment file loaded at startup (config dir). +pub fn env_file() -> PathBuf { + layout().config().join(".env") } -/// `~/.vcms/logs` — directory for rolling log files. -pub fn logs_dir() -> PathBuf { - home().join("logs") +/// `vcms.db` — the default SQLite database file (data dir). +pub fn default_db_path() -> PathBuf { + layout().data().join("vcms.db") } -/// `~/.vcms/storage` — default filesystem storage directory for uploads. +/// `storage/` — default filesystem storage directory for uploads (data dir). pub fn storage_dir() -> PathBuf { - home().join("storage") + layout().data().join("storage") } -/// `~/.vcms/backups` — default local destination for backup artifacts. +/// `backups/` — default local destination for backup artifacts (data dir). pub fn backups_dir() -> PathBuf { - home().join("backups") + layout().data().join("backups") } -/// `~/.vcms/search` — default location for the Tantivy full-text search index. +/// `search/` — default location for the derived Tantivy index (cache dir). pub fn search_dir() -> PathBuf { - home().join("search") + layout().cache().join("search") +} + +/// `logs/` — directory for rolling log files (state dir). +pub fn logs_dir() -> PathBuf { + layout().state().join("logs") } -/// Build the default `DATABASE_URL` (`sqlite:///vcms.db`). +/// Build the default `DATABASE_URL` (`sqlite:///vcms.db`). /// /// SQLite URLs use forward slashes, so backslashes are normalized for Windows. pub fn default_database_url() -> String { format!("sqlite://{}", default_db_path().to_string_lossy().replace('\\', "/")) } -/// Create the home directory and its subdirectories (`logs/`, `storage/`). -/// -/// Called by commands that own/initialize the instance (`serve`, `admin`). -/// Read-only commands such as `mcp stdio` must not create anything. +/// Fail fast when the active root is the conventional system service home but this +/// process can't access it (e.g. non-elevated CLI against the ACL-locked +/// `C:\ProgramData\vcms` or root-owned `/var/lib/vcms`). +fn preflight_system_home() -> std::io::Result<()> { + let Layout::Single(root) = layout() else { + return Ok(()); + }; + if system_home().as_deref() != Some(root.as_path()) || !root.is_dir() { + return Ok(()); + } + // A read probe: opening the db and secrets both need list+read on this dir. + match std::fs::read_dir(&root) { + Ok(_) => Ok(()), + Err(e) if e.kind() == std::io::ErrorKind::PermissionDenied => Err(std::io::Error::new( + std::io::ErrorKind::PermissionDenied, + format!( + "vcms data at {} requires Administrator/root; re-run in an elevated terminal.", + root.display() + ), + )), + Err(e) => Err(e), + } +} + +/// Create the directories the running instance writes to (config, data, storage, +/// logs). Called by commands that own/initialize the instance (`serve`, `admin`, +/// `backup`, `restore`); read-only commands such as `mcp stdio`, `config path/show` +/// never call this and so never create anything or trip the preflight below. pub fn ensure() -> std::io::Result<()> { - let root = home(); - std::fs::create_dir_all(&root)?; - std::fs::create_dir_all(logs_dir())?; - std::fs::create_dir_all(storage_dir())?; + preflight_system_home()?; + let layout = layout(); + for dir in [ + layout.config(), + layout.data(), + layout.state().join("logs"), + layout.data().join("storage"), + ] { + std::fs::create_dir_all(dir)?; + } Ok(()) } +/// Whether a file sits inside the home root in single-dir mode. Only meaningful for +/// `$VCMS_HOME`/legacy installs; in split mode there is no single root. +pub fn single_root() -> Option { + match layout() { + Layout::Single(root) => Some(root), + Layout::Split { .. } => None, + } +} + #[cfg(test)] mod tests { use super::*; use crate::test_helpers::with_home; + use std::path::Path; #[test] - fn cms_home_env_overrides_root() { + fn vcms_home_forces_single_layout() { let dir = tempfile::tempdir().expect("temp dir"); with_home(dir.path(), || { - assert_eq!(home(), dir.path()); + assert_eq!(single_root().as_deref(), Some(dir.path())); assert_eq!(config_file(), dir.path().join("config.toml")); assert_eq!(secrets_file(), dir.path().join("secrets.toml")); + assert_eq!(env_file(), dir.path().join(".env")); assert_eq!(default_db_path(), dir.path().join("vcms.db")); - assert_eq!(logs_dir(), dir.path().join("logs")); assert_eq!(storage_dir(), dir.path().join("storage")); + assert_eq!(backups_dir(), dir.path().join("backups")); + assert_eq!(search_dir(), dir.path().join("search")); + assert_eq!(logs_dir(), dir.path().join("logs")); }); } #[test] - fn ensure_creates_subdirectories() { + fn ensure_creates_subdirectories_in_single_mode() { let dir = tempfile::tempdir().expect("temp dir"); let root = dir.path().join("home"); with_home(&root, || { @@ -130,4 +278,88 @@ mod tests { assert!(!url.contains('\\')); }); } + + #[test] + fn split_layout_separates_file_classes() { + // Drive the pure mapping directly so the assertion is platform-independent + // (real ProjectDirs vary by OS and ambient env). + let layout = Layout::Split { + config: PathBuf::from("/cfg"), + data: PathBuf::from("/dat"), + cache: PathBuf::from("/cch"), + state: PathBuf::from("/st"), + }; + assert!(layout.config().join("config.toml").starts_with(Path::new("/cfg"))); + assert!(layout.data().join("vcms.db").starts_with(Path::new("/dat"))); + assert!(layout.cache().join("search").starts_with(Path::new("/cch"))); + assert!(layout.state().join("logs").starts_with(Path::new("/st"))); + // config / data / cache / state are genuinely distinct in split mode. + assert_ne!(layout.config(), layout.data()); + assert_ne!(layout.data(), layout.cache()); + assert_ne!(layout.cache(), layout.state()); + } + + #[test] + fn split_state_falls_back_to_local_data_when_absent() { + // When a platform lacks a state dir, logs must still resolve (we feed the + // local-data dir into `state`). Modeled here as state == data. + let layout = Layout::Split { + config: PathBuf::from("/cfg"), + data: PathBuf::from("/dat"), + cache: PathBuf::from("/cch"), + state: PathBuf::from("/dat"), + }; + assert!(layout.state().join("logs").starts_with(Path::new("/dat"))); + } + + fn split_sample() -> Option { + Some(Layout::Split { + config: PathBuf::from("/c"), + data: PathBuf::from("/d"), + cache: PathBuf::from("/ch"), + state: PathBuf::from("/s"), + }) + } + + fn single_root_of(layout: Layout) -> Option { + match layout { + Layout::Single(root) => Some(root), + Layout::Split { .. } => None, + } + } + + #[test] + fn resolve_env_home_outranks_everything() { + let out = resolve_layout( + Some(PathBuf::from("/env")), + Some(PathBuf::from("/system")), + Some(PathBuf::from("/legacy")), + split_sample(), + ); + assert_eq!(single_root_of(out), Some(PathBuf::from("/env"))); + } + + #[test] + fn resolve_system_home_when_present() { + let out = resolve_layout(None, Some(PathBuf::from("/system")), None, split_sample()); + assert_eq!(single_root_of(out), Some(PathBuf::from("/system"))); + } + + #[test] + fn resolve_legacy_home_when_no_env_or_system() { + let out = resolve_layout(None, None, Some(PathBuf::from("/legacy")), split_sample()); + assert_eq!(single_root_of(out), Some(PathBuf::from("/legacy"))); + } + + #[test] + fn resolve_falls_through_to_split() { + let out = resolve_layout(None, None, None, split_sample()); + assert!(matches!(out, Layout::Split { .. })); + } + + #[test] + fn resolve_local_blob_when_nothing_detectable() { + let out = resolve_layout(None, None, None, None); + assert_eq!(single_root_of(out), Some(PathBuf::from(".vcms"))); + } } diff --git a/apps/backend/src/repository/mysql/access_token.rs b/apps/backend/src/repository/mysql/access_token.rs index 6c2dc25f..5df706f0 100644 --- a/apps/backend/src/repository/mysql/access_token.rs +++ b/apps/backend/src/repository/mysql/access_token.rs @@ -71,7 +71,7 @@ impl AccessTokenRepository for MysqlAccessTokenRepository { async fn find_by_prefix(&self, prefix: &str) -> Result, RepositoryError> { let rows = sqlx::query_as::<_, AccessTokenLookupRow>( - "SELECT id, site_id, token_hash, token_hmac, CAST(expires_at AS CHAR) AS expires_at, CAST(revoked_at AS CHAR) AS revoked_at, permission + "SELECT id, site_id, token_hash, token_hmac, CAST(expires_at AS CHAR) AS expires_at, CAST(revoked_at AS CHAR) AS revoked_at, permission, CAST(last_used_at AS CHAR) AS last_used_at FROM access_tokens WHERE token_prefix = ?", ) .bind(prefix) diff --git a/apps/backend/src/repository/mysql/collection.rs b/apps/backend/src/repository/mysql/collection.rs index affed4a6..13ef3847 100644 --- a/apps/backend/src/repository/mysql/collection.rs +++ b/apps/backend/src/repository/mysql/collection.rs @@ -125,6 +125,9 @@ impl CollectionRepository for MysqlCollectionRepository { entry_items: &[Entry], rename_map: &std::collections::HashMap, ) -> Result<(), RepositoryError> { + // One transaction for the whole migration: per-statement commit overhead + // dominated this loop, and a partial rename is never left behind. + let mut tx = self.pool.begin().await?; for entry in entry_items { if let Ok(mut data) = serde_json::from_str::(&entry.data) && let Some(obj) = data.as_object_mut() @@ -140,10 +143,11 @@ impl CollectionRepository for MysqlCollectionRepository { sqlx::query("UPDATE entries SET data = ?, updated_at = NOW() WHERE id = ?") .bind(&new_data_str) .bind(&entry.id) - .execute(&self.pool) + .execute(&mut *tx) .await?; } } + tx.commit().await?; Ok(()) } } diff --git a/apps/backend/src/repository/mysql/entry.rs b/apps/backend/src/repository/mysql/entry.rs index 5e158d6f..bb0222d8 100644 --- a/apps/backend/src/repository/mysql/entry.rs +++ b/apps/backend/src/repository/mysql/entry.rs @@ -334,16 +334,22 @@ impl EntryRepository for MysqlEntryRepository { .execute(&mut *tx) .await?; - for file_id in &file_ids { - sqlx::query( - "INSERT IGNORE INTO entry_file_references (entry_id, file_id, site_id) SELECT ?, id, ? FROM files WHERE id = ? AND site_id = ?", - ) - .bind(entry_id) - .bind(site_id) - .bind(file_id) - .bind(site_id) - .execute(&mut *tx) - .await?; + if !file_ids.is_empty() { + // One multi-row statement instead of a query per file id; the + // SELECT keeps the "file must exist in this site" filter. + let placeholders = vec!["?"; file_ids.len()].join(", "); + let sql = format!( + "INSERT IGNORE INTO entry_file_references (entry_id, file_id, site_id) + SELECT ?, id, ? FROM files WHERE site_id = ? AND id IN ({placeholders})", + ); + let mut query = sqlx::query(sqlx::AssertSqlSafe(sql.as_str())) + .bind(entry_id) + .bind(site_id) + .bind(site_id); + for file_id in &file_ids { + query = query.bind(file_id); + } + query.execute(&mut *tx).await?; } tx.commit().await?; diff --git a/apps/backend/src/repository/mysql/file.rs b/apps/backend/src/repository/mysql/file.rs index 4d1807dd..47caa3f5 100644 --- a/apps/backend/src/repository/mysql/file.rs +++ b/apps/backend/src/repository/mysql/file.rs @@ -329,4 +329,21 @@ impl FileRepository for MysqlFileRepository { Ok(provider.unwrap_or_else(|| "filesystem".into())) } + + async fn set_thumbnail_meta( + &self, + id: &str, + thumbnail_key: &str, + width: Option, + height: Option, + ) -> Result<(), RepositoryError> { + sqlx::query("UPDATE files SET thumbnail_key = ?, width = ?, height = ? WHERE id = ?") + .bind(thumbnail_key) + .bind(width) + .bind(height) + .bind(id) + .execute(&self.pool) + .await?; + Ok(()) + } } diff --git a/apps/backend/src/repository/postgres/access_token.rs b/apps/backend/src/repository/postgres/access_token.rs index 025dda33..72409a6a 100644 --- a/apps/backend/src/repository/postgres/access_token.rs +++ b/apps/backend/src/repository/postgres/access_token.rs @@ -71,7 +71,7 @@ impl AccessTokenRepository for PostgresAccessTokenRepository { async fn find_by_prefix(&self, prefix: &str) -> Result, RepositoryError> { let rows = sqlx::query_as::<_, AccessTokenLookupRow>( - "SELECT id, site_id, token_hash, token_hmac, expires_at::text, revoked_at::text, permission + "SELECT id, site_id, token_hash, token_hmac, expires_at::text, revoked_at::text, permission, last_used_at::text FROM access_tokens WHERE token_prefix = $1", ) .bind(prefix) diff --git a/apps/backend/src/repository/postgres/collection.rs b/apps/backend/src/repository/postgres/collection.rs index 1fbc607c..11544168 100644 --- a/apps/backend/src/repository/postgres/collection.rs +++ b/apps/backend/src/repository/postgres/collection.rs @@ -127,6 +127,9 @@ impl CollectionRepository for PostgresCollectionRepository { entry_items: &[Entry], rename_map: &std::collections::HashMap, ) -> Result<(), RepositoryError> { + // One transaction for the whole migration: per-statement commit overhead + // dominated this loop, and a partial rename is never left behind. + let mut tx = self.pool.begin().await?; for entry in entry_items { if let Ok(mut data) = serde_json::from_str::(&entry.data) && let Some(obj) = data.as_object_mut() @@ -142,10 +145,11 @@ impl CollectionRepository for PostgresCollectionRepository { sqlx::query("UPDATE entries SET data = $1::jsonb, updated_at = NOW() WHERE id = $2") .bind(&new_data_str) .bind(&entry.id) - .execute(&self.pool) + .execute(&mut *tx) .await?; } } + tx.commit().await?; Ok(()) } } diff --git a/apps/backend/src/repository/postgres/entry.rs b/apps/backend/src/repository/postgres/entry.rs index a8cb9280..cbeaca43 100644 --- a/apps/backend/src/repository/postgres/entry.rs +++ b/apps/backend/src/repository/postgres/entry.rs @@ -346,14 +346,18 @@ impl EntryRepository for PostgresEntryRepository { .execute(&mut *tx) .await?; - for file_id in &file_ids { + if !file_ids.is_empty() { + // One multi-row statement instead of a query per file id; the + // SELECT keeps the "file must exist in this site" filter. sqlx::query( - "INSERT INTO entry_file_references (entry_id, file_id, site_id) SELECT $1, id, $2 FROM files WHERE id = $3 AND site_id = $4 ON CONFLICT DO NOTHING", + "INSERT INTO entry_file_references (entry_id, file_id, site_id) + SELECT $1, id, $2 FROM files WHERE site_id = $3 AND id = ANY($4) + ON CONFLICT DO NOTHING", ) .bind(entry_id) .bind(site_id) - .bind(file_id) .bind(site_id) + .bind(&file_ids) .execute(&mut *tx) .await?; } diff --git a/apps/backend/src/repository/postgres/file.rs b/apps/backend/src/repository/postgres/file.rs index 3061b8b0..ecef75f5 100644 --- a/apps/backend/src/repository/postgres/file.rs +++ b/apps/backend/src/repository/postgres/file.rs @@ -344,4 +344,21 @@ impl FileRepository for PostgresFileRepository { Ok(provider.unwrap_or_else(|| "filesystem".into())) } + + async fn set_thumbnail_meta( + &self, + id: &str, + thumbnail_key: &str, + width: Option, + height: Option, + ) -> Result<(), RepositoryError> { + sqlx::query("UPDATE files SET thumbnail_key = $1, width = $2, height = $3 WHERE id = $4") + .bind(thumbnail_key) + .bind(width) + .bind(height) + .bind(id) + .execute(&self.pool) + .await?; + Ok(()) + } } diff --git a/apps/backend/src/repository/sqlite/access_token.rs b/apps/backend/src/repository/sqlite/access_token.rs index c169b5f5..5164c0e8 100644 --- a/apps/backend/src/repository/sqlite/access_token.rs +++ b/apps/backend/src/repository/sqlite/access_token.rs @@ -71,7 +71,7 @@ impl AccessTokenRepository for SqliteAccessTokenRepository { async fn find_by_prefix(&self, prefix: &str) -> Result, RepositoryError> { let rows = sqlx::query_as::<_, AccessTokenLookupRow>( - "SELECT id, site_id, token_hash, token_hmac, expires_at, revoked_at, permission + "SELECT id, site_id, token_hash, token_hmac, expires_at, revoked_at, permission, last_used_at FROM access_tokens WHERE token_prefix = ?", ) .bind(prefix) diff --git a/apps/backend/src/repository/sqlite/collection.rs b/apps/backend/src/repository/sqlite/collection.rs index 436a790d..147ed49d 100644 --- a/apps/backend/src/repository/sqlite/collection.rs +++ b/apps/backend/src/repository/sqlite/collection.rs @@ -127,6 +127,9 @@ impl CollectionRepository for SqliteCollectionRepository { entry_items: &[Entry], rename_map: &std::collections::HashMap, ) -> Result<(), RepositoryError> { + // One transaction for the whole migration: per-statement commit overhead + // dominated this loop, and a partial rename is never left behind. + let mut tx = self.pool.begin().await?; for entry in entry_items { if let Ok(mut data) = serde_json::from_str::(&entry.data) && let Some(obj) = data.as_object_mut() @@ -142,10 +145,11 @@ impl CollectionRepository for SqliteCollectionRepository { sqlx::query("UPDATE entries SET data = ?, updated_at = datetime('now') WHERE id = ?") .bind(&new_data_str) .bind(&entry.id) - .execute(&self.pool) + .execute(&mut *tx) .await?; } } + tx.commit().await?; Ok(()) } } diff --git a/apps/backend/src/repository/sqlite/entry.rs b/apps/backend/src/repository/sqlite/entry.rs index dcdd3c1c..3d709416 100644 --- a/apps/backend/src/repository/sqlite/entry.rs +++ b/apps/backend/src/repository/sqlite/entry.rs @@ -376,16 +376,22 @@ impl EntryRepository for SqliteEntryRepository { .execute(&mut *tx) .await?; - for file_id in &file_ids { - sqlx::query( - "INSERT OR IGNORE INTO entry_file_references (entry_id, file_id, site_id) SELECT ?, id, ? FROM files WHERE id = ? AND site_id = ?", - ) - .bind(entry_id) - .bind(site_id) - .bind(file_id) - .bind(site_id) - .execute(&mut *tx) - .await?; + if !file_ids.is_empty() { + // One multi-row statement instead of a query per file id; the + // SELECT keeps the "file must exist in this site" filter. + let placeholders = vec!["?"; file_ids.len()].join(", "); + let sql = format!( + "INSERT OR IGNORE INTO entry_file_references (entry_id, file_id, site_id) + SELECT ?, id, ? FROM files WHERE site_id = ? AND id IN ({placeholders})", + ); + let mut query = sqlx::query(sqlx::AssertSqlSafe(sql.as_str())) + .bind(entry_id) + .bind(site_id) + .bind(site_id); + for file_id in &file_ids { + query = query.bind(file_id); + } + query.execute(&mut *tx).await?; } tx.commit().await?; diff --git a/apps/backend/src/repository/sqlite/file.rs b/apps/backend/src/repository/sqlite/file.rs index c8a84704..542974ef 100644 --- a/apps/backend/src/repository/sqlite/file.rs +++ b/apps/backend/src/repository/sqlite/file.rs @@ -339,4 +339,21 @@ impl FileRepository for SqliteFileRepository { Ok(provider.unwrap_or_else(|| "filesystem".into())) } + + async fn set_thumbnail_meta( + &self, + id: &str, + thumbnail_key: &str, + width: Option, + height: Option, + ) -> Result<(), RepositoryError> { + sqlx::query("UPDATE files SET thumbnail_key = ?, width = ?, height = ? WHERE id = ?") + .bind(thumbnail_key) + .bind(width) + .bind(height) + .bind(id) + .execute(&self.pool) + .await?; + Ok(()) + } } diff --git a/apps/backend/src/repository/traits.rs b/apps/backend/src/repository/traits.rs index f0aa425f..7d9a6676 100644 --- a/apps/backend/src/repository/traits.rs +++ b/apps/backend/src/repository/traits.rs @@ -268,16 +268,25 @@ pub trait FileRepository: Send + Sync { site_id: &str, ) -> Result, RepositoryError>; async fn get_storage_provider(&self, site_id: &str) -> Result; + /// Attach thumbnail metadata generated after the upload response (background task). + async fn set_thumbnail_meta( + &self, + id: &str, + thumbnail_key: &str, + width: Option, + height: Option, + ) -> Result<(), RepositoryError>; } pub type AccessTokenLookupRow = ( - String, - String, - String, - Option, - Option, - Option, - String, + String, // id + String, // site_id + String, // token_hash + Option, // token_hmac + Option, // expires_at + Option, // revoked_at + String, // permission + Option, // last_used_at (for the touch debounce) ); /// A new access token to persist (see [`AccessTokenRepository::create`]). diff --git a/apps/backend/src/router/dashboard.rs b/apps/backend/src/router/dashboard.rs index 58739107..06e680b9 100644 --- a/apps/backend/src/router/dashboard.rs +++ b/apps/backend/src/router/dashboard.rs @@ -6,14 +6,18 @@ pub fn dashboard_routes() -> Router { Router::new() .route( "/dashboard", - get(|| async { dashboard_handler(axum::extract::Path("".into())).await }), + get(|headers: axum::http::HeaderMap| async move { + dashboard_handler(axum::extract::Path("".into()), headers).await + }), ) // Bare `/dashboard/` (trailing slash) matches neither `/dashboard` nor the // `{*file}` wildcard (which needs ≥1 segment); serve the SPA shell here too so a // refresh on a client route URL ending in `/` still loads the app. .route( "/dashboard/", - get(|| async { dashboard_handler(axum::extract::Path("".into())).await }), + get(|headers: axum::http::HeaderMap| async move { + dashboard_handler(axum::extract::Path("".into()), headers).await + }), ) .route("/dashboard/{*file}", get(dashboard_handler)) } diff --git a/apps/backend/src/router/files.rs b/apps/backend/src/router/files.rs index 8ad48968..257fac30 100644 --- a/apps/backend/src/router/files.rs +++ b/apps/backend/src/router/files.rs @@ -1,13 +1,14 @@ use axum::extract::DefaultBodyLimit; use axum::{ Router, - routing::{delete, get, post}, + routing::{delete, get, post, put}, }; use tower_http::limit::RequestBodyLimitLayer; use crate::handlers::file_handler::{ batch_delete_files, batch_permanent_delete_files, batch_restore_files, delete_file_handler, get_file, get_file_references, list_files, restore_file, serve_file, serve_file_thumbnail, upload_file, + upload_via_signed_url, }; /// Public API CRUD routes (mounted at /api/v1) @@ -36,6 +37,16 @@ pub fn file_serve_routes() -> Router { .route("/api/files/{id}/thumbnail", get(serve_file_thumbnail)) } +/// Signed-URL upload — standalone, no auth middleware: the HMAC token in the +/// path is the credential (minted by the MCP `create_upload_url` tool). +/// Registered at the literal path the tool advertises. +pub fn signed_upload_routes(max_upload_bytes: usize) -> Router { + Router::new() + .route("/api/v1/files/upload/{token}", put(upload_via_signed_url)) + .layer(DefaultBodyLimit::disable()) + .layer(RequestBodyLimitLayer::new(max_upload_bytes)) +} + /// Dashboard routes (mounted under /api/dashboard/sites/{site_id}) pub fn dashboard_routes(max_upload_bytes: usize) -> Router { Router::new() diff --git a/apps/backend/src/router/health.rs b/apps/backend/src/router/health.rs new file mode 100644 index 00000000..6b8962b6 --- /dev/null +++ b/apps/backend/src/router/health.rs @@ -0,0 +1,45 @@ +use axum::extract::State; +use axum::http::StatusCode; +use axum::routing::get; +use axum::{Json, Router}; +use serde::Serialize; + +use crate::database::pool::DbPool; + +#[derive(Clone)] +pub struct HealthState { + pool: DbPool, +} + +impl HealthState { + pub fn new(pool: DbPool) -> Self { + Self { pool } + } +} + +#[derive(Serialize)] +struct HealthResponse { + status: &'static str, +} + +pub fn routes(pool: DbPool) -> Router { + Router::new() + .route("/health/live", get(live)) + .route("/health/ready", get(ready)) + .with_state(HealthState::new(pool)) +} + +async fn live() -> Json { + Json(HealthResponse { status: "ok" }) +} + +async fn ready(State(state): State) -> (StatusCode, Json) { + if state.pool.ping().await.is_ok() { + (StatusCode::OK, Json(HealthResponse { status: "ready" })) + } else { + ( + StatusCode::SERVICE_UNAVAILABLE, + Json(HealthResponse { status: "unavailable" }), + ) + } +} diff --git a/apps/backend/src/router/mod.rs b/apps/backend/src/router/mod.rs index 9bc212fd..2322fe1b 100644 --- a/apps/backend/src/router/mod.rs +++ b/apps/backend/src/router/mod.rs @@ -7,6 +7,7 @@ mod docs; mod entry; mod files; mod graphql; +mod health; mod instance; mod mcp; mod openapi; @@ -24,6 +25,7 @@ use tower_http::cors::{AllowOrigin, CorsLayer}; use tower_http::trace::TraceLayer; use crate::config::Config; +use crate::database::pool::DbPool; use crate::handlers::site_handler::get_current_site; use crate::middleware::api_auth::api_auth_middleware; use crate::middleware::authz::authz_middleware; @@ -76,6 +78,7 @@ fn dashboard_site_v1_routes(max_upload_bytes: usize) -> Router { } pub fn create_router( + pool: DbPool, repository: Repository, config: Config, storage_registry: Arc, @@ -130,6 +133,8 @@ pub fn create_router( }; let mut router = Router::new() + // Public, minimal operational probes. Detailed failures remain in logs/doctor. + .merge(health::routes(pool)) // ── Auth (no middleware) ── .merge(auth::auth_routes()) // ── Public API (/api/v1/*) ── @@ -142,6 +147,8 @@ pub fn create_router( ) // ── File serving (no auth — file IDs are effectively opaque) ── .merge(files::file_serve_routes()) + // ── Signed-URL upload (no auth — the HMAC token is the credential) ── + .merge(files::signed_upload_routes(max_upload_bytes)) // ── Dashboard API (/api/dashboard/*) ── .nest( "/api/dashboard", diff --git a/apps/backend/src/router/openapi.rs b/apps/backend/src/router/openapi.rs index df9fda4f..bbc16f0f 100644 --- a/apps/backend/src/router/openapi.rs +++ b/apps/backend/src/router/openapi.rs @@ -11,7 +11,7 @@ use crate::models::site::Site; title = "Velopulent CMS REST API", version = "0.1.0", description = "Headless CMS unified API. Consumer access uses site-bound vcms_site_* tokens with read or write permission. Dashboard access uses revocable opaque sessions.", - contact(name = "Velopulent CMS", url = "https://cms.velopulent.com"), + contact(name = "Velopulent CMS", url = "https://cms.velopulent.com/docs"), license(name = "AGPL-3.0", url = "https://github.com/velopulent/cms/blob/main/LICENSE"), ), paths( @@ -41,6 +41,7 @@ use crate::models::site::Site; // Public API: Files crate::handlers::file_handler::list_files, crate::handlers::file_handler::upload_file, + crate::handlers::file_handler::upload_via_signed_url, crate::handlers::file_handler::get_file, crate::handlers::file_handler::delete_file_handler, crate::handlers::file_handler::get_file_references, diff --git a/apps/backend/src/secrets.rs b/apps/backend/src/secrets.rs index e3800a3b..18e6e16d 100644 --- a/apps/backend/src/secrets.rs +++ b/apps/backend/src/secrets.rs @@ -1,4 +1,4 @@ -//! Persisted instance secrets (`~/.vcms/secrets.toml`). +//! Persisted instance secrets (`secrets.toml` in the config dir). //! //! On first `serve`/`admin`, a random `HMAC_SECRET` value is //! generated and written to `secrets.toml` (perms `0600` on unix). Every later diff --git a/apps/backend/src/server.rs b/apps/backend/src/server.rs new file mode 100644 index 00000000..3ffdbd2d --- /dev/null +++ b/apps/backend/src/server.rs @@ -0,0 +1,332 @@ +//! Server bootstrap shared by `vcms serve` and the platform service runners. +//! +//! The full startup sequence (home dir, secrets, config, migrations, REST + gRPC) +//! lives here rather than in `main.rs` so that the Windows Service Control Manager +//! entry point — which lives in the library — can host the exact same server. The +//! caller injects a *shutdown future*; whatever resolves it (Ctrl+C / SIGTERM on a +//! normal run, an SCM stop control on Windows) triggers one graceful drain of both +//! the REST and gRPC servers. + +use std::error::Error; +use std::future::Future; +use std::net::SocketAddr; +use std::sync::Arc; + +use tracing::{debug, info, warn}; + +use crate::cli::Cli; +use crate::config::Config; +use crate::database::init_db_with_config; +use crate::grpc::server::spawn_grpc_server; +use crate::repository::Repository; +use crate::router::create_router; +use crate::services::Services; +use crate::storage::{self, STORAGE_KIND_FILESYSTEM, STORAGE_KIND_S3, StorageRegistry}; + +/// Email identity of the default admin account. Login is by email, so the seed +/// gate keys on this (non-unique display name "admin" would be the wrong column). +const ADMIN_EMAIL: &str = "admin@cms.local"; + +/// Boot the full server and run until `shutdown` resolves. +/// +/// Shared by the foreground `serve` command and the OS service runners. The +/// `shutdown` future is the single trigger that gracefully drains both the REST +/// and gRPC listeners. `on_ready` fires once startup has actually succeeded — +/// database open + migrated and both listeners bound — so a service host (the +/// Windows SCM runner) can report `Running` truthfully instead of optimistically. +pub async fn run( + cli: &Cli, + shutdown: impl Future + Send + 'static, + on_ready: impl FnOnce(), +) -> Result<(), Box> { + // Each early step gets a context prefix: a bare io error ("Access is denied. + // (os error 5)") from a service host is undebuggable without knowing which + // file/step produced it. + crate::paths::ensure().map_err(|e| format!("preparing data directories: {e}"))?; + crate::secrets::ensure().map_err(|e| format!("initializing secrets.toml: {e}"))?; + let config = Config::load(cli).map_err(|e| format!("loading configuration: {e}"))?; + + let _guard = crate::tracing::init_tracing(&config); + + if let Err(e) = config.validate_security() { + return Err(format!("Invalid production security configuration: {e}").into()); + } + + let pool = init_db_with_config(&config) + .await + .map_err(|e| format!("opening database: {e}"))?; + + let repository = Repository::new(&pool); + + seed_admin(&repository).await; + + let storage_registry = initialize_storage(&config); + // Build these once and share them: the REST router and the gRPC server use the + // same `Services` so the single-writer search index is opened only once. + let repository_arc = Arc::new(repository.clone()); + let config_arc = Arc::new(config.clone()); + let services = Services::new(repository_arc.clone(), &pool, &config); + + let backup_destination = crate::services::backup::build_backup_destination(&config) + .map_err(|e| format!("Failed to initialize backup destination: {e}"))?; + let backup_service = Arc::new(crate::services::backup::BackupService::new( + pool.clone(), + storage_registry.clone(), + backup_destination, + &config, + )); + + let app = create_router( + pool.clone(), + repository.clone(), + config.clone(), + storage_registry.clone(), + services.clone(), + backup_service.clone(), + ); + + // Reconcile backups/restore jobs left mid-flight by a previous process: any + // running/pending row at startup is orphaned (backups only run in-process). + match crate::services::backup::meta::fail_orphaned(&pool, &crate::services::backup::now_iso()).await { + Ok(n) if n > 0 => info!("Reconciled {n} interrupted backup job(s) to failed"), + Ok(_) => {} + Err(e) => tracing::error!("Failed to reconcile interrupted backups: {e}"), + } + + if config.backup_enabled { + let scheduler_service = backup_service.clone(); + tokio::spawn(async move { + crate::services::backup::scheduler::run(scheduler_service).await; + }); + info!("Backup scheduler started"); + } + + // The search indexer is the single writer/consumer: it rebuilds the index when + // empty (first run / wiped), then drains the cross-process queue forever. + if let (Some(search), Some(queue)) = (services.search.clone(), services.search_queue.clone()) { + let repo = repository_arc.clone(); + tokio::spawn(async move { + if search.is_empty() { + info!("Search index is empty; building from database..."); + match search.rebuild_all(&repo).await { + Ok(n) => info!("Search index built: {} entries", n), + Err(e) => tracing::error!("Search index build failed: {}", e), + } + } + crate::services::search::indexer::run(search, queue, repo).await; + }); + } + + let addr: SocketAddr = config + .bind_address + .parse() + .map_err(|e| format!("Invalid BIND_ADDRESS '{}': {e}", config.bind_address))?; + info!("Dashboard UI available at http://{}/dashboard", addr); + info!("REST API server running on http://{}", addr); + info!("GraphQL endpoint at http://{}/api/graphql", addr); + if config.mcp_enabled { + info!("MCP HTTP endpoint at http://{}/mcp", addr); + } + + let grpc_addr: SocketAddr = config + .grpc_bind_address + .parse() + .map_err(|e| format!("Invalid GRPC_BIND_ADDRESS '{}': {e}", config.grpc_bind_address))?; + info!("gRPC server running on {}", grpc_addr); + + // Bind both listeners *before* declaring readiness (and before the serve loops + // spawn): a bind failure — the classic "port already taken" — must surface as a + // startup error, not as a background task that dies after we claimed to be up. + let rest_listener = tokio::net::TcpListener::bind(addr) + .await + .map_err(|e| format!("Failed to bind REST address {addr}: {e}"))?; + let grpc_listener = tokio::net::TcpListener::bind(grpc_addr) + .await + .map_err(|e| format!("Failed to bind gRPC address {grpc_addr}: {e}"))?; + + on_ready(); + + // One shutdown signal fans out to both listeners via a watch channel: the + // injected `shutdown` future flips it, and both servers drain on the change. + let (shutdown_tx, shutdown_rx) = tokio::sync::watch::channel(false); + let shutdown_tx2 = shutdown_tx.clone(); + tokio::spawn(async move { + shutdown.await; + let _ = shutdown_tx.send(true); + }); + + let rest_rx = shutdown_rx.clone(); + let mut rest_handle = tokio::spawn(async move { + axum::serve( + rest_listener, + app.into_make_service_with_connect_info::(), + ) + .with_graceful_shutdown(wait_for_shutdown(rest_rx)) + .await + .map_err(|e| { + tracing::error!("REST server error: {e}"); + e + }) + }); + + let mut grpc_handle = tokio::spawn(spawn_grpc_server( + services.clone(), + repository_arc.clone(), + config_arc.clone(), + storage_registry.clone(), + grpc_listener, + Box::pin(wait_for_shutdown(shutdown_rx)), + )); + + // Whichever server finishes first flips the shutdown signal; the sibling is + // then awaited so both are fully drained before `run()` returns. + let (rest_result, grpc_result) = tokio::select! { + result = &mut rest_handle => { + let _ = shutdown_tx2.send(true); + let grpc_result = grpc_handle.await; + (result, grpc_result) + } + result = &mut grpc_handle => { + let _ = shutdown_tx2.send(true); + let rest_result = rest_handle.await; + (rest_result, result) + } + }; + + match rest_result { + Ok(Ok(())) => {} + Ok(Err(e)) => return Err(e.into()), + Err(e) => return Err(e.into()), + } + match grpc_result { + Ok(Ok(())) => {} + // gRPC's error is `Box`; format it so the + // conversion to this fn's `Box` is unambiguous. + Ok(Err(e)) => return Err(format!("gRPC server error: {e}").into()), + Err(e) => return Err(format!("gRPC server task panicked: {e}").into()), + } + + Ok(()) +} + +/// Resolve once the watch channel reports a shutdown was requested. +async fn wait_for_shutdown(mut rx: tokio::sync::watch::Receiver) { + if *rx.borrow_and_update() { + return; + } + let _ = rx.changed().await; +} + +/// Wait for a Ctrl+C or (on unix) SIGTERM — the foreground `serve` shutdown trigger. +pub async fn shutdown_signal() { + let ctrl_c = async { + tokio::signal::ctrl_c().await.expect("failed to install Ctrl+C handler"); + }; + + #[cfg(unix)] + let terminate = async { + tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()) + .expect("failed to install signal handler") + .recv() + .await; + }; + + #[cfg(not(unix))] + let terminate = std::future::pending::<()>(); + + tokio::select! { + _ = ctrl_c => info!("Received Ctrl+C, shutting down..."), + _ = terminate => info!("Received SIGTERM, shutting down..."), + } +} + +/// Register the configured storage backends (filesystem and/or S3). +pub fn initialize_storage(config: &Config) -> Arc { + let mut storage_registry = StorageRegistry::new(); + + // Use an explicit filesystem path if set; otherwise default to the data dir's + // storage/ so uploads work out of the box — unless S3 is configured and takes over. + let fs_path = match (&config.storage_fs_path, config.has_s3()) { + (Some(path), _) => Some(path.clone()), + (None, false) => Some(crate::paths::storage_dir().to_string_lossy().into_owned()), + (None, true) => None, + }; + + if let Some(fs_path) = fs_path { + match storage::FileSystemStorage::new(&fs_path) { + Ok(fs) => { + storage_registry.register(STORAGE_KIND_FILESYSTEM, Arc::new(fs)); + info!("Filesystem storage initialized at {}", fs_path); + } + Err(error) => warn!("Failed to initialize filesystem storage: {}", error), + } + } + + if config.has_s3() { + match storage::S3Storage::new( + config.s3_access_key_id.as_deref().unwrap(), + config.s3_secret_access_key.as_deref().unwrap(), + config.s3_bucket.as_deref().unwrap(), + config.s3_region.as_deref().unwrap_or("us-east-1"), + config.s3_endpoint.as_deref(), + config.s3_public_url.as_deref(), + ) { + Ok(s3) => { + storage_registry.register(STORAGE_KIND_S3, Arc::new(s3)); + info!("S3 storage initialized"); + } + Err(error) => warn!("Failed to initialize S3 storage: {}", error), + } + } + + if storage_registry.get(STORAGE_KIND_FILESYSTEM).is_none() && storage_registry.get(STORAGE_KIND_S3).is_none() { + warn!("No storage providers configured. Set STORAGE_FS_PATH or S3_* env vars."); + } + + Arc::new(storage_registry) +} + +async fn seed_admin(repository: &Repository) { + debug!("Checking if admin user needs to be seeded"); + let exists = match repository.user.exists(ADMIN_EMAIL).await { + Ok(exists) => exists, + Err(e) => { + tracing::error!("Failed to check for existing admin user; skipping seed: {e}"); + return; + } + }; + if !exists { + info!("Seeding default admin user"); + let id = uuid::Uuid::now_v7().to_string(); + let password_hash = bcrypt::hash("admin", bcrypt::DEFAULT_COST).expect("Failed to hash password"); + repository + .user + .create(&id, "admin", ADMIN_EMAIL, &password_hash) + .await + .expect("Failed to seed admin user"); + repository + .user + .set_instance_role(&id, Some("instance_owner")) + .await + .expect("Failed to assign instance owner"); + repository + .user + .update_password(&id, &password_hash, true) + .await + .expect("Failed to require password change"); + + warn!("Seeded default admin user (admin@cms.local / admin) — CHANGE THE PASSWORD IMMEDIATELY!"); + eprintln!( + "\n\ + ============================ SECURITY WARNING ============================\n\ + A default admin account was created: email 'admin@cms.local' password 'admin'\n\ + Sign in with the email and password. Anyone who can reach this server can log\n\ + in until you change it. Run:\n\ + \n vcms admin reset-password --email admin@cms.local --password \n\n\ + or change it from the dashboard now. Do NOT expose this server until done.\n\ + =========================================================================\n" + ); + } else { + debug!("Admin user already exists, skipping seeding"); + } +} diff --git a/apps/backend/src/service/mod.rs b/apps/backend/src/service/mod.rs new file mode 100644 index 00000000..6f77067b --- /dev/null +++ b/apps/backend/src/service/mod.rs @@ -0,0 +1,20 @@ +//! Native service hosting and read-only service inspection. +//! +//! Registration and lifecycle mutations remain owned by OS installers/tools. + +use crate::cli::{Cli, ServiceAction}; + +pub const SERVICE_NAME: &str = "vcms"; +pub const SERVICE_DISPLAY_NAME: &str = "Velopulent CMS"; + +mod status; +#[cfg(windows)] +mod windows; + +pub async fn run_service(action: &ServiceAction, _cli: &Cli) -> Result<(), Box> { + match action { + ServiceAction::Status => status::print(), + #[cfg(windows)] + ServiceAction::Run => windows::dispatch(action, _cli), + } +} diff --git a/apps/backend/src/service/status.rs b/apps/backend/src/service/status.rs new file mode 100644 index 00000000..96bd74af --- /dev/null +++ b/apps/backend/src/service/status.rs @@ -0,0 +1,91 @@ +use std::process::Command; + +use super::SERVICE_NAME; + +pub fn print() -> Result<(), Box> { + let (manager, mut command) = native_command(); + let output = command + .output() + .map_err(|error| format!("cannot query {manager}: {error}"))?; + let stdout = String::from_utf8_lossy(&output.stdout).trim().to_owned(); + let stderr = String::from_utf8_lossy(&output.stderr).trim().to_owned(); + let state = normalized_state(output.status.success(), &stdout, &stderr); + println!("service={SERVICE_NAME}"); + println!("manager={manager}"); + println!("state={state}"); + if !stdout.is_empty() { + println!("details={}", stdout.replace(['\r', '\n'], " ")); + } else if !stderr.is_empty() { + println!("details={}", stderr.replace(['\r', '\n'], " ")); + } + if state == "unknown" { + return Err(format!("{manager} could not determine vcms service state").into()); + } + Ok(()) +} + +fn normalized_state(success: bool, stdout: &str, stderr: &str) -> &'static str { + let text = format!("{stdout} {stderr}").to_ascii_lowercase(); + if text.contains("running") || text.trim() == "active" { + "running" + } else if text.contains("stopped") || text.contains("inactive") || text.contains("not running") { + "stopped" + } else if text.contains("could not be found") || text.contains("not-found") || text.contains("no such process") { + "not-installed" + } else if success { + "installed" + } else { + "unknown" + } +} + +#[cfg(target_os = "linux")] +fn native_command() -> (&'static str, Command) { + let mut command = Command::new("systemctl"); + command.args([ + "show", + SERVICE_NAME, + "--property=ActiveState,SubState,LoadState", + "--no-pager", + ]); + ("systemd", command) +} + +#[cfg(target_os = "macos")] +fn native_command() -> (&'static str, Command) { + let mut command = Command::new("launchctl"); + command.args(["print", "system/com.velopulent.vcms"]); + ("launchd", command) +} + +#[cfg(windows)] +fn native_command() -> (&'static str, Command) { + let mut command = Command::new("sc.exe"); + command.args(["query", SERVICE_NAME]); + ("scm", command) +} + +#[cfg(not(any(target_os = "linux", target_os = "macos", windows)))] +fn native_command() -> (&'static str, Command) { + let mut command = Command::new("false"); + command.arg(SERVICE_NAME); + ("unsupported", command) +} + +#[cfg(test)] +mod tests { + use super::normalized_state; + + #[test] + fn normalizes_common_states() { + assert_eq!( + normalized_state(true, "ActiveState=active\nSubState=running", ""), + "running" + ); + assert_eq!(normalized_state(false, "ActiveState=inactive", ""), "stopped"); + assert_eq!( + normalized_state(false, "", "Unit vcms.service could not be found"), + "not-installed" + ); + } +} diff --git a/apps/backend/src/service/windows.rs b/apps/backend/src/service/windows.rs new file mode 100644 index 00000000..be2c563b --- /dev/null +++ b/apps/backend/src/service/windows.rs @@ -0,0 +1,175 @@ +//! Windows Service Control Manager host for `vcms service run`. +//! +//! The SCM launches this hidden entry point (registered with +//! `launch_arguments = ["service", "run"]`): it hands the thread to the SCM +//! dispatcher, hosts the shared server on a fresh tokio runtime, and translates a +//! Stop/Shutdown control into the server's graceful-shutdown signal. +//! +//! Service *registration* (CreateService, dir + ACL hardening) lives in the +//! Windows installer (.msi), not here. + +use std::ffi::OsString; +use std::sync::Arc; +use std::time::Duration; + +use windows_service::service::{ + ServiceControl, ServiceControlAccept, ServiceExitCode, ServiceState, ServiceStatus, ServiceType, +}; +use windows_service::service_control_handler::{self, ServiceControlHandlerResult}; +use windows_service::{define_windows_service, service_dispatcher}; + +use super::SERVICE_NAME; +use crate::cli::{Cli, ServiceAction}; + +/// Fixed home for the LocalSystem service (Windows convention is ProgramData). +/// The service host sets this before runtime startup so direct SCM launches always +/// use the machine-wide data directory. +fn service_home() -> std::path::PathBuf { + crate::paths::system_home().expect("system_home is always set on Windows") +} + +pub fn dispatch(action: &ServiceAction, _cli: &Cli) -> Result<(), Box> { + match action { + ServiceAction::Run => run_dispatcher(), + ServiceAction::Status => unreachable!("status is dispatched by service::run_service"), + } +} + +define_windows_service!(ffi_service_main, service_main); + +/// Entry point for `vcms service run`: hand the thread to the SCM dispatcher. +fn run_dispatcher() -> Result<(), Box> { + // Pin the service to the ProgramData home so it doesn't fall back to the + // LocalSystem profile. Set before any thread/runtime reads the environment. + if std::env::var_os(crate::paths::CMS_HOME_ENV).is_none() { + // SAFETY: called once at process start, before tokio/threads spin up. + unsafe { + std::env::set_var(crate::paths::CMS_HOME_ENV, service_home()); + } + } + // A service has no console, so the default stdout logging goes nowhere. Default + // to file output (lands in the home's logs/ — where install tells the user to + // look); an explicit LOG_OUTPUT (env or .env) still wins. + service_dispatcher::start(SERVICE_NAME, ffi_service_main)?; + Ok(()) +} + +fn service_main(_arguments: Vec) { + if let Err(e) = run_service_session() { + // No console exists here — stderr goes nowhere — so failures land in a file. + log_service_error(&format!("vcms service error: {e}")); + } +} + +/// Append a fatal service error to `\logs\service-error.log`. Deliberately +/// plain `std::fs` (no tracing): it must work even when startup failed before — +/// or because — tracing/config initialization broke. Best-effort: a service can't +/// report its reporter failing. +fn log_service_error(msg: &str) { + use std::io::Write; + let _ = std::process::Command::new("eventcreate.exe") + .args(["/T", "ERROR", "/ID", "1", "/L", "APPLICATION", "/SO", "vcms", "/D", msg]) + .status(); + let dir = service_home().join("logs"); + let _ = std::fs::create_dir_all(&dir); + if let Ok(mut file) = std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(dir.join("service-error.log")) + { + let _ = writeln!(file, "[{}] {msg}", crate::services::backup::now_iso()); + } +} + +fn run_service_session() -> Result<(), Box> { + use std::sync::Mutex; + use tokio::sync::Notify; + + // A Stop/Shutdown control fires this; the server's injected shutdown future + // awaits it. `notify_one` stores a permit, so an early stop is not lost. + let notify = Arc::new(Notify::new()); + let handler_notify = notify.clone(); + + // The handle is created by register() but needed inside the handler closure to + // publish StopPending. Store it behind Mutex so the closure can grab it. + let status_handle_cell: Arc>> = + Arc::new(Mutex::new(None)); + let handle_cell = status_handle_cell.clone(); + + let status_handle = service_control_handler::register(SERVICE_NAME, move |control| match control { + ServiceControl::Stop | ServiceControl::Shutdown => { + if let Ok(guard) = handle_cell.lock() { + if let Some(h) = guard.as_ref() { + let _ = h.set_service_status(ServiceStatus { + service_type: ServiceType::OWN_PROCESS, + current_state: ServiceState::StopPending, + controls_accepted: ServiceControlAccept::empty(), + exit_code: ServiceExitCode::Win32(0), + checkpoint: 1, + wait_hint: Duration::from_secs(30), + process_id: None, + }); + } + } + handler_notify.notify_one(); + ServiceControlHandlerResult::NoError + } + ServiceControl::Interrogate => ServiceControlHandlerResult::NoError, + _ => ServiceControlHandlerResult::NotImplemented, + })?; + *status_handle_cell.lock().map_err(|e| format!("{e}"))? = Some(status_handle); + + // Report StartPending while the server actually boots (home preflight, secrets, + // config, DB open + migrations, port binds). Running is published only from the + // readiness hook below — setting Running up front would hide a boot failure. + let start_pending = ServiceStatus { + service_type: ServiceType::OWN_PROCESS, + current_state: ServiceState::StartPending, + controls_accepted: ServiceControlAccept::empty(), + exit_code: ServiceExitCode::Win32(0), + checkpoint: 0, + wait_hint: Duration::from_secs(60), + process_id: None, + }; + status_handle.set_service_status(start_pending)?; + + let ready_handle = status_handle; + let on_ready = move || { + let _ = ready_handle.set_service_status(ServiceStatus { + service_type: ServiceType::OWN_PROCESS, + current_state: ServiceState::Running, + controls_accepted: ServiceControlAccept::STOP | ServiceControlAccept::SHUTDOWN, + exit_code: ServiceExitCode::Win32(0), + checkpoint: 0, + wait_hint: Duration::default(), + process_id: None, + }); + }; + + // This thread is owned by the SCM dispatcher (not a tokio worker), so building + // a fresh runtime here is safe — no nested-runtime panic. + let runtime = tokio::runtime::Builder::new_multi_thread().enable_all().build()?; + let shutdown = async move { notify.notified().await }; + let result = runtime.block_on(crate::server::run(&Cli::default(), shutdown, on_ready)); + + if let Err(e) = &result { + log_service_error(&format!("server startup/run failed: {e}")); + } + let stopped = ServiceStatus { + service_type: ServiceType::OWN_PROCESS, + current_state: ServiceState::Stopped, + controls_accepted: ServiceControlAccept::empty(), + // ServiceSpecific(1) marks "vcms itself failed" in the SCM/Event Log; + // details are in logs\service-error.log. + exit_code: if result.is_ok() { + ServiceExitCode::Win32(0) + } else { + ServiceExitCode::ServiceSpecific(1) + }, + checkpoint: 0, + wait_hint: Duration::default(), + process_id: None, + }; + status_handle.set_service_status(stopped)?; + result +} diff --git a/apps/backend/src/services/backup/mod.rs b/apps/backend/src/services/backup/mod.rs index 7f24e0bd..051bfb9d 100644 --- a/apps/backend/src/services/backup/mod.rs +++ b/apps/backend/src/services/backup/mod.rs @@ -230,11 +230,19 @@ impl BackupService { files: file_manifest, }; - let tar_bytes = build_tar(&manifest, &tar_tables, &file_blobs)?; - let compressed = - zstd::stream::encode_all(&tar_bytes[..], self.zstd_level).map_err(|e| BackupError::Io(e.to_string()))?; - let outer = self.wrap(compressed, encrypt)?; - Ok((manifest, outer)) + // tar + zstd + AES-GCM are CPU-bound; keep them off the async runtime + // so a large backup doesn't starve request handling. + let zstd_level = self.zstd_level; + let key = self.encryption_key; + tokio::task::spawn_blocking(move || { + let tar_bytes = build_tar(&manifest, &tar_tables, &file_blobs)?; + let compressed = + zstd::stream::encode_all(&tar_bytes[..], zstd_level).map_err(|e| BackupError::Io(e.to_string()))?; + let outer = Self::wrap_bytes(key, compressed, encrypt)?; + Ok((manifest, outer)) + }) + .await + .map_err(|e| BackupError::Io(format!("backup task failed: {e}")))? } /// Run a backup: build the artifact, write it to the destination, and record @@ -378,16 +386,16 @@ impl BackupService { // --- Restore --- /// Inspect a backup artifact's manifest (decrypting/decompressing as needed). - pub fn inspect(&self, bytes: &[u8]) -> Result { - let (manifest, _, _) = self.open(bytes)?; + pub async fn inspect(&self, bytes: Vec) -> Result { + let (manifest, _, _) = self.open_blocking(bytes).await?; Ok(manifest) } /// Inspect a backup artifact and list the sites it contains, so a caller can /// offer a pick-list. An instance backup yields every site in its `sites` /// table; a site backup yields its single site. - pub fn inspect_sites(&self, bytes: &[u8]) -> Result<(Manifest, Vec), BackupError> { - let (manifest, tables_ndjson, _) = self.open(bytes)?; + pub async fn inspect_sites(&self, bytes: Vec) -> Result<(Manifest, Vec), BackupError> { + let (manifest, tables_ndjson, _) = self.open_blocking(bytes).await?; let sites = match tables_ndjson.get("sites") { Some(ndjson) => parse_ndjson(ndjson)? .iter() @@ -442,7 +450,7 @@ impl BackupService { RestoreSource::Destination(key) => self.read_destination(key).await?, }; - let (manifest, tables_ndjson, file_blobs) = self.open(&bytes)?; + let (manifest, tables_ndjson, file_blobs) = self.open_blocking(bytes).await?; if manifest.format_version > FORMAT_VERSION { return Err(BackupError::Invalid(format!( @@ -583,18 +591,22 @@ impl BackupService { // --- Artifact wrapping --- - fn wrap(&self, compressed: Vec, encrypt: bool) -> Result, BackupError> { + fn wrap_bytes( + encryption_key: Option<[u8; 32]>, + compressed: Vec, + encrypt: bool, + ) -> Result, BackupError> { let mut out = Vec::with_capacity(compressed.len() + 32); out.extend_from_slice(MAGIC); if encrypt { - let key = self.encryption_key.ok_or(BackupError::MissingKey)?; + let key = encryption_key.ok_or(BackupError::MissingKey)?; out.push(FLAG_ENCRYPTED); - let cipher = Aes256Gcm::new(Key::::from_slice(&key)); + let cipher = Aes256Gcm::new(&Key::::from(key)); let mut nonce_bytes = [0u8; 12]; rand::rng().fill_bytes(&mut nonce_bytes); - let nonce = Nonce::from_slice(&nonce_bytes); + let nonce = Nonce::from(nonce_bytes); let ciphertext = cipher - .encrypt(nonce, compressed.as_ref()) + .encrypt(&nonce, compressed.as_ref()) .map_err(|e| BackupError::Crypto(e.to_string()))?; out.extend_from_slice(&nonce_bytes); out.extend_from_slice(&ciphertext); @@ -605,8 +617,17 @@ impl BackupService { Ok(out) } + /// Async wrapper for [`Self::open_bytes`]: decrypt/zstd-decode/untar are + /// CPU-bound, so they run on the blocking pool. + async fn open_blocking(&self, bytes: Vec) -> Result { + let key = self.encryption_key; + tokio::task::spawn_blocking(move || Self::open_bytes(key, &bytes)) + .await + .map_err(|e| BackupError::Io(format!("restore task failed: {e}")))? + } + /// Decrypt/decompress an artifact into (manifest, table NDJSON, file blobs). - fn open(&self, bytes: &[u8]) -> Result { + fn open_bytes(encryption_key: Option<[u8; 32]>, bytes: &[u8]) -> Result { if bytes.len() < MAGIC.len() + 1 || &bytes[..MAGIC.len()] != MAGIC { return Err(BackupError::Invalid("not a CMS backup artifact".into())); } @@ -614,14 +635,18 @@ impl BackupService { let body = &bytes[MAGIC.len() + 1..]; let compressed = if flags & FLAG_ENCRYPTED != 0 { - let key = self.encryption_key.ok_or(BackupError::MissingKey)?; + let key = encryption_key.ok_or(BackupError::MissingKey)?; if body.len() < 12 { return Err(BackupError::Invalid("truncated encrypted backup".into())); } let (nonce_bytes, ciphertext) = body.split_at(12); - let cipher = Aes256Gcm::new(Key::::from_slice(&key)); + let cipher = Aes256Gcm::new(&Key::::from(key)); + let nonce_arr: [u8; 12] = nonce_bytes + .try_into() + .map_err(|_| BackupError::Invalid("truncated encrypted backup".into()))?; + let nonce = Nonce::from(nonce_arr); cipher - .decrypt(Nonce::from_slice(nonce_bytes), ciphertext) + .decrypt(&nonce, ciphertext) .map_err(|_| BackupError::Crypto("decryption failed (wrong key?)".into()))? } else { body.to_vec() diff --git a/apps/backend/src/services/error.rs b/apps/backend/src/services/error.rs index 7c1c2064..fc7bddbb 100644 --- a/apps/backend/src/services/error.rs +++ b/apps/backend/src/services/error.rs @@ -106,6 +106,8 @@ impl ServiceError { FileError::StorageError(_) => StatusCode::INTERNAL_SERVER_ERROR, FileError::NoStorageConfigured => StatusCode::INTERNAL_SERVER_ERROR, FileError::DatabaseError(_) => StatusCode::INTERNAL_SERVER_ERROR, + FileError::AlreadyExists => StatusCode::CONFLICT, + FileError::ReadError(_) => StatusCode::BAD_REQUEST, }, ServiceError::Singleton(e) => match e { SingletonError::NotFound | SingletonError::NotASingleton => StatusCode::NOT_FOUND, diff --git a/apps/backend/src/services/file.rs b/apps/backend/src/services/file.rs index eca303fd..e3ed38ac 100644 --- a/apps/backend/src/services/file.rs +++ b/apps/backend/src/services/file.rs @@ -2,7 +2,10 @@ use std::sync::Arc; use axum::{Json, http::StatusCode, response::IntoResponse}; use bytes::Bytes; +use dashmap::DashMap; +use futures_util::{Stream, StreamExt}; use image::{DynamicImage, ImageEncoder, ImageReader}; +use object_store::WriteMultipart; use serde_json::json; use thiserror::Error; use tracing::{debug, error, info, warn}; @@ -10,13 +13,25 @@ use uuid::Uuid; use crate::config::Config; use crate::models::file::{File, FileReference, FileWithUrl}; +use crate::repository::error::RepositoryError; use crate::repository::traits::{FileListResult, FileRepository, ListFilesParams, NewFile}; use crate::storage::StorageProvider; +use crate::utils::magic_bytes::{self, SNIFF_LEN, Sniff}; + +/// Multipart part size for streaming uploads. S3 requires >= 5 MiB for every +/// part but the last; with ~2 parts in flight, worst case is ~15 MiB of memory +/// per upload regardless of file size. +const MULTIPART_CHUNK_SIZE: usize = 5 * 1024 * 1024; +/// Max multipart parts concurrently in flight per upload. +const MULTIPART_MAX_CONCURRENCY: usize = 2; #[derive(Clone)] pub struct FileService { file_repo: Arc, config: Arc, + /// Pre-generated file ids currently being uploaded (signed-URL flow); makes + /// one-time-use race-free in-process. The `files` PK is the durable backstop. + in_flight: Arc>, } /// Inputs for [`FileService::upload_file`]. @@ -30,6 +45,46 @@ pub struct UploadFileRequest<'a> { pub storage_provider: &'a str, } +/// Inputs for [`FileService::upload_file_streaming`]. `file_id` is set by the +/// signed-URL flow (the token pre-generates it, enforcing one-time use). +pub struct StreamingUploadRequest<'a> { + pub site_id: &'a str, + pub file_id: Option<&'a str>, + pub filename: &'a str, + pub content_type: &'a str, + pub created_by: Option<&'a str>, + pub storage: Arc, + pub storage_provider: &'a str, +} + +/// RAII claim on an in-flight pre-generated file id; released on drop. +struct InFlightClaim { + map: Arc>, + key: String, +} + +impl InFlightClaim { + fn acquire(map: &Arc>, key: &str) -> Option { + use dashmap::mapref::entry::Entry; + match map.entry(key.to_string()) { + Entry::Occupied(_) => None, + Entry::Vacant(v) => { + v.insert(()); + Some(Self { + map: map.clone(), + key: key.to_string(), + }) + } + } + } +} + +impl Drop for InFlightClaim { + fn drop(&mut self) { + self.map.remove(&self.key); + } +} + #[derive(Error, Debug)] pub enum FileError { #[error("Not found")] @@ -55,6 +110,12 @@ pub enum FileError { #[error("Database error: {0}")] DatabaseError(String), + + #[error("File already exists")] + AlreadyExists, + + #[error("Failed to read upload body: {0}")] + ReadError(String), } impl FileError { @@ -77,6 +138,14 @@ impl FileError { ), FileError::DatabaseError(msg) => (StatusCode::INTERNAL_SERVER_ERROR, Json(json!({"error": msg}))), FileError::InvalidContentType(msg) => (StatusCode::BAD_REQUEST, Json(json!({"error": msg}))), + FileError::AlreadyExists => ( + StatusCode::CONFLICT, + Json(json!({"error": "File already exists (upload URL already used)"})), + ), + FileError::ReadError(msg) => ( + StatusCode::BAD_REQUEST, + Json(json!({"error": format!("Failed to read upload body: {}", msg)})), + ), }; (status, body).into_response() } @@ -84,7 +153,11 @@ impl FileError { impl FileService { pub fn new(file_repo: Arc, config: Arc) -> Self { - Self { file_repo, config } + Self { + file_repo, + config, + in_flight: Arc::new(DashMap::new()), + } } pub async fn list_files(&self, params: ListFilesParams<'_>) -> Result { @@ -108,6 +181,8 @@ impl FileService { .map_err(|e| FileError::DatabaseError(e.to_string())) } + /// Buffered upload — thin wrapper over [`Self::upload_file_streaming`] for + /// callers that already hold the whole payload (gRPC, tests). pub async fn upload_file(&self, req: UploadFileRequest<'_>) -> Result { let UploadFileRequest { site_id, @@ -118,25 +193,50 @@ impl FileService { storage, storage_provider, } = req; - info!( - "Uploading file: site_id={}, content_type={}, size={} bytes", + + let stream = futures_util::stream::iter(std::iter::once(Ok(data))); + self.upload_file_streaming( + StreamingUploadRequest { + site_id, + file_id: None, + filename, + content_type, + created_by, + storage, + storage_provider, + }, + stream, + ) + .await + } + + /// Streaming upload: constant memory regardless of file size. Enforces the + /// content-type whitelist, magic-byte sniffing on the first bytes, and the + /// size cap while streaming; aborts the multipart upload on any failure so + /// no partial object is left behind. Thumbnails are generated in a + /// background task after the DB record exists. + pub async fn upload_file_streaming( + &self, + req: StreamingUploadRequest<'_>, + mut stream: S, + ) -> Result + where + S: Stream>> + Unpin + Send, + { + let StreamingUploadRequest { site_id, + file_id, + filename, content_type, - data.len() - ); + created_by, + storage, + storage_provider, + } = req; - if data.len() as u64 > self.config.max_upload_size_bytes as u64 { - warn!( - "File too large: site_id={}, size={} bytes, max={} bytes", - site_id, - data.len(), - self.config.max_upload_size_bytes - ); - return Err(FileError::FileTooLarge(format!( - "File too large. Maximum size is {}MB", - self.config.max_upload_size_bytes / (1024 * 1024) - ))); - } + info!( + "Uploading file (streaming): site_id={}, content_type={}", + site_id, content_type + ); if !crate::utils::content_types::is_allowed(content_type) { return Err(FileError::InvalidContentType(format!( @@ -145,9 +245,29 @@ impl FileService { ))); } + // One-time-use for pre-generated ids (signed URLs): claim the id + // in-process, then verify no record exists. The PK insert below is the + // durable backstop. + let _claim = match file_id { + Some(id) => { + let claim = InFlightClaim::acquire(&self.in_flight, id).ok_or(FileError::AlreadyExists)?; + let existing = self + .file_repo + .get_by_id_any(id) + .await + .map_err(|e| FileError::DatabaseError(e.to_string()))?; + if existing.is_some() { + return Err(FileError::AlreadyExists); + } + Some(claim) + } + None => None, + }; + + let file_id = file_id + .map(str::to_string) + .unwrap_or_else(|| Uuid::now_v7().to_string()); let original_name = filename.to_string(); - let file_size = data.len() as i64; - let file_id = Uuid::now_v7().to_string(); let ext = std::path::Path::new(filename) .extension() .and_then(|e| e.to_str()) @@ -157,87 +277,84 @@ impl FileService { } else { format!("{}.{}", &file_id[..8], ext) }; - let storage_key = format!("s_{}/f_{}/{}", site_id, file_id, generated_filename); let mime_type = content_type.to_string(); + let max = self.config.max_upload_size_bytes; + + let upload = storage.start_multipart(&storage_key).await.map_err(|e| { + error!("Failed to start multipart upload: key={}, error={}", storage_key, e); + FileError::StorageError(e.to_string()) + })?; + let mut writer = WriteMultipart::new_with_chunk_size(upload, MULTIPART_CHUNK_SIZE); + + let too_large = + || FileError::FileTooLarge(format!("File too large. Maximum size is {}MB", max / (1024 * 1024))); + + // Hold back the first SNIFF_LEN bytes so nothing is written before the + // magic-byte check passes; afterwards chunks stream straight through. + let mut sniff_buf: Vec = Vec::new(); + let mut sniffed = false; + let mut total: usize = 0; + + loop { + let chunk = match stream.next().await { + Some(Ok(chunk)) => chunk, + Some(Err(e)) => { + let _ = writer.abort().await; + return Err(FileError::ReadError(e.to_string())); + } + None => break, + }; + + total += chunk.len(); + if total > max { + let _ = writer.abort().await; + warn!("File too large: site_id={}, size>{} bytes, max={}", site_id, total, max); + return Err(too_large()); + } - debug!( - "File metadata: id={}, storage_key={}, mime_type={}, size={}", - file_id, storage_key, mime_type, file_size - ); - - let mut width: Option = None; - let mut height: Option = None; - let mut thumbnail_data: Option<(Vec, String)> = None; - let mut thumbnail_key: Option = None; - - if mime_type.starts_with("image/") { - debug!("Processing image for thumbnail: mime_type={}", mime_type); - let data_clone = data.clone(); - let file_id_owned = file_id.clone(); - let site_id_owned = site_id.to_string(); - - let result = tokio::task::spawn_blocking(move || { - let mut w: Option = None; - let mut h: Option = None; - let mut tdata: Option<(Vec, String)> = None; - let mut tkey: Option = None; - - if let Ok(reader) = ImageReader::new(std::io::Cursor::new(&data_clone)).with_guessed_format() - && let Ok(img) = reader.decode() - { - w = Some(img.width() as i32); - h = Some(img.height() as i32); - - if let Some((thumb_bytes, thumb_mime)) = generate_thumbnail(&img) { - tkey = Some(format!( - "s_{}/f_{}/thumb_{}.avif", - site_id_owned, - file_id_owned, - &file_id_owned[..8] - )); - tdata = Some((thumb_bytes, thumb_mime)); + if !sniffed { + sniff_buf.extend_from_slice(&chunk); + if sniff_buf.len() >= SNIFF_LEN { + if let Sniff::Mismatch(detected) = magic_bytes::check(&mime_type, &sniff_buf) { + let _ = writer.abort().await; + return Err(content_mismatch(&mime_type, detected)); } + sniffed = true; + if let Err(e) = writer.wait_for_capacity(MULTIPART_MAX_CONCURRENCY).await { + let _ = writer.abort().await; + return Err(FileError::StorageError(e.to_string())); + } + writer.write(&sniff_buf); + sniff_buf = Vec::new(); } - - (w, h, tdata, tkey) - }) - .await; - - if let Ok((w, h, tdata, tkey)) = result { - width = w; - height = h; - thumbnail_data = tdata; - thumbnail_key = tkey; - debug!( - "Generated thumbnail: width={:?}, height={:?}, key={:?}", - width, height, thumbnail_key - ); } else { - warn!("Failed to generate thumbnail for image: {}", mime_type); + if let Err(e) = writer.wait_for_capacity(MULTIPART_MAX_CONCURRENCY).await { + let _ = writer.abort().await; + return Err(FileError::StorageError(e.to_string())); + } + writer.write(&chunk); } } - debug!("Storing file to storage: key={}", storage_key); - match storage.put(&storage_key, data.clone(), &mime_type).await { - Ok(_) => debug!("File stored successfully: key={}", storage_key), - Err(e) => { - error!("Failed to store file: key={}, error={}", storage_key, e); - return Err(FileError::StorageError(e.to_string())); + // EOF before the sniff buffer filled: check whatever we have. + if !sniffed { + if let Sniff::Mismatch(detected) = magic_bytes::check(&mime_type, &sniff_buf) { + let _ = writer.abort().await; + return Err(content_mismatch(&mime_type, detected)); + } + if !sniff_buf.is_empty() { + writer.write(&sniff_buf); } } - if let (Some((thumb_data, thumb_mime)), Some(thumb_key)) = (&thumbnail_data, &thumbnail_key) { - debug!("Storing thumbnail: key={}", thumb_key); - let _ = storage - .put(thumb_key, Bytes::from(thumb_data.clone()), thumb_mime) - .await; + if let Err(e) = writer.finish().await { + error!("Failed to finish multipart upload: key={}, error={}", storage_key, e); + return Err(FileError::StorageError(e.to_string())); } + debug!("File stored successfully: key={}, size={} bytes", storage_key, total); - let thumb_key_str = thumbnail_key.as_deref(); - - debug!("Creating file record in repository: id={}", file_id); - let file = self + let file = match self .file_repo .create(NewFile { id: &file_id, @@ -245,31 +362,95 @@ impl FileService { filename: &generated_filename, original_name: &original_name, mime_type: &mime_type, - size: file_size, + size: total as i64, storage_provider, storage_key: &storage_key, - thumbnail_key: thumb_key_str, - width, - height, + thumbnail_key: None, + width: None, + height: None, created_by, }) .await - .map_err(|e| { - error!( - "Failed to create file record in repository: id={}, error={}", - file_id, e - ); - FileError::DatabaseError(e.to_string()) - })?; + { + Ok(file) => file, + Err(RepositoryError::UniqueViolation(_)) => { + // A concurrent upload with the same pre-generated id won the + // insert. Do NOT delete the blob — the winner shares the key. + warn!("Duplicate file id on insert (upload URL reused): id={}", file_id); + return Err(FileError::AlreadyExists); + } + Err(e) => { + error!("Failed to create file record: id={}, error={}", file_id, e); + // Orphan cleanup: the blob was written but has no DB record. + if let Err(del_err) = storage.delete(&storage_key).await { + warn!( + "Failed to clean up orphaned blob: key={}, error={}", + storage_key, del_err + ); + } + return Err(FileError::DatabaseError(e.to_string())); + } + }; info!( "File uploaded successfully: id={}, site_id={}, size={} bytes", file.id, site_id, file.size ); + if mime_type.starts_with("image/") { + self.spawn_thumbnail_task(file.clone(), storage.clone()); + } + Ok(self.file_to_with_url(&file, &*storage)) } + /// Generate thumbnail + dimensions off the upload critical path: read the + /// image back from storage, encode in a blocking task, update the record. + /// Failures are logged and leave the thumbnail fields NULL. + fn spawn_thumbnail_task(&self, file: File, storage: Arc) { + let repo = self.file_repo.clone(); + tokio::spawn(async move { + let data = match storage.get(&file.storage_key).await { + Ok(data) => data, + Err(e) => { + warn!("Thumbnail task: failed to read {}: {}", file.storage_key, e); + return; + } + }; + + let result = tokio::task::spawn_blocking(move || { + let reader = ImageReader::new(std::io::Cursor::new(&data)) + .with_guessed_format() + .ok()?; + let img = reader.decode().ok()?; + let (w, h) = (img.width() as i32, img.height() as i32); + let thumb = generate_thumbnail(&img)?; + Some((w, h, thumb)) + }) + .await; + + match result { + Ok(Some((width, height, (thumb_bytes, thumb_mime)))) => { + let thumb_key = format!("s_{}/f_{}/thumb_{}.avif", file.site_id, file.id, &file.id[..8]); + if let Err(e) = storage.put(&thumb_key, Bytes::from(thumb_bytes), &thumb_mime).await { + warn!("Thumbnail task: failed to store {}: {}", thumb_key, e); + return; + } + if let Err(e) = repo + .set_thumbnail_meta(&file.id, &thumb_key, Some(width), Some(height)) + .await + { + warn!("Thumbnail task: failed to update record {}: {}", file.id, e); + } else { + debug!("Thumbnail generated: id={}, key={}", file.id, thumb_key); + } + } + Ok(None) => debug!("Thumbnail task: {} not decodable as image", file.id), + Err(e) => warn!("Thumbnail task panicked for {}: {}", file.id, e), + } + }); + } + pub async fn soft_delete(&self, id: &str, site_id: &str) -> Result { info!("Soft deleting file"); @@ -498,6 +679,17 @@ impl FileService { } } +/// 400 error for a declared/detected content-type contradiction. +fn content_mismatch(declared: &str, detected: Option<&'static str>) -> FileError { + FileError::InvalidContentType(match detected { + Some(d) => format!( + "File content does not match declared type '{}' (detected '{}')", + declared, d + ), + None => format!("File content does not match declared type '{}'", declared), + }) +} + fn generate_thumbnail(img: &DynamicImage) -> Option<(Vec, String)> { let thumb = img.resize(260, 260, image::imageops::FilterType::Lanczos3); let rgba = thumb.to_rgba8(); @@ -620,6 +812,90 @@ mod tests { assert!(matches!(result, Err(FileError::FileTooLarge(_)))); } + #[tokio::test] + async fn test_upload_file_streams_to_storage_and_creates_record() { + let file_repo = test_file_repo(); + let config = test_config(); + let service = FileService::new(file_repo.clone(), config); + + let storage = Arc::new(MockStorage::default()); + let result = service + .upload_file(UploadFileRequest { + site_id: "site-123", + data: Bytes::from("hello world"), + filename: "notes.txt", + content_type: "text/plain", + created_by: None, + storage: storage.clone(), + storage_provider: "filesystem", + }) + .await + .expect("upload should succeed"); + + assert_eq!(result.size, 11); + assert!(result.thumbnail_url.is_none()); + let stored = storage.files.lock().unwrap(); + assert_eq!(stored.get(&result.storage_key).unwrap().as_ref(), b"hello world"); + } + + #[tokio::test] + async fn test_upload_streaming_magic_byte_mismatch_rejected() { + let file_repo = test_file_repo(); + let config = test_config(); + let service = FileService::new(file_repo, config); + + let storage = Arc::new(MockStorage::default()); + let result = service + .upload_file(UploadFileRequest { + site_id: "site-123", + data: Bytes::from("definitely not a png"), + filename: "fake.png", + content_type: "image/png", + created_by: None, + storage: storage.clone(), + storage_provider: "filesystem", + }) + .await; + + assert!(matches!(result, Err(FileError::InvalidContentType(_)))); + // Aborted upload must leave nothing behind. + assert!(storage.files.lock().unwrap().is_empty()); + } + + #[tokio::test] + async fn test_upload_streaming_pregenerated_id_single_use() { + let file_repo = test_file_repo(); + let config = test_config(); + let service = FileService::new(file_repo, config); + let storage = Arc::new(MockStorage::default()); + + let req = |storage: Arc| StreamingUploadRequest { + site_id: "site-123", + file_id: Some("0197-fixed-id"), + filename: "a.txt", + content_type: "text/plain", + created_by: None, + storage, + storage_provider: "filesystem", + }; + + let first = service + .upload_file_streaming( + req(storage.clone()), + futures_util::stream::iter(std::iter::once(Ok(Bytes::from("one")))), + ) + .await; + assert!(first.is_ok()); + + let second = service + .upload_file_streaming( + req(storage.clone()), + futures_util::stream::iter(std::iter::once(Ok(Bytes::from("two")))), + ) + .await; + assert!(matches!(second, Err(FileError::AlreadyExists))); + } + #[tokio::test] async fn test_soft_delete_success() { let file_repo = test_file_repo(); diff --git a/apps/backend/src/services/search/indexer.rs b/apps/backend/src/services/search/indexer.rs index 5e2f9f52..2064ab3e 100644 --- a/apps/backend/src/services/search/indexer.rs +++ b/apps/backend/src/services/search/indexer.rs @@ -1,36 +1,57 @@ //! The single-consumer search indexer. //! //! Spawned once by the running server (it owns the Tantivy writer). It drains the -//! [`SearchQueue`] — populated by content writes from *any* process — and applies -//! the changes to the index. It wakes immediately on a local enqueue (via the -//! queue's `Notify`) and also polls on an interval to pick up enqueues from other -//! processes (e.g. `vcms mcp stdio`). +//! [`SearchQueue`] and applies the changes to the index. The server is the only +//! process that writes content (`vcms mcp stdio` is an HTTP proxy to it), and +//! every enqueue rings the queue's in-process `Notify`, so the indexer is purely +//! event-driven: one startup drain for rows a previous process left behind +//! (crash recovery), then sleep until notified. use std::collections::HashSet; use std::sync::Arc; -use std::time::Duration; use super::SearchError; use super::SearchService; use super::queue::SearchQueue; use crate::repository::Repository; -/// Fallback poll interval for enqueues made by other processes (which can't ring -/// this process's in-memory `Notify`). -const POLL_INTERVAL: Duration = Duration::from_secs(2); /// Max rows processed per drain iteration. const BATCH_SIZE: i64 = 256; +/// Retries after a failed drain before giving up until the next enqueue. +const MAX_DRAIN_RETRIES: u32 = 5; + /// Run the indexer loop forever. Owns the writer-side `SearchService`. pub async fn run(search: Arc, queue: Arc, repository: Arc) { let notify = queue.notify_handle(); + + // Startup drain: pick up rows committed before a crash/restart. `Notify` + // stores a permit, so an enqueue racing this drain is never lost — it just + // wakes the loop below for an empty (cheap) drain in the worst case. + drain_with_retry(&search, &queue, &repository).await; + loop { - tokio::select! { - _ = notify.notified() => {} - _ = tokio::time::sleep(POLL_INTERVAL) => {} - } - if let Err(e) = drain(&search, &queue, &repository).await { - tracing::error!("Search indexer drain failed: {}", e); + notify.notified().await; + drain_with_retry(&search, &queue, &repository).await; + } +} + +/// Drain, retrying transient failures with exponential backoff so queued rows +/// don't sit stuck until the next enqueue. After [`MAX_DRAIN_RETRIES`] the rows +/// stay in the durable queue for the next notification or restart. +async fn drain_with_retry(search: &SearchService, queue: &SearchQueue, repository: &Repository) { + let mut delay = std::time::Duration::from_millis(500); + for attempt in 0..=MAX_DRAIN_RETRIES { + match drain(search, queue, repository).await { + Ok(()) => return, + Err(e) if attempt == MAX_DRAIN_RETRIES => { + tracing::error!("Search indexer drain failed after {MAX_DRAIN_RETRIES} retries: {e}"); + } + Err(e) => { + tracing::warn!("Search indexer drain failed (attempt {}): {e}; retrying", attempt + 1); + tokio::time::sleep(delay).await; + delay = (delay * 2).min(std::time::Duration::from_secs(10)); + } } } } diff --git a/apps/backend/src/services/search/queue.rs b/apps/backend/src/services/search/queue.rs index dab97821..756c34d4 100644 --- a/apps/backend/src/services/search/queue.rs +++ b/apps/backend/src/services/search/queue.rs @@ -1,10 +1,11 @@ -//! Durable, cross-process queue of pending search-index updates -//! (`search_index_queue` table). +//! Durable queue of pending search-index updates (`search_index_queue` table). //! -//! Producers (any process that writes entry content) call [`SearchQueue::enqueue`] -//! after a write. The single writer-owning server drains the queue via the -//! [`indexer`](super::indexer). Because the queue lives in the database it works -//! across processes (e.g. a separate `vcms mcp stdio`) and survives restarts. +//! Content writes call [`SearchQueue::enqueue`] after committing; the server's +//! [`indexer`](super::indexer) drains the queue. All producers run in the server +//! process (`vcms mcp stdio` is an HTTP proxy), so every enqueue also rings the +//! in-process `Notify` and the indexer needs no polling. The queue lives in the +//! database purely for durability: rows enqueued before a crash are drained on +//! the next startup. use std::sync::Arc; @@ -31,8 +32,7 @@ pub struct QueueRow { /// Handle for enqueuing index updates and draining them. pub struct SearchQueue { pool: DbPool, - /// Wakes the in-process indexer immediately on a local enqueue (cross-process - /// enqueues are picked up by the indexer's poll fallback instead). + /// Wakes the indexer immediately on an enqueue (all producers are in-process). notify: Arc, } diff --git a/apps/backend/src/services/webhook.rs b/apps/backend/src/services/webhook.rs index fc30bf0f..7094a1fd 100644 --- a/apps/backend/src/services/webhook.rs +++ b/apps/backend/src/services/webhook.rs @@ -512,11 +512,11 @@ fn derive_encryption_key(secret: &str) -> [u8; 32] { /// `base64(nonce[12] || ciphertext||tag)`. A fresh random nonce is used per call. fn encrypt_headers(headers: &HashMap, key: &[u8; 32]) -> String { let json = serde_json::to_string(headers).unwrap_or_default(); - let cipher = Aes256Gcm::new(Key::::from_slice(key)); + let cipher = Aes256Gcm::new(&Key::::from(*key)); let mut nonce_bytes = [0u8; 12]; rand::rng().fill_bytes(&mut nonce_bytes); - let nonce = Nonce::from_slice(&nonce_bytes); - match cipher.encrypt(nonce, json.as_bytes()) { + let nonce = Nonce::from(nonce_bytes); + match cipher.encrypt(&nonce, json.as_bytes()) { Ok(ciphertext) => { let mut out = Vec::with_capacity(nonce_bytes.len() + ciphertext.len()); out.extend_from_slice(&nonce_bytes); @@ -539,9 +539,13 @@ fn decrypt_headers(encrypted: &str, key: &[u8; 32]) -> HashMap { return HashMap::new(); } let (nonce_bytes, ciphertext) = raw.split_at(12); - let cipher = Aes256Gcm::new(Key::::from_slice(key)); - let nonce = Nonce::from_slice(nonce_bytes); - match cipher.decrypt(nonce, ciphertext) { + let cipher = Aes256Gcm::new(&Key::::from(*key)); + let nonce_arr: [u8; 12] = match nonce_bytes.try_into() { + Ok(n) => n, + Err(_) => return HashMap::new(), + }; + let nonce = Nonce::from(nonce_arr); + match cipher.decrypt(&nonce, ciphertext) { Ok(plaintext) => match String::from_utf8(plaintext) { Ok(s) => serde_json::from_str(&s).unwrap_or_default(), Err(_) => HashMap::new(), diff --git a/apps/backend/src/signed_upload.rs b/apps/backend/src/signed_upload.rs index a8f99372..cbaa3db9 100644 --- a/apps/backend/src/signed_upload.rs +++ b/apps/backend/src/signed_upload.rs @@ -5,7 +5,9 @@ use thiserror::Error; type HmacSha256 = Hmac; -const UPLOAD_TOKEN_EXPIRY_SECS: i64 = 900; // 15 minutes +/// Default signed-URL lifetime; the runtime value comes from +/// `Config::upload_token_expiry_secs` (default 900). +pub const DEFAULT_UPLOAD_TOKEN_EXPIRY_SECS: i64 = 900; #[derive(Debug, Clone)] pub struct SignedUploadToken { @@ -35,31 +37,14 @@ pub enum SignedUploadError { impl SignedUploadToken { pub fn generate(site_id: &str, filename: &str, content_type: &str, hmac_secret: &str) -> (Self, String) { - let file_id = uuid::Uuid::now_v7().to_string(); - let storage_provider = "filesystem".to_string(); - let expires_at = chrono::Utc::now().timestamp() + UPLOAD_TOKEN_EXPIRY_SECS; - - let payload = format!( - "{}:{}:{}:{}:{}:{}", - file_id, site_id, filename, content_type, storage_provider, expires_at - ); - - let mut mac = HmacSha256::new_from_slice(hmac_secret.as_bytes()).expect("HMAC can take key of any size"); - mac.update(payload.as_bytes()); - let signature = hex::encode(mac.finalize().into_bytes()); - - let token = Self { - file_id, - site_id: site_id.to_string(), - filename: filename.to_string(), - content_type: content_type.to_string(), - storage_provider, - expires_at, - signature, - }; - - let encoded = Self::encode(&token); - (token, encoded) + Self::generate_with_storage_provider( + site_id, + filename, + content_type, + "filesystem", + hmac_secret, + DEFAULT_UPLOAD_TOKEN_EXPIRY_SECS, + ) } pub fn generate_with_storage_provider( @@ -68,9 +53,10 @@ impl SignedUploadToken { content_type: &str, storage_provider: &str, hmac_secret: &str, + expiry_secs: i64, ) -> (Self, String) { let file_id = uuid::Uuid::now_v7().to_string(); - let expires_at = chrono::Utc::now().timestamp() + UPLOAD_TOKEN_EXPIRY_SECS; + let expires_at = chrono::Utc::now().timestamp() + expiry_secs; let payload = format!( "{}:{}:{}:{}:{}:{}", @@ -211,11 +197,31 @@ mod tests { #[test] fn test_generate_with_storage_provider() { - let (token, encoded) = - SignedUploadToken::generate_with_storage_provider("site-123", "doc.pdf", "application/pdf", "s3", "secret"); + let (token, encoded) = SignedUploadToken::generate_with_storage_provider( + "site-123", + "doc.pdf", + "application/pdf", + "s3", + "secret", + DEFAULT_UPLOAD_TOKEN_EXPIRY_SECS, + ); assert_eq!(token.storage_provider, "s3"); let verified = SignedUploadToken::verify(&encoded, "secret").unwrap(); assert_eq!(verified.storage_provider, "s3"); } + + #[test] + fn test_verify_expired_token() { + let (_, encoded) = SignedUploadToken::generate_with_storage_provider( + "site-123", + "doc.pdf", + "application/pdf", + "filesystem", + "secret", + -10, + ); + let result = SignedUploadToken::verify(&encoded, "secret"); + assert!(matches!(result, Err(SignedUploadError::Expired))); + } } diff --git a/apps/backend/src/storage/filesystem.rs b/apps/backend/src/storage/filesystem.rs index 088ed316..6094df41 100644 --- a/apps/backend/src/storage/filesystem.rs +++ b/apps/backend/src/storage/filesystem.rs @@ -81,6 +81,14 @@ impl StorageProvider for FileSystemStorage { fn url(&self, key: &str, file_id: &str) -> String { self.url(key, file_id) } + + async fn start_multipart( + &self, + key: &str, + ) -> Result, Box> { + let path = ObjectPath::from(key); + Ok(self.store.put_multipart(&path).await?) + } } #[cfg(test)] diff --git a/apps/backend/src/storage/mod.rs b/apps/backend/src/storage/mod.rs index ec225bff..95143ac1 100644 --- a/apps/backend/src/storage/mod.rs +++ b/apps/backend/src/storage/mod.rs @@ -78,11 +78,51 @@ pub trait StorageProvider: Send + Sync { async fn get(&self, key: &str) -> Result>; async fn delete(&self, key: &str) -> Result<(), Box>; fn url(&self, key: &str, file_id: &str) -> String; + /// Begin a streaming (multipart) upload to `key`. Drive it with + /// [`object_store::WriteMultipart`]; `abort()` cleans up partial state. + async fn start_multipart( + &self, + key: &str, + ) -> Result, Box>; } #[derive(Default)] pub struct MockStorage { - pub files: std::sync::Mutex>, + pub files: Arc>>, +} + +/// In-memory [`object_store::MultipartUpload`] so `MockStorage` supports the +/// streaming path: parts accumulate in a buffer, `complete` publishes to the map. +#[derive(Debug)] +struct MockMultipartUpload { + files: Arc>>, + key: String, + buf: Vec, +} + +#[async_trait] +impl object_store::MultipartUpload for MockMultipartUpload { + fn put_part(&mut self, data: object_store::PutPayload) -> object_store::UploadPart { + for chunk in data.iter() { + self.buf.extend_from_slice(chunk); + } + Box::pin(futures_util::future::ready(Ok(()))) + } + + async fn complete(&mut self) -> object_store::Result { + let data = Bytes::from(std::mem::take(&mut self.buf)); + self.files.lock().unwrap().insert(self.key.clone(), data); + Ok(object_store::PutResult { + e_tag: None, + version: None, + extensions: Default::default(), + }) + } + + async fn abort(&mut self) -> object_store::Result<()> { + self.buf.clear(); + Ok(()) + } } #[async_trait] @@ -109,6 +149,17 @@ impl StorageProvider for MockStorage { fn url(&self, _key: &str, file_id: &str) -> String { format!("/mock/{}", file_id) } + + async fn start_multipart( + &self, + key: &str, + ) -> Result, Box> { + Ok(Box::new(MockMultipartUpload { + files: self.files.clone(), + key: key.to_string(), + buf: Vec::new(), + })) + } } #[cfg(test)] diff --git a/apps/backend/src/storage/s3.rs b/apps/backend/src/storage/s3.rs index cb066a12..cff1db56 100644 --- a/apps/backend/src/storage/s3.rs +++ b/apps/backend/src/storage/s3.rs @@ -95,4 +95,12 @@ impl StorageProvider for S3Storage { fn url(&self, key: &str, file_id: &str) -> String { self.url(key, file_id) } + + async fn start_multipart( + &self, + key: &str, + ) -> Result, Box> { + let path = ObjectPath::from(key); + Ok(self.store.put_multipart(&path).await?) + } } diff --git a/apps/backend/src/test_helpers.rs b/apps/backend/src/test_helpers.rs index 983c3294..810b9b30 100644 --- a/apps/backend/src/test_helpers.rs +++ b/apps/backend/src/test_helpers.rs @@ -993,6 +993,9 @@ impl FileRepository for InMemoryFileRepository { created_by, } = file; let mut files = self.files.lock().unwrap(); + if files.iter().any(|f| f.id == id) { + return Err(RepositoryError::UniqueViolation("files.id".into())); + } let file = File { id: id.to_string(), site_id: site_id.to_string(), @@ -1100,6 +1103,22 @@ impl FileRepository for InMemoryFileRepository { .map(|f| f.storage_provider.clone()) .unwrap_or_else(|| "filesystem".to_string())) } + + async fn set_thumbnail_meta( + &self, + id: &str, + thumbnail_key: &str, + width: Option, + height: Option, + ) -> Result<(), RepositoryError> { + let mut files = self.files.lock().unwrap(); + if let Some(file) = files.iter_mut().find(|f| f.id == id) { + file.thumbnail_key = Some(thumbnail_key.to_string()); + file.width = width; + file.height = height; + } + Ok(()) + } } #[derive(Clone)] @@ -1183,6 +1202,7 @@ impl AccessTokenRepository for InMemoryAccessTokenRepository { t.expires_at.clone(), t.revoked_at.clone(), t.permission.clone(), + t.last_used_at.clone(), ) }) .collect()) diff --git a/apps/backend/src/tracing.rs b/apps/backend/src/tracing.rs index d45d1477..7fa34a06 100644 --- a/apps/backend/src/tracing.rs +++ b/apps/backend/src/tracing.rs @@ -85,33 +85,13 @@ pub fn init_tracing(config: &crate::config::Config) -> Option Response { diff --git a/apps/backend/src/utils/magic_bytes.rs b/apps/backend/src/utils/magic_bytes.rs new file mode 100644 index 00000000..ff7a74ee --- /dev/null +++ b/apps/backend/src/utils/magic_bytes.rs @@ -0,0 +1,142 @@ +//! Content sniffing for uploads: compare the declared MIME type against the +//! file's magic bytes so a payload can't claim to be something it isn't +//! (e.g. an executable declared as `image/png`). +//! +//! Only types with reliable signatures are checked; signature-less formats +//! (plain text, CSV, SVG, ...) and declared types we have no table entry for +//! are skipped rather than rejected. + +/// How many leading bytes the sniffer needs. Callers should buffer up to this +/// many bytes (or the whole body, if smaller) before checking. +pub const SNIFF_LEN: usize = 8192; + +/// Result of a sniff check. +#[derive(Debug, PartialEq, Eq)] +pub enum Sniff { + /// Detected type is compatible with the declared type. + Match, + /// Detected type contradicts the declared type (detected type inside, if any). + Mismatch(Option<&'static str>), + /// Declared type has no reliable signature — nothing to verify. + Skip, +} + +/// MIME types accepted as a detection result for each checkable declared type. +/// Groups exist because containers overlap (mp4/quicktime brands, ogg family, +/// webm-is-matroska, OOXML-is-zip, legacy Office CFB). +fn accepted_detections(declared: &str) -> Option<&'static [&'static str]> { + const OOXML_OR_ZIP: &[&str] = &[ + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "application/vnd.openxmlformats-officedocument.presentationml.presentation", + "application/zip", + ]; + const LEGACY_OFFICE: &[&str] = &[ + "application/msword", + "application/vnd.ms-excel", + "application/vnd.ms-powerpoint", + "application/x-ole-storage", + ]; + const OGG: &[&str] = &["application/ogg", "audio/ogg", "video/ogg"]; + const MP4_FAMILY: &[&str] = &["video/mp4", "video/quicktime", "video/x-m4v"]; + const MATROSKA: &[&str] = &["video/webm", "video/x-matroska", "audio/webm"]; + + Some(match declared { + "image/jpeg" | "image/jpg" => &["image/jpeg"], + "image/png" => &["image/png"], + "image/gif" => &["image/gif"], + "image/webp" => &["image/webp"], + "image/avif" => &["image/avif", "image/heif", "image/heic"], + "image/tiff" => &["image/tiff"], + "image/bmp" => &["image/bmp"], + "video/mp4" | "video/quicktime" => MP4_FAMILY, + "video/webm" | "audio/webm" => MATROSKA, + "video/ogg" | "audio/ogg" => OGG, + "video/x-msvideo" => &["video/x-msvideo"], + "audio/mpeg" => &["audio/mpeg", "audio/mp3"], + "audio/wav" => &["audio/wav", "audio/x-wav"], + "audio/aac" => &["audio/aac"], + "audio/flac" => &["audio/flac", "audio/x-flac"], + "application/pdf" => &["application/pdf"], + "application/msword" | "application/vnd.ms-excel" | "application/vnd.ms-powerpoint" => LEGACY_OFFICE, + "application/vnd.openxmlformats-officedocument.wordprocessingml.document" + | "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" + | "application/vnd.openxmlformats-officedocument.presentationml.presentation" + | "application/zip" => OOXML_OR_ZIP, + "application/gzip" => &["application/gzip"], + "application/x-tar" => &["application/x-tar"], + "application/x-7z-compressed" => &["application/x-7z-compressed"], + "application/x-rar-compressed" => &["application/vnd.rar", "application/x-rar-compressed"], + // Signature-less or unknown declared types: nothing to verify. + _ => return None, + }) +} + +/// Check `prefix` (the first bytes of the payload, up to [`SNIFF_LEN`]) against +/// the declared MIME type. +pub fn check(declared_mime: &str, prefix: &[u8]) -> Sniff { + let Some(accepted) = accepted_detections(declared_mime) else { + return Sniff::Skip; + }; + + match infer::get(prefix) { + Some(kind) if accepted.contains(&kind.mime_type()) => Sniff::Match, + Some(kind) => Sniff::Mismatch(Some(kind.mime_type())), + None => Sniff::Mismatch(None), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn png_signature_matches() { + let png = [0x89, b'P', b'N', b'G', 0x0D, 0x0A, 0x1A, 0x0A, 0, 0, 0, 0]; + assert_eq!(check("image/png", &png), Sniff::Match); + } + + #[test] + fn jpeg_signature_matches() { + let jpeg = [0xFF, 0xD8, 0xFF, 0xE0, 0, 0, 0, 0]; + assert_eq!(check("image/jpeg", &jpeg), Sniff::Match); + } + + #[test] + fn pdf_signature_matches() { + assert_eq!(check("application/pdf", b"%PDF-1.7 rest of file"), Sniff::Match); + } + + #[test] + fn text_declared_as_png_is_mismatch() { + assert_eq!(check("image/png", b"definitely not a png"), Sniff::Mismatch(None)); + } + + #[test] + fn png_declared_as_jpeg_is_mismatch() { + let png = [0x89, b'P', b'N', b'G', 0x0D, 0x0A, 0x1A, 0x0A, 0, 0, 0, 0]; + assert_eq!(check("image/jpeg", &png), Sniff::Mismatch(Some("image/png"))); + } + + #[test] + fn signature_less_types_are_skipped() { + assert_eq!(check("text/plain", b"hello"), Sniff::Skip); + assert_eq!(check("text/csv", b"a,b,c"), Sniff::Skip); + assert_eq!(check("image/svg+xml", b""), Sniff::Skip); + assert_eq!(check("application/octet-stream", &[0, 1, 2]), Sniff::Skip); + } + + #[test] + fn ooxml_zip_family_is_interchangeable() { + // Minimal ZIP local-file-header signature. + let zip = [b'P', b'K', 0x03, 0x04, 0, 0, 0, 0]; + assert_eq!( + check( + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + &zip + ), + Sniff::Match + ); + assert_eq!(check("application/zip", &zip), Sniff::Match); + } +} diff --git a/apps/backend/src/utils/mod.rs b/apps/backend/src/utils/mod.rs index 7dffe145..8c8fd09a 100644 --- a/apps/backend/src/utils/mod.rs +++ b/apps/backend/src/utils/mod.rs @@ -1,2 +1,3 @@ pub mod content_types; pub mod diff; +pub mod magic_bytes; diff --git a/apps/backend/tests/common/grpc.rs b/apps/backend/tests/common/grpc.rs index a50f3446..5fe2ff24 100644 --- a/apps/backend/tests/common/grpc.rs +++ b/apps/backend/tests/common/grpc.rs @@ -96,6 +96,7 @@ impl GrpcTestContext { )); let app = create_router( + pool.clone(), repository.clone(), (*config).clone(), storage_registry.clone(), diff --git a/apps/backend/tests/common/server.rs b/apps/backend/tests/common/server.rs index 7f5803fa..909eb6ba 100644 --- a/apps/backend/tests/common/server.rs +++ b/apps/backend/tests/common/server.rs @@ -74,6 +74,8 @@ impl TestServer { db_acquire_timeout_secs: 30, db_idle_timeout_secs: 5, max_upload_size_bytes: 50 * 1024 * 1024, + // `Config::default()` leaves this 0 => tokens minted already expired. + upload_token_expiry_secs: 900, public_registration_enabled: true, bcrypt_cost: bcrypt::DEFAULT_COST, webhook_allow_private_targets: true, @@ -124,6 +126,7 @@ impl TestServer { )); let app = create_router( + pool.clone(), repository.clone(), config.clone(), storage_registry, diff --git a/apps/backend/tests/grpc/files_tests.rs b/apps/backend/tests/grpc/files_tests.rs index 01b3967b..e6bd89fb 100644 --- a/apps/backend/tests/grpc/files_tests.rs +++ b/apps/backend/tests/grpc/files_tests.rs @@ -19,10 +19,26 @@ async fn seed_file(ctx: &GrpcTestContext, site_id: &str, name: &str) -> String { body["id"].as_str().unwrap().to_string() } +/// Minimal bytes carrying the right magic signature — uploads are now sniffed +/// against the declared mime type, so plain text can't pose as png/jpeg/pdf. +fn body_for_mime(mime_type: &str) -> Vec { + match mime_type { + "image/png" => vec![0x89, b'P', b'N', b'G', 0x0D, 0x0A, 0x1A, 0x0A, 0, 0, 0, 0], + "image/jpeg" => vec![0xFF, 0xD8, 0xFF, 0xE0, 0, 0, 0, 0], + "application/pdf" => b"%PDF-1.4 test".to_vec(), + _ => b"test content".to_vec(), + } +} + async fn seed_file_with_mime(ctx: &GrpcTestContext, site_id: &str, name: &str, mime_type: &str) -> String { let ext = mime_type.split('/').next_back().unwrap_or("bin"); let body = ctx - .upload_file(site_id, &format!("{}.{}", name, ext), b"test content", mime_type) + .upload_file( + site_id, + &format!("{}.{}", name, ext), + &body_for_mime(mime_type), + mime_type, + ) .await; body["id"].as_str().unwrap().to_string() } diff --git a/apps/backend/tests/mcp/file_tests.rs b/apps/backend/tests/mcp/file_tests.rs index 7d01e77b..c1d976d0 100644 --- a/apps/backend/tests/mcp/file_tests.rs +++ b/apps/backend/tests/mcp/file_tests.rs @@ -53,6 +53,87 @@ async fn test_create_upload_url() { assert_eq!(data["content_type"].as_str().unwrap(), "text/plain"); } +/// End-to-end: mint an upload URL via MCP, PUT the bytes to it, and verify the +/// file exists (both via the MCP get_file tool and the returned record). +#[tokio::test] +async fn test_create_upload_url_then_put_uploads_file() { + let server = start_mcp_server().await; + let (_, token) = setup_site_token(&server).await; + + let result = mcp_call_tool( + &server.base_url, + &token, + "create_upload_url", + serde_json::json!({ + "filename": "e2e.txt", + "content_type": "text/plain", + }), + ) + .await; + let data = mcp_tool_json(&result); + let upload_url = data["upload_url"].as_str().unwrap(); + let file_id = data["file_id"].as_str().unwrap(); + + // The URL derives its host from the request's Host header, so it points at + // this test server and is directly PUT-able. + let client = reqwest::Client::new(); + let resp = client + .put(upload_url) + .header(reqwest::header::CONTENT_TYPE, "text/plain") + .body("mcp e2e upload") + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 201, "{}", resp.text().await.unwrap_or_default()); + let created: serde_json::Value = resp.json().await.unwrap(); + assert_eq!(created["id"].as_str().unwrap(), file_id); + + // Reuse must be rejected (single-use). + let again = client.put(upload_url).body("again").send().await.unwrap(); + assert_eq!(again.status(), 409); + + // The file is visible through MCP. + let got = mcp_call_tool( + &server.base_url, + &token, + "get_file", + serde_json::json!({"file_id": file_id}), + ) + .await; + let got_data = mcp_tool_json(&got); + assert_eq!(got_data["original_name"].as_str().unwrap(), "e2e.txt"); +} + +#[tokio::test] +async fn test_create_upload_url_requires_editor() { + let server = start_mcp_server().await; + let (_, token) = setup_site_read_token(&server).await; + + let result = mcp_call_tool( + &server.base_url, + &token, + "create_upload_url", + serde_json::json!({"filename": "x.txt", "content_type": "text/plain"}), + ) + .await; + assert!(mcp_is_error(&result), "read-only token must not mint upload URLs"); +} + +#[tokio::test] +async fn test_create_upload_url_rejects_disallowed_content_type() { + let server = start_mcp_server().await; + let (_, token) = setup_site_token(&server).await; + + let result = mcp_call_tool( + &server.base_url, + &token, + "create_upload_url", + serde_json::json!({"filename": "x.exe", "content_type": "application/x-executable"}), + ) + .await; + assert!(mcp_is_error(&result), "disallowed content type must fail at mint time"); +} + #[tokio::test] async fn test_list_files_with_pagination() { let server = start_mcp_server().await; diff --git a/apps/backend/tests/mcp/main.rs b/apps/backend/tests/mcp/main.rs index 9450c8d8..050aa844 100644 --- a/apps/backend/tests/mcp/main.rs +++ b/apps/backend/tests/mcp/main.rs @@ -8,4 +8,5 @@ mod protocol_tests; mod resource_tests; mod singleton_tests; mod site_tests; +mod stdio_proxy_tests; mod webhook_tests; diff --git a/apps/backend/tests/mcp/stdio_proxy_tests.rs b/apps/backend/tests/mcp/stdio_proxy_tests.rs new file mode 100644 index 00000000..ff58ddaf --- /dev/null +++ b/apps/backend/tests/mcp/stdio_proxy_tests.rs @@ -0,0 +1,168 @@ +//! End-to-end tests for `vcms mcp stdio` as a thin HTTP proxy. +//! +//! The real `vcms` binary is spawned in stdio mode and pointed at a live +//! `TestServer`'s `/mcp` endpoint via `VCMS_MCP_URL` + `VCMS_MCP_TOKEN`. It touches +//! no database or secrets of its own — everything is forwarded to the server. + +use std::process::Stdio; + +use serde_json::{Value, json}; +use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader}; +use tokio::process::{Child, ChildStdin, ChildStdout, Command}; + +use crate::common::{TestServer, fixtures}; + +struct StdioClient { + child: Child, + stdin: ChildStdin, + stdout: BufReader, +} + +impl StdioClient { + /// Spawn the proxy the way an MCP client does: it knows only the server URL and a + /// site token — no `DATABASE_URL`, `HMAC_SECRET`, or `VCMS_HOME`. + async fn start(url: &str, token: &str) -> Self { + let mut child = Command::new(env!("CARGO_BIN_EXE_vcms")) + .args(["mcp", "stdio"]) + .env_remove("DATABASE_URL") + .env_remove("HMAC_SECRET") + .env_remove("VCMS_HOME") + .env("VCMS_MCP_URL", url) + .env("VCMS_MCP_TOKEN", token) + .env("RUST_LOG", "warn") + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .kill_on_drop(true) + .spawn() + .expect("spawn vcms mcp stdio"); + + let stdin = child.stdin.take().expect("child stdin"); + let stdout = BufReader::new(child.stdout.take().expect("child stdout")); + Self { child, stdin, stdout } + } + + async fn request(&mut self, id: u64, method: &str, params: Option) -> Value { + let mut request = json!({"jsonrpc": "2.0", "id": id, "method": method}); + if let Some(params) = params { + request["params"] = params; + } + self.stdin + .write_all(format!("{request}\n").as_bytes()) + .await + .expect("write MCP request"); + self.stdin.flush().await.expect("flush MCP request"); + + let mut line = String::new(); + self.stdout.read_line(&mut line).await.expect("read MCP response"); + serde_json::from_str(&line).unwrap_or_else(|error| panic!("stdout was not pure MCP JSON: {line:?}: {error}")) + } + + async fn notify(&mut self, method: &str) { + let request = json!({"jsonrpc": "2.0", "method": method}); + self.stdin + .write_all(format!("{request}\n").as_bytes()) + .await + .expect("write MCP notification"); + self.stdin.flush().await.expect("flush MCP notification"); + } + + async fn close(mut self) -> std::process::ExitStatus { + drop(self.stdin); + let mut stderr = self.child.stderr.take().expect("child stderr"); + let status = self.child.wait().await.expect("wait for stdio process"); + let mut logs = String::new(); + stderr.read_to_string(&mut logs).await.expect("read stderr logs"); + status + } +} + +async fn initialize(client: &mut StdioClient) -> Value { + let response = client + .request( + 1, + "initialize", + Some(json!({ + "protocolVersion": "2024-11-05", + "capabilities": {}, + "clientInfo": {"name": "stdio-proxy-test", "version": "1.0"} + })), + ) + .await; + client.notify("notifications/initialized").await; + response +} + +#[tokio::test] +async fn stdio_proxy_round_trips_tools_and_calls_through_the_server() { + let server = TestServer::start().await; + let (_site_id, token) = fixtures::create_site_and_token(&server, "write").await; + + let mut client = StdioClient::start(&server.base_url, &token).await; + let init = initialize(&mut client).await; + assert_eq!(init["result"]["serverInfo"]["name"], "cms"); + + let tools = client.request(2, "tools/list", None).await; + assert!( + tools["result"]["tools"].as_array().is_some_and(|t| !t.is_empty()), + "tools/list should return the site's tools: {tools}" + ); + + let site = client + .request(3, "tools/call", Some(json!({"name": "get_site", "arguments": {}}))) + .await; + assert_eq!( + site["result"]["isError"], false, + "get_site should succeed through the proxy: {site}" + ); + + assert!(client.close().await.success(), "proxy should exit cleanly on stdin EOF"); +} + +#[tokio::test] +async fn stdio_proxy_enforces_token_permission() { + let server = TestServer::start().await; + // A read-only token: reads pass, writes are denied by the server. + let (_site_id, token) = fixtures::create_site_and_token(&server, "read").await; + + let mut client = StdioClient::start(&server.base_url, &token).await; + initialize(&mut client).await; + + let mutation = client + .request( + 2, + "tools/call", + Some(json!({"name": "update_site", "arguments": {"name": "Forbidden"}})), + ) + .await; + assert_eq!( + mutation["result"]["isError"], true, + "read token must not write: {mutation}" + ); + + assert!(client.close().await.success()); +} + +#[tokio::test] +async fn stdio_proxy_wraps_bad_token_as_jsonrpc_error() { + let server = TestServer::start().await; + + let mut client = StdioClient::start(&server.base_url, "vcms_site_definitely_invalid").await; + let response = client + .request( + 1, + "initialize", + Some(json!({ + "protocolVersion": "2024-11-05", + "capabilities": {}, + "clientInfo": {"name": "bad", "version": "1.0"} + })), + ) + .await; + assert!( + response.get("error").is_some(), + "an invalid token must surface as a JSON-RPC error: {response}" + ); + + assert!(client.close().await.success()); +} diff --git a/apps/backend/tests/mcp_stdio.rs b/apps/backend/tests/mcp_stdio.rs deleted file mode 100644 index d8d36456..00000000 --- a/apps/backend/tests/mcp_stdio.rs +++ /dev/null @@ -1,289 +0,0 @@ -use std::process::Stdio; - -use cms::config::Config; -use cms::database::{init_db_with_config, pool::DbPool}; -use cms::models::access_token::AccessTokenPermission; -use cms::repository::Repository; -use cms::services::access_token::AccessTokenService; -use serde_json::{Value, json}; -use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader}; -use tokio::process::{Child, ChildStdin, ChildStdout, Command}; - -struct StdioClient { - child: Child, - stdin: ChildStdin, - stdout: BufReader, -} - -impl StdioClient { - async fn start(database_url: &str, hmac_secret: &str, token: &str) -> Self { - let mut child = Command::new(env!("CARGO_BIN_EXE_vcms")) - .args(["mcp", "stdio"]) - .env("DATABASE_URL", database_url) - .env("HMAC_SECRET", hmac_secret) - .env("VCMS_MCP_TOKEN", token) - .env("DB_MIN_CONNECTIONS", "1") - .env("DB_MAX_CONNECTIONS", "2") - .env("RUST_LOG", "cms=debug,vcms=debug") - .env("LOG_FORMAT", "pretty") - .stdin(Stdio::piped()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .kill_on_drop(true) - .spawn() - .expect("spawn cms mcp stdio"); - - let stdin = child.stdin.take().expect("child stdin"); - let stdout = BufReader::new(child.stdout.take().expect("child stdout")); - Self { child, stdin, stdout } - } - - /// Start the stdio process the way a real MCP client does: it knows only - /// `VCMS_HOME` and `VCMS_MCP_TOKEN`. No `DATABASE_URL` / `HMAC_SECRET` in the - /// environment, and a working directory with no `.env` — so both the database - /// path and the HMAC secret must be resolved from `~/.vcms`. - async fn start_from_home(home: &std::path::Path, token: &str) -> Self { - let mut child = Command::new(env!("CARGO_BIN_EXE_vcms")) - .args(["mcp", "stdio"]) - .env_remove("DATABASE_URL") - .env_remove("HMAC_SECRET") - .env("VCMS_HOME", home) - .env("VCMS_MCP_TOKEN", token) - .env("DB_MIN_CONNECTIONS", "1") - .env("DB_MAX_CONNECTIONS", "2") - .env("RUST_LOG", "cms=debug,vcms=debug") - .env("LOG_FORMAT", "pretty") - .current_dir(home) - .stdin(Stdio::piped()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .kill_on_drop(true) - .spawn() - .expect("spawn cms mcp stdio"); - - let stdin = child.stdin.take().expect("child stdin"); - let stdout = BufReader::new(child.stdout.take().expect("child stdout")); - Self { child, stdin, stdout } - } - - async fn request(&mut self, id: u64, method: &str, params: Option) -> Value { - let mut request = json!({ - "jsonrpc": "2.0", - "id": id, - "method": method, - }); - if let Some(params) = params { - request["params"] = params; - } - self.stdin - .write_all(format!("{}\n", request).as_bytes()) - .await - .expect("write MCP request"); - self.stdin.flush().await.expect("flush MCP request"); - - let mut line = String::new(); - self.stdout.read_line(&mut line).await.expect("read MCP response"); - serde_json::from_str(&line).unwrap_or_else(|error| panic!("stdout was not pure MCP JSON: {line:?}: {error}")) - } - - async fn notify(&mut self, method: &str) { - let request = json!({"jsonrpc": "2.0", "method": method}); - self.stdin - .write_all(format!("{}\n", request).as_bytes()) - .await - .expect("write MCP notification"); - self.stdin.flush().await.expect("flush MCP notification"); - } - - async fn close(mut self) -> (std::process::ExitStatus, String) { - drop(self.stdin); - let mut stderr = self.child.stderr.take().expect("child stderr"); - let status = self.child.wait().await.expect("wait for stdio process"); - let mut logs = String::new(); - stderr.read_to_string(&mut logs).await.expect("read stderr logs"); - (status, logs) - } -} - -async fn setup_database(permission: AccessTokenPermission) -> (tempfile::TempDir, Config, String, String) { - let directory = tempfile::tempdir().expect("temp directory"); - let database_path = directory.path().join("vcms.db"); - let database_url = format!("sqlite://{}", database_path.to_string_lossy().replace('\\', "/")); - let hmac_secret = "stdio-test-hmac-secret".to_string(); - let config = Config { - database_url, - hmac_secret: hmac_secret.clone(), - bcrypt_cost: 4, - db_max_connections: 2, - db_min_connections: 1, - db_acquire_timeout_secs: 5, - db_idle_timeout_secs: 60, - ..Config::default() - }; - let pool = init_db_with_config(&config).await.expect("initialize database"); - let repository = Repository::new(&pool); - let password_hash = bcrypt::hash("password", 4).expect("password hash"); - repository - .user - .create("stdio-user", "stdio", "stdio@example.com", &password_hash) - .await - .expect("create user"); - repository - .site - .create("stdio-site", "Stdio Site", "filesystem", "stdio-user") - .await - .expect("create site"); - let token = AccessTokenService::new(repository.access_token.clone(), hmac_secret.clone(), 4) - .create_site_token("stdio-site", "stdio".to_string(), permission, Some("stdio-user")) - .await - .expect("create token"); - drop(repository); - drop(pool); - - (directory, config, token.id, token.token) -} - -/// Provision a `~/.vcms`-style home: a `secrets.toml` and a database at the -/// default location (`/vcms.db`), with a site token signed by the persisted -/// HMAC secret. Mirrors what `vcms serve` leaves behind on first run. -async fn setup_home_instance(home: &std::path::Path) -> String { - let hmac_secret = "home-instance-hmac-secret".to_string(); - std::fs::write(home.join("secrets.toml"), format!("hmac_secret = \"{hmac_secret}\"\n")) - .expect("write secrets.toml"); - - let database_path = home.join("vcms.db"); - let database_url = format!("sqlite://{}", database_path.to_string_lossy().replace('\\', "/")); - let config = Config { - database_url, - hmac_secret: hmac_secret.clone(), - bcrypt_cost: 4, - db_max_connections: 2, - db_min_connections: 1, - db_acquire_timeout_secs: 5, - db_idle_timeout_secs: 60, - ..Config::default() - }; - let pool = init_db_with_config(&config).await.expect("initialize database"); - let repository = Repository::new(&pool); - let password_hash = bcrypt::hash("password", 4).expect("password hash"); - repository - .user - .create("home-user", "home", "home@example.com", &password_hash) - .await - .expect("create user"); - repository - .site - .create("home-site", "Home Site", "filesystem", "home-user") - .await - .expect("create site"); - let token = AccessTokenService::new(repository.access_token.clone(), hmac_secret, 4) - .create_site_token( - "home-site", - "home".to_string(), - AccessTokenPermission::Write, - Some("home-user"), - ) - .await - .expect("create token"); - drop(repository); - drop(pool); - - token.token -} - -/// Proves the MCP-stdio fix: a cwd-less client process authenticates and serves -/// using only `VCMS_HOME` + `VCMS_MCP_TOKEN`, with the database path and HMAC -/// secret resolved from `~/.vcms` rather than a cwd `.env`. -#[tokio::test] -async fn stdio_resolves_database_and_secret_from_cms_home() { - let home = tempfile::tempdir().expect("temp home"); - let token = setup_home_instance(home.path()).await; - - let mut client = StdioClient::start_from_home(home.path(), &token).await; - initialize(&mut client).await; - - let site = client - .request(2, "tools/call", Some(json!({"name": "get_site", "arguments": {}}))) - .await; - assert_eq!(site["result"]["isError"], false, "stdio must authenticate from ~/.vcms"); - - let (status, logs) = client.close().await; - assert!(status.success(), "stdio process should exit cleanly; logs:\n{logs}"); -} - -async fn initialize(client: &mut StdioClient) { - let response = client - .request( - 1, - "initialize", - Some(json!({ - "protocolVersion": "2024-11-05", - "capabilities": {}, - "clientInfo": {"name": "stdio-test", "version": "1.0"} - })), - ) - .await; - assert_eq!(response["result"]["serverInfo"]["name"], "cms"); - client.notify("notifications/initialized").await; -} - -#[tokio::test] -async fn stdio_serves_mcp_with_protocol_only_stdout_and_lifecycle_logs_on_stderr() { - let (_directory, config, _token_id, token) = setup_database(AccessTokenPermission::Write).await; - let mut client = StdioClient::start(&config.database_url, &config.hmac_secret, &token).await; - initialize(&mut client).await; - - let tools = client.request(2, "tools/list", None).await; - assert!( - tools["result"]["tools"] - .as_array() - .is_some_and(|tools| !tools.is_empty()) - ); - - let site = client - .request(3, "tools/call", Some(json!({"name": "get_site", "arguments": {}}))) - .await; - assert_eq!(site["result"]["isError"], false); - - let (status, logs) = client.close().await; - assert!(status.success()); - assert!(logs.contains("Starting standalone MCP stdio process")); - assert!(logs.contains("no migrations were run")); - assert!(logs.contains("MCP stdio transport active")); - assert!(logs.contains("exited cleanly")); -} - -#[tokio::test] -async fn stdio_enforces_read_permission_and_revalidates_deleted_token() { - let (_directory, config, token_id, token) = setup_database(AccessTokenPermission::Read).await; - let mut client = StdioClient::start(&config.database_url, &config.hmac_secret, &token).await; - initialize(&mut client).await; - - let mutation = client - .request( - 2, - "tools/call", - Some(json!({ - "name": "update_site", - "arguments": {"name": "Forbidden"} - })), - ) - .await; - assert_eq!(mutation["result"]["isError"], true); - - let pool = DbPool::from_existing_with_config(&config).await.expect("open database"); - let repository = Repository::new(&pool); - repository - .access_token - .delete(&token_id, "stdio-site") - .await - .expect("delete token"); - drop(repository); - drop(pool); - - let rejected = client.request(3, "tools/list", None).await; - assert!(rejected.get("error").is_some(), "deleted token must be rejected"); - - let (status, _logs) = client.close().await; - assert!(status.success()); -} diff --git a/apps/backend/tests/rest/files_tests.rs b/apps/backend/tests/rest/files_tests.rs index 043376a9..b921a904 100644 --- a/apps/backend/tests/rest/files_tests.rs +++ b/apps/backend/tests/rest/files_tests.rs @@ -342,3 +342,177 @@ async fn test_upload_file_invalid_mime_type() { error ); } + +// ── Signed-URL uploads ────────────────────────────────────────────────────── +// +// Tokens are minted directly with the lib (same HMAC secret as TestServer), so +// these tests cover the PUT endpoint without an MCP round-trip. + +const TEST_HMAC_SECRET: &str = "test-hmac-secret-integration"; + +fn mint_upload_url(server: &TestServer, site_id: &str, filename: &str, mime: &str, expiry_secs: i64) -> String { + let (_, encoded) = cms::signed_upload::SignedUploadToken::generate_with_storage_provider( + site_id, + filename, + mime, + "filesystem", + TEST_HMAC_SECRET, + expiry_secs, + ); + format!("{}/api/v1/files/upload/{}", server.base_url, encoded) +} + +#[tokio::test] +async fn test_signed_upload_happy_path() { + let server = TestServer::start().await; + let (_token, _csrf, site_id) = setup(&server).await; + let client = reqwest::Client::builder().build().unwrap(); + + let url = mint_upload_url(&server, &site_id, "signed.txt", "text/plain", 900); + let resp = client + .put(&url) + .header(reqwest::header::CONTENT_TYPE, "text/plain") + .body("signed upload body") + .send() + .await + .unwrap(); + + assert_eq!(resp.status(), 201, "{}", resp.text().await.unwrap_or_default()); + let body: Value = resp.json().await.unwrap(); + let file_id = body["id"].as_str().unwrap(); + assert_eq!(body["original_name"], "signed.txt"); + assert_eq!(body["mime_type"], "text/plain"); + assert_eq!(body["size"], 18); + + // The stored bytes are servable through the public file route. + let served = client + .get(format!("{}/api/files/{}", server.base_url, file_id)) + .send() + .await + .unwrap(); + assert_eq!(served.status(), 200); + assert_eq!(served.text().await.unwrap(), "signed upload body"); +} + +#[tokio::test] +async fn test_signed_upload_expired_token() { + let server = TestServer::start().await; + let (_token, _csrf, site_id) = setup(&server).await; + let client = reqwest::Client::builder().build().unwrap(); + + let url = mint_upload_url(&server, &site_id, "late.txt", "text/plain", -10); + let resp = client.put(&url).body("too late").send().await.unwrap(); + assert_eq!(resp.status(), 410); +} + +#[tokio::test] +async fn test_signed_upload_tampered_signature() { + let server = TestServer::start().await; + let (_token, _csrf, site_id) = setup(&server).await; + let client = reqwest::Client::builder().build().unwrap(); + + // Minted with a different secret => signature check must fail. + let (_, encoded) = cms::signed_upload::SignedUploadToken::generate_with_storage_provider( + &site_id, + "evil.txt", + "text/plain", + "filesystem", + "not-the-server-secret", + 900, + ); + let url = format!("{}/api/v1/files/upload/{}", server.base_url, encoded); + let resp = client.put(&url).body("nope").send().await.unwrap(); + assert_eq!(resp.status(), 401); +} + +#[tokio::test] +async fn test_signed_upload_single_use() { + let server = TestServer::start().await; + let (_token, _csrf, site_id) = setup(&server).await; + let client = reqwest::Client::builder().build().unwrap(); + + let url = mint_upload_url(&server, &site_id, "once.txt", "text/plain", 900); + let first = client.put(&url).body("first").send().await.unwrap(); + assert_eq!(first.status(), 201); + + let second = client.put(&url).body("second").send().await.unwrap(); + assert_eq!(second.status(), 409, "reused upload URL must be rejected"); +} + +#[tokio::test] +async fn test_signed_upload_magic_byte_mismatch() { + let server = TestServer::start().await; + let (_token, _csrf, site_id) = setup(&server).await; + let client = reqwest::Client::builder().build().unwrap(); + + // Token says PNG; body is plain text => sniffed mismatch => 400. + let url = mint_upload_url(&server, &site_id, "fake.png", "image/png", 900); + let resp = client + .put(&url) + .header(reqwest::header::CONTENT_TYPE, "image/png") + .body("definitely not a png") + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 400); +} + +#[tokio::test] +async fn test_signed_upload_content_type_header_mismatch() { + let server = TestServer::start().await; + let (_token, _csrf, site_id) = setup(&server).await; + let client = reqwest::Client::builder().build().unwrap(); + + let url = mint_upload_url(&server, &site_id, "a.txt", "text/plain", 900); + let resp = client + .put(&url) + .header(reqwest::header::CONTENT_TYPE, "application/pdf") + .body("hello") + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 400); +} + +#[tokio::test] +async fn test_signed_upload_oversize_rejected() { + let server = TestServer::start().await; + let (_token, _csrf, site_id) = setup(&server).await; + let client = reqwest::Client::builder().build().unwrap(); + + let url = mint_upload_url(&server, &site_id, "big.txt", "text/plain", 900); + // 52MB > the 50MB cap; the Content-Length fast path rejects upfront. + let body = vec![b'a'; 52 * 1024 * 1024]; + let result = client.put(&url).body(body).send().await; + match result { + Ok(resp) => assert_eq!(resp.status(), 413, "{}", resp.text().await.unwrap_or_default()), + Err(e) => assert!( + !e.is_connect() && !e.is_timeout() && !e.is_builder(), + "unexpected transport error: {e}" + ), + } +} + +/// Multipart path enforces the same sniffing as the signed path. +#[tokio::test] +async fn test_multipart_upload_magic_byte_mismatch() { + let server = TestServer::start().await; + let (token, csrf, site_id) = setup(&server).await; + let api_key = get_api_key(&server, &token, &csrf, &site_id).await; + let client = reqwest::Client::builder().build().unwrap(); + + let part = reqwest::multipart::Part::bytes(b"just some text".to_vec()) + .file_name("fake.png") + .mime_str("image/png") + .unwrap(); + let form = reqwest::multipart::Form::new().part("file", part); + + let resp = client + .post(format!("{}/files", server.base_url)) + .headers(api_key_header(&api_key)) + .multipart(form) + .send() + .await + .unwrap(); + assert_eq!(resp.status(), 400); +} diff --git a/apps/backend/tests/rest/health_tests.rs b/apps/backend/tests/rest/health_tests.rs new file mode 100644 index 00000000..e6eccc3d --- /dev/null +++ b/apps/backend/tests/rest/health_tests.rs @@ -0,0 +1,29 @@ +use crate::common::TestServer; + +#[tokio::test] +async fn health_probes_are_public_and_minimal() { + let server = TestServer::start().await; + let client = reqwest::Client::new(); + + let live = client + .get(format!("{}/health/live", server.base_url)) + .send() + .await + .unwrap(); + assert_eq!(live.status(), 200); + assert_eq!( + live.json::().await.unwrap(), + serde_json::json!({ "status": "ok" }) + ); + + let ready = client + .get(format!("{}/health/ready", server.base_url)) + .send() + .await + .unwrap(); + assert_eq!(ready.status(), 200); + assert_eq!( + ready.json::().await.unwrap(), + serde_json::json!({ "status": "ready" }) + ); +} diff --git a/apps/backend/tests/rest/main.rs b/apps/backend/tests/rest/main.rs index 80372da0..719c70b6 100644 --- a/apps/backend/tests/rest/main.rs +++ b/apps/backend/tests/rest/main.rs @@ -7,6 +7,7 @@ mod backups_tests; mod collections_tests; mod entries_tests; mod files_tests; +mod health_tests; mod roles_tests; mod singletons_tests; mod sites_tests; diff --git a/apps/dashboard/index.html b/apps/dashboard/index.html index 648bfe88..9a4eb97a 100644 --- a/apps/dashboard/index.html +++ b/apps/dashboard/index.html @@ -8,7 +8,7 @@ - CMS + Velopulent CMS @@ -17,4 +17,4 @@ - \ No newline at end of file + diff --git a/apps/dashboard/src/components/dashboard-header.tsx b/apps/dashboard/src/components/dashboard-header.tsx index 8eee43ac..f05bf383 100644 --- a/apps/dashboard/src/components/dashboard-header.tsx +++ b/apps/dashboard/src/components/dashboard-header.tsx @@ -1,5 +1,5 @@ import { Link, useNavigate } from "@tanstack/react-router"; -import { LogOut, Settings, User } from "lucide-react"; +import { BookOpen, LogOut, Settings, User } from "lucide-react"; import { ModeToggle } from "@/components/theme-toggle"; import { Button, buttonVariants } from "@/components/ui/button"; import { @@ -36,7 +36,7 @@ export function DashboardHeader() { className="size-8" /> - CMS + Velopulent CMS

@@ -78,6 +78,22 @@ export function DashboardHeader() { + + ( + + )} + > + + Documentation + + + navigate({ to: "/account" })}> Account diff --git a/apps/dashboard/src/components/sidebar/nav-user.tsx b/apps/dashboard/src/components/sidebar/nav-user.tsx index ebc082ee..51740ff5 100644 --- a/apps/dashboard/src/components/sidebar/nav-user.tsx +++ b/apps/dashboard/src/components/sidebar/nav-user.tsx @@ -4,6 +4,7 @@ import { useNavigate } from "@tanstack/react-router"; import { BadgeCheck, Bell, + BookOpen, ChevronsUpDown, CreditCard, LogOut, @@ -79,6 +80,22 @@ export function NavUser({ + + ( + + )} + > + + Documentation + + + diff --git a/packaging/README.md b/packaging/README.md new file mode 100644 index 00000000..57d35ddd --- /dev/null +++ b/packaging/README.md @@ -0,0 +1,16 @@ +# Native packaging + +This directory is source of truth for service and installer definitions. Keep native +files reviewable here; `xtask` only stages them, substitutes release metadata, invokes +platform tools, and writes artifact manifests/checksums. + +- `linux/`: shared systemd unit +- `debian/`: Debian metadata and maintainer scripts +- `rpm/`: RPM spec template +- `macos/`: launchd definition and installer scripts +- `windows/`: WiX source template +- `arch/`: post-release AUR recipe template + +GitHub releases contain portable archives plus DEB, RPM, MSI, and macOS PKG artifacts. +Arch uses the same Linux archives through a rendered, checksummed PKGBUILD submitted to +AUR. It is intentionally not represented by a `PKGBUILD.tar.gz` release attachment. diff --git a/packaging/arch/PKGBUILD.template b/packaging/arch/PKGBUILD.template new file mode 100644 index 00000000..5f94d712 --- /dev/null +++ b/packaging/arch/PKGBUILD.template @@ -0,0 +1,22 @@ +pkgname=vcms +pkgver=@VERSION@ +pkgrel=1 +pkgdesc="Velopulent CMS" +arch=('x86_64' 'aarch64') +url="https://cms.velopulent.com/docs" +license=('AGPL-3.0-or-later') +depends=('glibc') +source=('vcms.service') +source_x86_64=("vcms-${pkgver}-x86_64.tar.gz::https://github.com/velopulent/cms/releases/download/v${pkgver}/vcms-${pkgver}-linux-amd64.tar.gz") +source_aarch64=("vcms-${pkgver}-aarch64.tar.gz::https://github.com/velopulent/cms/releases/download/v${pkgver}/vcms-${pkgver}-linux-arm64.tar.gz") +sha256sums=('@SHA256_SERVICE@') +sha256sums_x86_64=('@SHA256_AMD64@') +sha256sums_aarch64=('@SHA256_ARM64@') + +package() { + local release_arch=amd64 + [[ $CARCH == aarch64 ]] && release_arch=arm64 + install -Dm755 "$srcdir/vcms-${pkgver}-linux-${release_arch}/vcms" "$pkgdir/usr/bin/vcms" + install -Dm644 "$srcdir/vcms.service" "$pkgdir/usr/lib/systemd/system/vcms.service" + install -dm750 "$pkgdir/var/lib/vcms" +} diff --git a/packaging/arch/README.md b/packaging/arch/README.md new file mode 100644 index 00000000..6e8eb252 --- /dev/null +++ b/packaging/arch/README.md @@ -0,0 +1,18 @@ +# Arch Linux package + +`PKGBUILD.template` is maintained release input, not a GitHub release artifact. +After portable Linux archives exist, release tooling replaces the version and both +archive checksums, generates `.SRCINFO`, and submits the reviewed recipe to AUR. + +Render it with: + +```bash +cargo run -p xtask -- arch-render --version 1.2.3 \ + --amd64 dist/packages/vcms-1.2.3-linux-amd64.tar.gz \ + --arm64 dist/packages/vcms-1.2.3-linux-arm64.tar.gz \ + --out dist/arch +``` + +Validate generated recipes in an Arch container with `makepkg --verifysource`, +`makepkg`, and `namcap`. AUR publication remains manual until dedicated credentials +and repository ownership are configured. diff --git a/packaging/debian/control.template b/packaging/debian/control.template new file mode 100644 index 00000000..b04dd150 --- /dev/null +++ b/packaging/debian/control.template @@ -0,0 +1,9 @@ +Package: vcms +Version: @VERSION@ +Section: web +Priority: optional +Architecture: @ARCH@ +Maintainer: vcms maintainers +Homepage: https://cms.velopulent.com/docs +Description: Velopulent CMS + Velopulent CMS serves the dashboard, REST, GraphQL, gRPC, and MCP APIs. diff --git a/packaging/debian/postinst b/packaging/debian/postinst new file mode 100644 index 00000000..5e2c6d8a --- /dev/null +++ b/packaging/debian/postinst @@ -0,0 +1,30 @@ +#!/bin/sh +set -eu +getent group vcms >/dev/null || groupadd --system vcms +id vcms >/dev/null 2>&1 || useradd --system --gid vcms --home-dir /var/lib/vcms --shell /usr/sbin/nologin vcms +install -d -o vcms -g vcms -m 0750 /var/lib/vcms /var/lib/vcms/storage /var/lib/vcms/backups /var/lib/vcms/logs /var/lib/vcms/search +if systemctl is-active --quiet vcms.service; then systemctl stop vcms.service; fi +config=/var/lib/vcms/config.toml +sample=/var/lib/vcms/config.toml.sample +if [ -L "$config" ] || [ -L "$sample" ]; then + echo "Refusing to install Velopulent CMS configuration through a symlink." >&2 + exit 1 +fi +if [ ! -e "$config" ]; then + tmp=$(mktemp /var/lib/vcms/.config.toml.XXXXXX) + trap 'rm -f "$tmp"' EXIT HUP INT TERM + install -o vcms -g vcms -m 0640 "$sample" "$tmp" + mv -T "$tmp" "$config" + trap - EXIT HUP INT TERM +fi +chown --no-dereference vcms:vcms "$config" +systemctl daemon-reload >/dev/null 2>&1 || true +systemctl enable vcms.service >/dev/null 2>&1 || true +if [ -f /run/vcms-was-active ]; then + systemctl restart vcms.service + rm -f /run/vcms-was-active + echo "Velopulent CMS upgraded and restarted." +else + echo "Velopulent CMS installed but not started. Run: vcms doctor && systemctl start vcms" +fi +echo "Velopulent CMS security: change admin@cms.local password immediately after first start." diff --git a/packaging/debian/postrm b/packaging/debian/postrm new file mode 100644 index 00000000..c9be054c --- /dev/null +++ b/packaging/debian/postrm @@ -0,0 +1,8 @@ +#!/bin/sh +set -eu +systemctl daemon-reload >/dev/null 2>&1 || true +if [ "$1" = purge ]; then + echo "State remains in /var/lib/vcms. Remove it manually only when irreversible purge is intended." +else + echo "vcms config and state were preserved." +fi diff --git a/packaging/debian/preinst b/packaging/debian/preinst new file mode 100644 index 00000000..673feb2d --- /dev/null +++ b/packaging/debian/preinst @@ -0,0 +1,3 @@ +#!/bin/sh +set -eu +if [ "$1" = upgrade ] && systemctl is-active --quiet vcms.service; then touch /run/vcms-was-active; fi diff --git a/packaging/debian/prerm b/packaging/debian/prerm new file mode 100644 index 00000000..0a43b1fa --- /dev/null +++ b/packaging/debian/prerm @@ -0,0 +1,6 @@ +#!/bin/sh +set -eu +if [ "$1" = remove ] || [ "$1" = deconfigure ]; then + systemctl stop vcms.service >/dev/null 2>&1 || true + systemctl disable vcms.service >/dev/null 2>&1 || true +fi diff --git a/packaging/linux/vcms.service b/packaging/linux/vcms.service new file mode 100644 index 00000000..51ae9bd2 --- /dev/null +++ b/packaging/linux/vcms.service @@ -0,0 +1,36 @@ +[Unit] +Description=Velopulent CMS +Documentation=https://cms.velopulent.com/docs +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=vcms +Group=vcms +Environment=VCMS_HOME=/var/lib/vcms +ExecStart=/usr/bin/vcms serve +Restart=on-failure +RestartSec=5s +TimeoutStartSec=90s +TimeoutStopSec=45s +NoNewPrivileges=true +PrivateTmp=true +PrivateDevices=true +ProtectSystem=strict +ProtectHome=true +ProtectKernelTunables=true +ProtectKernelModules=true +ProtectKernelLogs=true +ProtectControlGroups=true +ProtectClock=true +RestrictSUIDSGID=true +RestrictRealtime=true +LockPersonality=true +MemoryDenyWriteExecute=true +CapabilityBoundingSet= +AmbientCapabilities= +ReadWritePaths=/var/lib/vcms + +[Install] +WantedBy=multi-user.target diff --git a/packaging/macos/com.velopulent.vcms.plist b/packaging/macos/com.velopulent.vcms.plist new file mode 100644 index 00000000..5d87916d --- /dev/null +++ b/packaging/macos/com.velopulent.vcms.plist @@ -0,0 +1,18 @@ + + + + + Labelcom.velopulent.vcms + ProgramArguments + /usr/local/bin/vcmsserve--config/Library/Application Support/vcms/config.toml + EnvironmentVariables + VCMS_HOME/Library/Application Support/vcms + UserName_vcms + GroupName_vcms + RunAtLoad + KeepAliveSuccessfulExit + ProcessTypeBackground + StandardOutPath/Library/Logs/vcms/vcms.log + StandardErrorPath/Library/Logs/vcms/vcms-error.log + + diff --git a/packaging/macos/postinstall b/packaging/macos/postinstall new file mode 100644 index 00000000..cb440e88 --- /dev/null +++ b/packaging/macos/postinstall @@ -0,0 +1,25 @@ +#!/bin/sh +set -eu +if ! dscl . -read /Groups/_vcms >/dev/null 2>&1; then dseditgroup -o create -r "vcms service" _vcms; fi +if ! id _vcms >/dev/null 2>&1; then + next_uid=$(dscl . -list /Users UniqueID | awk '$2 >= 200 && $2 < 500 {print $2}' | sort -n | awk 'BEGIN{n=200} {if($1==n)n++} END{print n}') + dscl . -create /Users/_vcms + dscl . -create /Users/_vcms UniqueID "$next_uid" + dscl . -create /Users/_vcms PrimaryGroupID "$(dscl . -read /Groups/_vcms PrimaryGroupID | awk '{print $2}')" + dscl . -create /Users/_vcms UserShell /usr/bin/false + dscl . -create /Users/_vcms NFSHomeDirectory /var/empty +fi +install -d -o _vcms -g _vcms -m 0750 "/Library/Application Support/vcms" "/Library/Application Support/vcms/storage" "/Library/Application Support/vcms/backups" "/Library/Application Support/vcms/search" +if [ ! -f "/Library/Application Support/vcms/config.toml" ]; then + cp "/Library/Application Support/vcms/config.toml.sample" "/Library/Application Support/vcms/config.toml" +fi +chown _vcms:_vcms "/Library/Application Support/vcms/config.toml" +install -d -o _vcms -g _vcms -m 0750 /Library/Logs/vcms +chown root:wheel /Library/LaunchDaemons/com.velopulent.vcms.plist +chmod 0644 /Library/LaunchDaemons/com.velopulent.vcms.plist +if [ -f /var/run/vcms-was-active ]; then + rm -f /var/run/vcms-was-active + launchctl bootstrap system /Library/LaunchDaemons/com.velopulent.vcms.plist + launchctl kickstart -k system/com.velopulent.vcms +fi +echo "vcms installed but not started. Run: vcms doctor && sudo launchctl bootstrap system /Library/LaunchDaemons/com.velopulent.vcms.plist" diff --git a/packaging/macos/preinstall b/packaging/macos/preinstall new file mode 100644 index 00000000..7c20ffd4 --- /dev/null +++ b/packaging/macos/preinstall @@ -0,0 +1,6 @@ +#!/bin/sh +set -eu +if launchctl print system/com.velopulent.vcms >/dev/null 2>&1; then + touch /var/run/vcms-was-active + launchctl bootout system/com.velopulent.vcms >/dev/null 2>&1 || true +fi diff --git a/packaging/rpm/vcms.spec.template b/packaging/rpm/vcms.spec.template new file mode 100644 index 00000000..6164b527 --- /dev/null +++ b/packaging/rpm/vcms.spec.template @@ -0,0 +1,42 @@ +Name: vcms +Version: @VERSION@ +Release: 1%{?dist} +Summary: Velopulent CMS +License: AGPL-3.0-or-later +URL: https://cms.velopulent.com/docs +Source0: %{name}-%{version}.tar.gz + +%description +Velopulent CMS serves the dashboard, REST, GraphQL, gRPC, and MCP APIs. + +%prep +%setup -q +%build +%install +mkdir -p %{buildroot} +cp -a . %{buildroot}/ + +%pre +getent group vcms >/dev/null || groupadd -r vcms +getent passwd vcms >/dev/null || useradd -r -g vcms -d /var/lib/vcms -s /sbin/nologin vcms + +%post +[ -f /var/lib/vcms/config.toml ] || cp /var/lib/vcms/config.toml.sample /var/lib/vcms/config.toml +chown vcms:vcms /var/lib/vcms/config.toml +systemctl daemon-reload >/dev/null 2>&1 || true +systemctl enable vcms.service >/dev/null 2>&1 || true +if [ "$1" -gt 1 ]; then systemctl try-restart vcms.service >/dev/null 2>&1 || true; fi +echo "vcms installed but not started. Run: vcms doctor && systemctl start vcms" + +%preun +if [ "$1" = 0 ]; then systemctl stop vcms.service >/dev/null 2>&1 || true; systemctl disable vcms.service >/dev/null 2>&1 || true; fi +%postun +systemctl daemon-reload >/dev/null 2>&1 || true +echo "vcms config and state were preserved." + +%files +/usr/bin/vcms +/usr/lib/systemd/system/vcms.service +%config(noreplace) /var/lib/vcms/config.toml.sample +%config(noreplace) /var/lib/vcms/config.toml +%dir %attr(0750,vcms,vcms) /var/lib/vcms diff --git a/packaging/windows/vcms.wxs.template b/packaging/windows/vcms.wxs.template new file mode 100644 index 00000000..e1e96f54 --- /dev/null +++ b/packaging/windows/vcms.wxs.template @@ -0,0 +1,26 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml new file mode 100644 index 00000000..d56f5667 --- /dev/null +++ b/xtask/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "xtask" +version = "0.1.0" +edition = "2024" +publish = false + +[dependencies] +sha2 = "0.11" +uuid = { version = "1", features = ["v5"] } diff --git a/xtask/README.md b/xtask/README.md new file mode 100644 index 00000000..11eb2f62 --- /dev/null +++ b/xtask/README.md @@ -0,0 +1,186 @@ +# `xtask`: build and package Velopulent CMS + +`xtask` contains repository automation for producing Velopulent CMS release packages. +It builds the dashboard and backend, stages the package payload, invokes the native +packaging tool, and writes release metadata. + +Run every command from the repository root: + +```console +cargo run -p xtask -- help +``` + +Generated packages are written to `dist/packages/`. Temporary staging files are +written to `target/package/`. + +## Supported outputs + +| Target OS | `--kind` | Output | +| --- | --- | --- | +| Linux | `portable` | `vcms--linux-.tar.gz` | +| Linux | `deb` | `vcms__.deb` | +| Linux | `rpm` | `vcms--1..rpm` | +| macOS | `portable` | `vcms--macos-.tar.gz` | +| macOS | `pkg` | `vcms--macos-.pkg` | +| Windows | `portable` | `vcms--windows-.exe` | +| Windows | `msi` | `vcms--windows-.msi` | + +Every package run also creates: + +- `vcms----manifest.json` +- `vcms----SHA256SUMS` + +`host` and `all` currently mean the same thing: build the portable artifact plus +all native package formats supported by the selected host OS. + +## Build packages + +Build every package supported by the current machine: + +```console +cargo run -p xtask --release -- package --kind host +``` + +Build one format: + +```console +# Linux +cargo run -p xtask --release -- package --kind portable +cargo run -p xtask --release -- package --kind deb +cargo run -p xtask --release -- package --kind rpm + +# macOS +cargo run -p xtask --release -- package --kind pkg + +# Windows +cargo run -p xtask --release -- package --kind msi +``` + +Specify release version and architecture when needed: + +```console +cargo run -p xtask --release -- package \ + --kind all \ + --version 1.2.3 \ + --target-os linux \ + --arch amd64 +``` + +PowerShell equivalent: + +```powershell +cargo run -p xtask --release -- package ` + --kind all ` + --version 1.2.3 ` + --target-os windows ` + --arch amd64 +``` + +If `--version` is omitted, `xtask` uses `GITHUB_REF_NAME` with a leading `v` +removed, or falls back to the version in `apps/backend/Cargo.toml`. + +If `--target-os` or `--arch` is omitted, current host values are used. Accepted +architecture aliases include `amd64`, `x86_64`, `x64`, `arm64`, and `aarch64`. +`darwin` is accepted as an alias for `macos`. + +## Build prerequisites + +All real package builds require Rust, Cargo, Bun, and project dependencies. By +default, `xtask` runs: + +1. `bun run build:dashboard` +2. `cargo build --release --locked` for the backend +3. selected package builder + +Native package tools are also required: + +| Format | Required tool | Build host | +| --- | --- | --- | +| Portable Unix archive | `tar` | Linux or macOS | +| DEB | `dpkg-deb` | Linux | +| RPM | `rpmbuild` and `tar` | Linux | +| macOS PKG | `pkgbuild` and `productbuild` | macOS | +| MSI | WiX v7 `wix` CLI | Windows | + +MSI automation uses WiX v7's direct `-acceptEula wix7` build switch. Running an +MSI build therefore accepts the WiX v7 OSMF EULA for that invocation. +Install the matching UI extension once before local MSI builds: + +```console +wix eula accept wix7 +wix extension add --global WixToolset.UI.wixext/7.0.0 +``` + +Real native packages must be built on their matching OS. `--target-os` selects and +labels the target; it does not cross-compile the backend binary. + +## Reuse an existing release binary + +Use `--skip-build` when dashboard and backend were already built. `xtask` expects +the binary at `target/release/vcms` or `target/release/vcms.exe`: + +```console +bun run build:dashboard +cargo build --release --locked --manifest-path apps/backend/Cargo.toml +cargo run -p xtask --release -- package --kind all --skip-build +``` + +`--skip-build` does not skip native packaging tools. + +## Deterministic dry-runs + +Dry-runs stage templates and create deterministic placeholder artifacts without +building the application or invoking external packaging tools. They are useful for +CI and package-definition validation: + +```console +cargo run -p xtask -- package-dry-run \ + --kind all \ + --version 1.2.3 \ + --target-os linux \ + --arch amd64 +``` + +These two forms are equivalent: + +```console +cargo run -p xtask -- package-dry-run --kind all +cargo run -p xtask -- package --kind all --dry-run +``` + +Dry-run files in `dist/packages/` are placeholders, not installable packages. + +## Render the Arch Linux package recipe + +Arch support is distributed as a `PKGBUILD` consuming published Linux portable +archives. First produce or download both Linux archives, then render the recipe: + +```console +cargo run -p xtask -- arch-render \ + --version 1.2.3 \ + --amd64 dist/packages/vcms-1.2.3-linux-amd64.tar.gz \ + --arm64 dist/packages/vcms-1.2.3-linux-arm64.tar.gz \ + --out dist/arch +``` + +This writes: + +```text +dist/arch/PKGBUILD +dist/arch/vcms.service +``` + +The renderer calculates SHA-256 hashes from the supplied archives and inserts them +into `PKGBUILD`. The rendered recipe is intended for the AUR workflow; it is not a +release archive itself. + +## Test `xtask` + +```console +cargo test -p xtask --locked +cargo fmt --all -- --check +``` + +Package and service source files live in [`../packaging`](../packaging/README.md). +Keep native definitions there; keep `xtask` focused on validation, staging, +orchestration, artifact naming, manifests, and checksums. diff --git a/xtask/src/arch.rs b/xtask/src/arch.rs new file mode 100644 index 00000000..4549179e --- /dev/null +++ b/xtask/src/arch.rs @@ -0,0 +1,49 @@ +use std::fs; +use std::path::{Path, PathBuf}; + +use crate::artifact::sha256_file; +use crate::model::Result; +use crate::shared::write_file; + +const TEMPLATE: &str = include_str!("../../packaging/arch/PKGBUILD.template"); +const SERVICE: &str = include_str!("../../packaging/linux/vcms.service"); + +pub fn render(version: &str, amd64: &Path, arm64: &Path, output: &Path) -> Result> { + if !amd64.is_file() || !arm64.is_file() { + return Err("both Linux release archives must exist before rendering the Arch recipe".into()); + } + fs::create_dir_all(output)?; + let service_path = output.join("vcms.service"); + write_file(&service_path, SERVICE)?; + let pkgbuild = TEMPLATE + .replace("@VERSION@", version) + .replace("@SHA256_AMD64@", &sha256_file(amd64)?) + .replace("@SHA256_ARM64@", &sha256_file(arm64)?) + .replace("@SHA256_SERVICE@", &sha256_file(&service_path)?); + let pkgbuild_path = output.join("PKGBUILD"); + write_file(&pkgbuild_path, pkgbuild)?; + Ok(vec![pkgbuild_path, service_path]) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn renders_complete_recipe_with_real_hashes() { + let root = std::env::temp_dir().join(format!("vcms-arch-render-{}", std::process::id())); + let _ = fs::remove_dir_all(&root); + fs::create_dir_all(&root).unwrap(); + let amd64 = root.join("amd64.tar.gz"); + let arm64 = root.join("arm64.tar.gz"); + fs::write(&amd64, b"amd64").unwrap(); + fs::write(&arm64, b"arm64").unwrap(); + let output = root.join("out"); + render("1.2.3", &amd64, &arm64, &output).unwrap(); + let recipe = fs::read_to_string(output.join("PKGBUILD")).unwrap(); + assert!(recipe.contains("pkgver=1.2.3")); + assert!(!recipe.contains('@')); + assert!(!recipe.contains("'SKIP'")); + fs::remove_dir_all(root).unwrap(); + } +} diff --git a/xtask/src/artifact.rs b/xtask/src/artifact.rs new file mode 100644 index 00000000..7c1f2c7f --- /dev/null +++ b/xtask/src/artifact.rs @@ -0,0 +1,85 @@ +use std::fs; +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; + +use sha2::{Digest, Sha256}; + +use crate::model::{Context, PACKAGE_NAME, Result}; + +pub fn write_metadata(context: &Context, artifacts: &[PathBuf]) -> Result> { + let stem = format!( + "vcms-{}-{}-{}", + context.version, + context.target_os.as_str(), + context.arch.as_str() + ); + let manifest = context.out_dir.join(format!("{stem}-manifest.json")); + let checksums = context.out_dir.join(format!("{stem}-SHA256SUMS")); + + let entries = artifacts + .iter() + .map(|artifact| { + let name = artifact.file_name().unwrap_or_default().to_string_lossy(); + Ok(format!( + " {{\"name\":\"{}\",\"sha256\":\"{}\"}}", + name, + sha256_file(artifact)? + )) + }) + .collect::>>()?; + fs::write( + &manifest, + format!( + "{{\n \"schema_version\": 1,\n \"name\": \"{PACKAGE_NAME}\",\n \"version\": \"{}\",\n \"target_os\": \"{}\",\n \"arch\": \"{}\",\n \"artifacts\": [\n{}\n ]\n}}\n", + context.version, + context.target_os.as_str(), + context.arch.as_str(), + entries.join(",\n") + ), + )?; + let mut checksum_file = fs::File::create(&checksums)?; + for artifact in artifacts { + writeln!( + checksum_file, + "{} {}", + sha256_file(artifact)?, + artifact.file_name().unwrap_or_default().to_string_lossy() + )?; + } + Ok(vec![manifest, checksums]) +} + +pub fn sha256_file(path: &Path) -> Result { + let mut file = fs::File::open(path)?; + let mut hasher = Sha256::new(); + let mut buffer = [0_u8; 16 * 1024]; + loop { + let read = file.read(&mut buffer)?; + if read == 0 { + break; + } + hasher.update(&buffer[..read]); + } + let mut output = String::with_capacity(64); + for byte in hasher.finalize() { + use std::fmt::Write as _; + write!(&mut output, "{byte:02x}")?; + } + Ok(output) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn sha256_is_stable() { + let path = std::env::temp_dir().join(format!("vcms-xtask-sha-{}", std::process::id())); + fs::write(&path, b"vcms").unwrap(); + assert_eq!( + sha256_file(&path).unwrap(), + "fac1f37320e181f97fa88d454816f82ab8fefcc875d338a57bf9f4a974c9ffb7" + ); + fs::remove_file(path).unwrap(); + } +} diff --git a/xtask/src/cli.rs b/xtask/src/cli.rs new file mode 100644 index 00000000..99f26a8e --- /dev/null +++ b/xtask/src/cli.rs @@ -0,0 +1,156 @@ +use std::env; +use std::fs; +use std::path::PathBuf; + +use crate::arch; +use crate::model::{Architecture, PackageKind, Result, TargetOs}; +use crate::package::{self, PackageRequest}; + +pub fn run() -> Result<()> { + let mut raw = env::args().skip(1); + let command = raw.next().unwrap_or_else(|| "help".to_owned()); + if command == "arch-render" { + return render_arch(raw); + } + let args = Args::parse(std::iter::once(command).chain(raw))?; + match args.command.as_str() { + "package" => package::run(args.into()), + "package-dry-run" => package::run(PackageRequest { + dry_run: true, + ..args.into() + }), + "help" | "-h" | "--help" => { + print_help(); + Ok(()) + } + other => Err(format!("unknown xtask command '{other}'").into()), + } +} + +fn render_arch(mut raw: I) -> Result<()> +where + I: Iterator, +{ + let mut version = None; + let mut amd64 = None; + let mut arm64 = None; + let mut output = PathBuf::from("dist/arch"); + while let Some(arg) = raw.next() { + match arg.as_str() { + "--version" => version = Some(need_value("--version", raw.next())?), + "--amd64" => amd64 = Some(PathBuf::from(need_value("--amd64", raw.next())?)), + "--arm64" => arm64 = Some(PathBuf::from(need_value("--arm64", raw.next())?)), + "--out" => output = PathBuf::from(need_value("--out", raw.next())?), + other => return Err(format!("unknown arch-render argument '{other}'").into()), + } + } + let files = arch::render( + version.as_deref().ok_or("arch-render needs --version")?, + amd64.as_deref().ok_or("arch-render needs --amd64")?, + arm64.as_deref().ok_or("arch-render needs --arm64")?, + &output, + )?; + for file in files { + println!("{}", file.display()); + } + Ok(()) +} + +#[derive(Debug, Clone)] +struct Args { + command: String, + kind: PackageKind, + version: String, + target_os: TargetOs, + arch: Architecture, + dry_run: bool, + skip_build: bool, +} + +impl Args { + fn parse(mut raw: I) -> Result + where + I: Iterator, + { + let command = raw.next().unwrap_or_else(|| "help".to_owned()); + let mut output = Self { + command, + kind: PackageKind::Host, + version: match env::var("GITHUB_REF_NAME").ok() { + Some(value) => value.strip_prefix('v').unwrap_or(&value).to_owned(), + None => backend_version()?, + }, + target_os: TargetOs::parse(env::consts::OS)?, + arch: Architecture::parse(env::consts::ARCH)?, + dry_run: false, + skip_build: false, + }; + while let Some(arg) = raw.next() { + match arg.as_str() { + "--kind" => output.kind = PackageKind::parse(&need_value("--kind", raw.next())?)?, + "--version" => output.version = need_value("--version", raw.next())?, + "--target-os" => output.target_os = TargetOs::parse(&need_value("--target-os", raw.next())?)?, + "--arch" => output.arch = Architecture::parse(&need_value("--arch", raw.next())?)?, + "--dry-run" => output.dry_run = true, + "--skip-build" => output.skip_build = true, + "-h" | "--help" => output.command = "help".to_owned(), + other => return Err(format!("unknown package argument '{other}'").into()), + } + } + Ok(output) + } +} + +impl From for PackageRequest { + fn from(value: Args) -> Self { + Self { + kind: value.kind, + version: value.version, + target_os: value.target_os, + arch: value.arch, + dry_run: value.dry_run, + skip_build: value.skip_build, + } + } +} + +fn need_value(name: &str, value: Option) -> Result { + value.ok_or_else(|| format!("{name} needs a value").into()) +} + +fn backend_version() -> Result { + let root = package::repo_root()?; + let cargo = fs::read_to_string(root.join("apps/backend/Cargo.toml"))?; + cargo + .lines() + .find_map(|line| { + line.trim() + .strip_prefix("version = ") + .map(|value| value.trim_matches('"').to_owned()) + }) + .ok_or_else(|| "apps/backend/Cargo.toml has no version".into()) +} + +fn print_help() { + println!( + "usage:\n cargo run -p xtask -- package [--kind host|portable|deb|rpm|msi|pkg|all] [--version X] [--target-os linux|macos|windows] [--arch amd64|arm64] [--dry-run] [--skip-build]\n cargo run -p xtask -- arch-render --version X --amd64 ARCHIVE --arm64 ARCHIVE [--out DIR]" + ); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_typed_target_aliases() { + let args = Args::parse( + ["package", "--target-os", "darwin", "--arch", "aarch64", "--dry-run"] + .into_iter() + .map(str::to_owned), + ) + .unwrap(); + assert_eq!(args.target_os, TargetOs::Macos); + assert_eq!(args.arch, Architecture::Arm64); + assert!(args.dry_run); + } +} diff --git a/xtask/src/linux.rs b/xtask/src/linux.rs new file mode 100644 index 00000000..6bdd1f37 --- /dev/null +++ b/xtask/src/linux.rs @@ -0,0 +1,158 @@ +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::Command; + +use crate::model::{Architecture, Context, Result, TargetOs}; +use crate::shared::{config_sample, copy_binary, find_file, reset_dir, run_cmd, write_file}; + +pub fn build_deb(context: &Context) -> Result { + context.require_os(TargetOs::Linux, "deb")?; + let root = reset_dir(&context.work_dir.join("deb"))?; + let package_root = root.join("root"); + let debian = package_root.join("DEBIAN"); + fs::create_dir_all(&debian)?; + stage_payload(context, &package_root, "lib/systemd/system/vcms.service")?; + write_file(&debian.join("control"), deb_control(context))?; + write_file(&debian.join("preinst"), deb_preinst())?; + write_file(&debian.join("postinst"), deb_postinst())?; + write_file(&debian.join("prerm"), deb_prerm())?; + write_file(&debian.join("postrm"), deb_postrm())?; + let artifact = context + .out_dir + .join(format!("vcms_{}_{}.deb", context.version, deb_arch(context.arch))); + if context.dry_run { + fs::write(&artifact, b"dry-run deb\n")?; + } else { + run_cmd(Command::new("chmod").arg("755").args([ + debian.join("preinst"), + debian.join("postinst"), + debian.join("prerm"), + debian.join("postrm"), + ]))?; + run_cmd( + Command::new("dpkg-deb") + .args(["--build", "--root-owner-group"]) + .arg(&package_root) + .arg(&artifact), + )?; + } + Ok(artifact) +} + +pub fn build_rpm(context: &Context) -> Result { + context.require_os(TargetOs::Linux, "rpm")?; + let root = reset_dir(&context.work_dir.join("rpm"))?; + let rpmbuild = root.join("rpmbuild"); + for name in ["BUILD", "BUILDROOT", "RPMS", "SOURCES", "SPECS", "SRPMS"] { + fs::create_dir_all(rpmbuild.join(name))?; + } + let source_root = root.join(format!("vcms-{}", context.version)); + stage_payload(context, &source_root, "usr/lib/systemd/system/vcms.service")?; + let source_tar = rpmbuild + .join("SOURCES") + .join(format!("vcms-{}.tar.gz", context.version)); + if context.dry_run { + fs::write(&source_tar, b"dry-run rpm source\n")?; + } else { + run_cmd( + Command::new("tar") + .arg("-czf") + .arg(&source_tar) + .arg("-C") + .arg(&root) + .arg(format!("vcms-{}", context.version)), + )?; + } + let spec = rpmbuild.join("SPECS/vcms.spec"); + write_file(&spec, rpm_spec(context))?; + let artifact = context + .out_dir + .join(format!("vcms-{}-1.{}.rpm", context.version, rpm_arch(context.arch))); + if context.dry_run { + fs::write(&artifact, b"dry-run rpm\n")?; + } else { + run_cmd( + Command::new("rpmbuild") + .arg("-bb") + .arg("--define") + .arg(format!("_topdir {}", rpmbuild.display())) + .arg(&spec), + )?; + fs::copy(find_file(&rpmbuild.join("RPMS"), "rpm")?, &artifact)?; + } + Ok(artifact) +} + +fn stage_payload(context: &Context, root: &Path, service_path: &str) -> Result<()> { + copy_binary(context, &root.join("usr/bin"))?; + fs::create_dir_all(root.join("var/lib/vcms"))?; + write_file(&root.join("var/lib/vcms/config.toml.sample"), config_sample())?; + write_file(&root.join(service_path), systemd_service()) +} + +fn systemd_service() -> &'static str { + include_str!("../../packaging/linux/vcms.service") +} + +fn deb_control(context: &Context) -> String { + include_str!("../../packaging/debian/control.template") + .replace("@VERSION@", &context.version) + .replace("@ARCH@", deb_arch(context.arch)) +} + +fn deb_preinst() -> &'static str { + include_str!("../../packaging/debian/preinst") +} + +fn deb_postinst() -> &'static str { + include_str!("../../packaging/debian/postinst") +} + +fn deb_prerm() -> &'static str { + include_str!("../../packaging/debian/prerm") +} + +fn deb_postrm() -> &'static str { + include_str!("../../packaging/debian/postrm") +} + +fn rpm_spec(context: &Context) -> String { + include_str!("../../packaging/rpm/vcms.spec.template").replace("@VERSION@", &context.version) +} + +const fn deb_arch(arch: Architecture) -> &'static str { + match arch { + Architecture::Amd64 => "amd64", + Architecture::Arm64 => "arm64", + } +} + +const fn rpm_arch(arch: Architecture) -> &'static str { + match arch { + Architecture::Amd64 => "x86_64", + Architecture::Arm64 => "aarch64", + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn service_is_hardened_and_uses_journal() { + let unit = systemd_service(); + assert!(unit.contains("ProtectSystem=strict")); + assert!(unit.contains("CapabilityBoundingSet=")); + assert!(!unit.contains("LOG_OUTPUT=file")); + } + + #[test] + fn first_install_does_not_start() { + assert!( + !deb_postinst() + .lines() + .any(|line| line.trim().starts_with("systemctl start")) + ); + assert!(deb_postinst().contains("try-restart") || deb_preinst().contains("was-active")); + } +} diff --git a/xtask/src/macos.rs b/xtask/src/macos.rs new file mode 100644 index 00000000..d1a3a916 --- /dev/null +++ b/xtask/src/macos.rs @@ -0,0 +1,89 @@ +use std::fs; +use std::path::PathBuf; +use std::process::Command; + +use crate::model::{Context, Result, TargetOs}; +use crate::shared::{config_sample, copy_binary, reset_dir, run_cmd, write_file}; + +pub fn build(context: &Context) -> Result { + context.require_os(TargetOs::Macos, "pkg")?; + let root = reset_dir(&context.work_dir.join("pkg"))?; + let payload = root.join("payload"); + copy_binary(context, &payload.join("usr/local/bin"))?; + write_file( + &payload.join("Library/Application Support/vcms/config.toml.sample"), + config_sample(), + )?; + write_file( + &payload.join("Library/LaunchDaemons/com.velopulent.vcms.plist"), + launchd_plist(), + )?; + write_file(&root.join("preinstall"), preinstall())?; + write_file(&root.join("postinstall"), postinstall())?; + let component = root.join("vcms-component.pkg"); + let artifact = context + .out_dir + .join(format!("vcms-{}-macos-{}.pkg", context.version, context.arch.as_str())); + if context.dry_run { + fs::write(&artifact, b"dry-run pkg\n")?; + } else { + run_cmd( + Command::new("chmod") + .arg("755") + .arg(root.join("preinstall")) + .arg(root.join("postinstall")), + )?; + run_cmd( + Command::new("pkgbuild") + .arg("--root") + .arg(&payload) + .arg("--scripts") + .arg(&root) + .arg("--identifier") + .arg("com.velopulent.vcms") + .arg("--version") + .arg(&context.version) + .arg(&component), + )?; + run_cmd( + Command::new("productbuild") + .arg("--package") + .arg(&component) + .arg(&artifact), + )?; + } + Ok(artifact) +} + +fn launchd_plist() -> &'static str { + include_str!("../../packaging/macos/com.velopulent.vcms.plist") +} + +fn preinstall() -> &'static str { + include_str!("../../packaging/macos/preinstall") +} + +fn postinstall() -> &'static str { + include_str!("../../packaging/macos/postinstall") +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn plist_uses_dedicated_identity_and_paths() { + let plist = launchd_plist(); + assert!(plist.contains("UserName_vcms")); + assert!(plist.contains("/Library/Application Support/vcms")); + assert!(plist.contains("StandardErrorPath")); + } + + #[test] + fn lifecycle_uses_modern_launchctl() { + assert!(preinstall().contains("launchctl bootout")); + assert!(postinstall().contains("launchctl bootstrap")); + assert!(!postinstall().contains("launchctl load")); + assert!(postinstall().contains("/Library/LaunchDaemons/com.velopulent.vcms.plist")); + } +} diff --git a/xtask/src/main.rs b/xtask/src/main.rs new file mode 100644 index 00000000..5b00ca2c --- /dev/null +++ b/xtask/src/main.rs @@ -0,0 +1,17 @@ +mod arch; +mod artifact; +mod cli; +mod linux; +mod macos; +mod model; +mod package; +mod portable; +mod shared; +mod windows; + +fn main() { + if let Err(error) = cli::run() { + eprintln!("error: {error}"); + std::process::exit(1); + } +} diff --git a/xtask/src/model.rs b/xtask/src/model.rs new file mode 100644 index 00000000..71277907 --- /dev/null +++ b/xtask/src/model.rs @@ -0,0 +1,103 @@ +use std::path::PathBuf; + +pub type Result = std::result::Result>; + +pub const APP_NAME: &str = "vcms"; +pub const PACKAGE_NAME: &str = "vcms"; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TargetOs { + Linux, + Macos, + Windows, +} + +impl TargetOs { + pub fn parse(raw: &str) -> Result { + match raw { + "linux" => Ok(Self::Linux), + "macos" | "darwin" => Ok(Self::Macos), + "windows" => Ok(Self::Windows), + other => Err(format!("unsupported target OS '{other}'").into()), + } + } + + pub const fn as_str(self) -> &'static str { + match self { + Self::Linux => "linux", + Self::Macos => "macos", + Self::Windows => "windows", + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Architecture { + Amd64, + Arm64, +} + +impl Architecture { + pub fn parse(raw: &str) -> Result { + match raw { + "amd64" | "x86_64" | "x64" => Ok(Self::Amd64), + "arm64" | "aarch64" => Ok(Self::Arm64), + other => Err(format!("unsupported architecture '{other}'").into()), + } + } + + pub const fn as_str(self) -> &'static str { + match self { + Self::Amd64 => "amd64", + Self::Arm64 => "arm64", + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PackageKind { + Host, + Portable, + Deb, + Rpm, + Msi, + Pkg, + All, +} + +impl PackageKind { + pub fn parse(raw: &str) -> Result { + match raw { + "host" => Ok(Self::Host), + "portable" => Ok(Self::Portable), + "deb" => Ok(Self::Deb), + "rpm" => Ok(Self::Rpm), + "msi" => Ok(Self::Msi), + "pkg" | "macos-pkg" => Ok(Self::Pkg), + "all" => Ok(Self::All), + other => Err(format!("unknown package kind '{other}'").into()), + } + } +} + +#[derive(Debug, Clone)] +pub struct Context { + pub out_dir: PathBuf, + pub work_dir: PathBuf, + pub version: String, + pub target_os: TargetOs, + pub arch: Architecture, + pub bin_name: String, + pub binary: PathBuf, + pub dry_run: bool, +} + +impl Context { + pub fn require_os(&self, expected: TargetOs, kind: &str) -> Result<()> { + if self.target_os == expected || self.dry_run { + Ok(()) + } else { + Err(format!("{kind} packages must be built on --target-os {}", expected.as_str()).into()) + } + } +} diff --git a/xtask/src/package.rs b/xtask/src/package.rs new file mode 100644 index 00000000..57f26aec --- /dev/null +++ b/xtask/src/package.rs @@ -0,0 +1,97 @@ +use std::env; +use std::fs; +use std::path::PathBuf; +use std::process::Command; + +use crate::artifact; +use crate::model::{Architecture, Context, PackageKind, Result, TargetOs}; +use crate::{linux, macos, portable, shared, windows}; + +#[derive(Debug, Clone)] +pub struct PackageRequest { + pub kind: PackageKind, + pub version: String, + pub target_os: TargetOs, + pub arch: Architecture, + pub dry_run: bool, + pub skip_build: bool, +} + +pub fn run(request: PackageRequest) -> Result<()> { + let root = repo_root()?; + let out_dir = root.join("dist/packages"); + let work_dir = root.join("target/package"); + fs::create_dir_all(&out_dir)?; + fs::create_dir_all(&work_dir)?; + + if !request.skip_build && !request.dry_run { + shared::run_cmd(Command::new("bun").args(["run", "build:dashboard"]).current_dir(&root))?; + shared::run_cmd( + Command::new("cargo") + .args(["build", "--release", "--locked"]) + .current_dir(root.join("apps/backend")) + .env("SKIP_DASHBOARD_BUILD", "1"), + )?; + } + + let bin_name = if request.target_os == TargetOs::Windows { + "vcms.exe" + } else { + "vcms" + } + .to_owned(); + let binary = root.join("target/release").join(&bin_name); + if !request.dry_run && !binary.is_file() { + return Err(format!("release binary missing at {}", binary.display()).into()); + } + let context = Context { + out_dir, + work_dir, + version: request.version, + target_os: request.target_os, + arch: request.arch, + bin_name, + binary, + dry_run: request.dry_run, + }; + let artifacts = build(&context, request.kind)?; + let metadata = artifact::write_metadata(&context, &artifacts)?; + for path in artifacts.into_iter().chain(metadata) { + println!("{}", path.display()); + } + Ok(()) +} + +fn build(context: &Context, kind: PackageKind) -> Result> { + match kind { + PackageKind::Portable => Ok(vec![portable::build(context)?]), + PackageKind::Deb => Ok(vec![linux::build_deb(context)?]), + PackageKind::Rpm => Ok(vec![linux::build_rpm(context)?]), + PackageKind::Msi => Ok(vec![windows::build(context)?]), + PackageKind::Pkg => Ok(vec![macos::build(context)?]), + PackageKind::Host | PackageKind::All => { + let mut output = vec![portable::build(context)?]; + match context.target_os { + TargetOs::Linux => { + output.push(linux::build_deb(context)?); + output.push(linux::build_rpm(context)?); + } + TargetOs::Macos => output.push(macos::build(context)?), + TargetOs::Windows => output.push(windows::build(context)?), + } + Ok(output) + } + } +} + +pub fn repo_root() -> Result { + let mut directory = env::current_dir()?; + loop { + if directory.join("Cargo.toml").is_file() && directory.join("apps/backend/Cargo.toml").is_file() { + return Ok(directory); + } + if !directory.pop() { + return Err("could not find repo root".into()); + } + } +} diff --git a/xtask/src/portable.rs b/xtask/src/portable.rs new file mode 100644 index 00000000..c7afd315 --- /dev/null +++ b/xtask/src/portable.rs @@ -0,0 +1,48 @@ +use std::fs; +use std::path::PathBuf; +use std::process::Command; + +use crate::model::{APP_NAME, Context, Result, TargetOs}; +use crate::shared::{copy_binary, reset_dir, run_cmd, write_file}; + +pub fn build(context: &Context) -> Result { + let name = format!( + "vcms-{}-{}-{}", + context.version, + context.target_os.as_str(), + context.arch.as_str() + ); + if context.target_os == TargetOs::Windows { + let artifact = context.out_dir.join(format!("{name}.exe")); + if context.dry_run { + fs::write(&artifact, b"dry-run portable exe\n")?; + } else { + fs::copy(&context.binary, &artifact)?; + } + Ok(artifact) + } else { + let stage = reset_dir(&context.work_dir.join(&name))?; + copy_binary(context, &stage)?; + write_file( + &stage.join("README.txt"), + format!( + "Velopulent CMS portable package\n\nRun `{APP_NAME}` directly. No service is registered. Runtime files use platform user directories unless VCMS_HOME is set.\nTarget OS: {}\n", + context.target_os.as_str() + ), + )?; + let artifact = context.out_dir.join(format!("{name}.tar.gz")); + if context.dry_run { + fs::write(&artifact, b"dry-run portable tarball\n")?; + } else { + run_cmd( + Command::new("tar") + .args(["-czf"]) + .arg(&artifact) + .arg("-C") + .arg(&context.work_dir) + .arg(&name), + )?; + } + Ok(artifact) + } +} diff --git a/xtask/src/shared.rs b/xtask/src/shared.rs new file mode 100644 index 00000000..7e7ac000 --- /dev/null +++ b/xtask/src/shared.rs @@ -0,0 +1,76 @@ +use std::ffi::OsStr; +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::Command; + +use crate::model::{Context, Result}; + +pub fn run_cmd(command: &mut Command) -> Result<()> { + let status = command.status()?; + if status.success() { + Ok(()) + } else { + Err(format!("command failed with {status:?}: {command:?}").into()) + } +} + +pub fn reset_dir(path: &Path) -> Result { + if path.exists() { + fs::remove_dir_all(path)?; + } + fs::create_dir_all(path)?; + Ok(path.to_path_buf()) +} + +pub fn write_file(path: &Path, contents: impl AsRef<[u8]>) -> Result<()> { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent)?; + } + fs::write(path, contents)?; + Ok(()) +} + +pub fn copy_binary(ctx: &Context, directory: &Path) -> Result<()> { + fs::create_dir_all(directory)?; + let destination = directory.join(&ctx.bin_name); + if ctx.dry_run { + fs::write(destination, b"dry-run vcms binary\n")?; + } else { + fs::copy(&ctx.binary, destination)?; + } + Ok(()) +} + +pub fn find_file(root: &Path, extension: &str) -> Result { + for entry in fs::read_dir(root)? { + let path = entry?.path(); + if path.is_dir() { + if let Ok(found) = find_file(&path, extension) { + return Ok(found); + } + } else if path.extension() == Some(OsStr::new(extension)) { + return Ok(path); + } + } + Err(format!("no .{extension} found in {}", root.display()).into()) +} + +pub fn config_sample() -> &'static str { + r#"bind_address = "0.0.0.0:3000" +grpc_bind_address = "0.0.0.0:50051" +max_upload_size_mb = 50 +cookie_secure = false +session_lifetime_hours = 24 +db_max_connections = 10 +rate_limit_max_requests = 100 +mcp_enabled = true +mcp_allowed_hosts = ["localhost", "127.0.0.1"] + +[log] +level = "cms=info,vcms=info,tower_http=info,axum=info" +output = "stdout" +format = "json" +annotations = false +dir = "logs" +"# +} diff --git a/xtask/src/windows.rs b/xtask/src/windows.rs new file mode 100644 index 00000000..d31822b2 --- /dev/null +++ b/xtask/src/windows.rs @@ -0,0 +1,158 @@ +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::{env, fs}; +use uuid::Uuid; + +use crate::model::{Architecture, Context, Result, TargetOs}; +use crate::shared::{config_sample, copy_binary, reset_dir, run_cmd, write_file}; + +pub fn build(context: &Context) -> Result { + context.require_os(TargetOs::Windows, "msi")?; + let root = reset_dir(&context.work_dir.join("msi"))?; + let payload = root.join("payload"); + copy_binary(context, &payload)?; + write_file(&payload.join("config.sample.toml"), config_sample())?; + write_file(&root.join("license.rtf"), license_rtf())?; + write_file(&root.join("vcms.wxs"), wix_source(context))?; + let artifact = context.out_dir.join(format!( + "vcms-{}-windows-{}.msi", + context.version, + context.arch.as_str() + )); + remove_legacy_outputs(context, &artifact)?; + if context.dry_run { + fs::write(&artifact, b"dry-run msi\n")?; + } else { + let package = root.join("vcms.msi"); + let ui_extension = wix_ui_extension()?; + run_cmd( + Command::new("wix") + .arg("build") + .args(["-acceptEula", "wix7"]) + .args(["-arch", wix_arch(context.arch)]) + .args(["-pdbtype", "none"]) + .arg("-ext") + .arg(ui_extension) + .arg(root.join("vcms.wxs")) + .arg("-out") + .arg(&package) + // WiX resolves the relative `payload\\...` paths in the source + // template from its working directory, not from the `.wxs` path. + .current_dir(&root), + )?; + fs::copy(package, &artifact)?; + } + Ok(artifact) +} + +fn wix_ui_extension() -> Result { + let profile = env::var_os("USERPROFILE").ok_or("USERPROFILE is not set")?; + let extension = + PathBuf::from(profile).join(".wix/extensions/WixToolset.UI.wixext/7.0.0/wixext7/WixToolset.UI.wixext.dll"); + if extension.is_file() { + Ok(extension) + } else { + Err("WiX UI extension missing; run `wix extension add --global WixToolset.UI.wixext/7.0.0`".into()) + } +} + +fn remove_legacy_outputs(context: &Context, artifact: &Path) -> Result<()> { + // Older builds wrote WiX's external cabinet and debug symbols directly into + // dist/packages. They are not release artifacts and can break an MSI-only upload. + for path in [context.out_dir.join("cab1.cab"), artifact.with_extension("wixpdb")] { + if path.is_file() { + fs::remove_file(path)?; + } + } + Ok(()) +} + +fn wix_arch(arch: Architecture) -> &'static str { + match arch { + Architecture::Amd64 => "x64", + Architecture::Arm64 => "arm64", + } +} + +fn wix_source(context: &Context) -> String { + include_str!("../../packaging/windows/vcms.wxs.template") + .replace("@VERSION@", &context.version) + .replace("@PRODUCT_CODE@", &product_code(context)) +} + +fn product_code(context: &Context) -> String { + const PRODUCT_NAMESPACE: Uuid = Uuid::from_u128(0x2c22e086_4486_46b5_9760_cbd71cb7adf0); + let identity = format!("vcms-msi:{}:{}", context.version, context.arch.as_str()); + Uuid::new_v5(&PRODUCT_NAMESPACE, identity.as_bytes()) + .hyphenated() + .to_string() + .to_uppercase() +} + +fn license_rtf() -> String { + let mut rtf = String::from(r#"{\rtf1\ansi\deff0{\fonttbl{\f0 Segoe UI;}}\fs18 "#); + for character in include_str!("../../LICENSE").chars() { + match character { + '\\' | '{' | '}' => { + rtf.push('\\'); + rtf.push(character); + } + '\n' => rtf.push_str("\\line\n"), + '\r' => {} + _ => rtf.push(character), + } + } + rtf.push('}'); + rtf +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::model::{Architecture, TargetOs}; + + #[test] + fn msi_starts_service_automatically_and_uses_least_privilege() { + let context = Context { + out_dir: PathBuf::new(), + work_dir: PathBuf::new(), + version: "1.2.3".into(), + target_os: TargetOs::Windows, + arch: Architecture::Amd64, + bin_name: "vcms.exe".into(), + binary: PathBuf::new(), + dry_run: true, + }; + let source = wix_source(&context); + assert!(source.contains(r#"Start="auto""#)); + assert!(source.contains(r#""# + )); + assert!(source.contains("ProgramDataVcms")); + assert!(source.contains(r#""#)); + assert!(source.contains(r#"ProductCode=""#)); + assert!(!source.contains("@PRODUCT_CODE@")); + assert!(source.contains(r#"