From f56dd7036e9a35f9b3c978831a5953d9f9038946 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Sat, 6 Jun 2026 13:58:57 -0500 Subject: [PATCH 1/4] feat: render Atlas Operator AtlasSchema resources from desired-state SQL MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add distributed_tooling::render_atlas_schema — a pure producer that wraps desired-state schema SQL (e.g. DistributedProjectManifest::sql_statements) into an AtlasSchema (db.atlasgo.io/v1alpha1) custom resource. DB URL via a Secret reference (GitOps) or inline (dev), optional devURL, SQL as a literal block scalar. The caller prints/redirects the result anywhere (stdout → any file or a separate schema repo); the crate deliberately does not pick a .gitops/ location. Implements [[tasks/atlas-operator-schema-gitops]] Co-Authored-By: Claude Opus 4.8 --- distributed_tooling/src/atlas.rs | 207 +++++++++++++++++++++++++++++++ distributed_tooling/src/lib.rs | 6 + 2 files changed, 213 insertions(+) create mode 100644 distributed_tooling/src/atlas.rs diff --git a/distributed_tooling/src/atlas.rs b/distributed_tooling/src/atlas.rs new file mode 100644 index 0000000..d33a1d2 --- /dev/null +++ b/distributed_tooling/src/atlas.rs @@ -0,0 +1,207 @@ +//! Atlas Operator schema-resource generation. Pure: wraps desired-state schema +//! SQL into an `AtlasSchema` custom resource (YAML) for the ariga +//! [atlas-operator]. The result is plain text the caller prints/writes wherever +//! it wants (e.g. stdout → any file, or a separate schema repo) — this crate +//! intentionally does **not** decide a `.gitops/` location for it. +//! +//! The desired-state SQL (e.g. `DistributedProjectManifest::sql_statements`) goes +//! into `spec.schema.sql`; the operator diffs the live database against it and +//! applies the change. +//! +//! [atlas-operator]: https://github.com/ariga/atlas-operator + +use crate::ScaffoldError; + +/// How the generated `AtlasSchema` reaches its target database URL. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum AtlasDatabaseUrl { + /// Reference a key in a Kubernetes `Secret` (`spec.urlFrom.secretKeyRef`). + /// The GitOps-friendly choice — no credentials in the manifest. + SecretKeyRef { + /// Secret name. + name: String, + /// Key within the secret holding the connection URL. + key: String, + }, + /// Inline connection URL (`spec.url`). Convenient for dev; avoid committing + /// real credentials this way. + Inline(String), +} + +/// Input for [`render_atlas_schema`]. Plain data — the caller maps its flags onto +/// this and supplies the desired-state schema SQL. +#[derive(Clone, Debug)] +pub struct AtlasSchemaSpec { + /// `metadata.name` of the AtlasSchema resource. + pub name: String, + /// Optional `metadata.namespace`. + pub namespace: Option, + /// Target database connection. + pub database: AtlasDatabaseUrl, + /// Optional `spec.devURL` — a scratch database Atlas uses to plan changes. + pub dev_url: Option, + /// Desired-state schema SQL placed verbatim into `spec.schema.sql`. + pub sql: String, +} + +/// Render an `AtlasSchema` (`db.atlasgo.io/v1alpha1`) resource as YAML. +/// +/// Returns an error for an empty name or empty schema SQL (nothing to apply), or +/// an incompletely specified database reference. +pub fn render_atlas_schema(spec: &AtlasSchemaSpec) -> Result { + let name = spec.name.trim(); + if name.is_empty() { + return Err(ScaffoldError::new("AtlasSchema name must not be empty")); + } + if spec.sql.trim().is_empty() { + return Err(ScaffoldError::new( + "AtlasSchema has no schema SQL to apply (no tables registered?)", + )); + } + + let mut out = String::new(); + out.push_str("apiVersion: db.atlasgo.io/v1alpha1\n"); + out.push_str("kind: AtlasSchema\n"); + out.push_str("metadata:\n"); + out.push_str(&format!(" name: {name}\n")); + if let Some(namespace) = trimmed_non_empty(spec.namespace.as_deref()) { + out.push_str(&format!(" namespace: {namespace}\n")); + } + + out.push_str("spec:\n"); + match &spec.database { + AtlasDatabaseUrl::SecretKeyRef { name: secret, key } => { + let secret = secret.trim(); + let key = key.trim(); + if secret.is_empty() || key.is_empty() { + return Err(ScaffoldError::new( + "AtlasSchema secret reference needs both a secret name and a key", + )); + } + out.push_str(" urlFrom:\n"); + out.push_str(" secretKeyRef:\n"); + out.push_str(&format!(" name: {secret}\n")); + out.push_str(&format!(" key: {key}\n")); + } + AtlasDatabaseUrl::Inline(url) => { + let url = url.trim(); + if url.is_empty() { + return Err(ScaffoldError::new( + "AtlasSchema inline database URL is empty", + )); + } + out.push_str(&format!(" url: {}\n", yaml_quote(url))); + } + } + + if let Some(dev_url) = trimmed_non_empty(spec.dev_url.as_deref()) { + out.push_str(&format!(" devURL: {}\n", yaml_quote(dev_url))); + } + + out.push_str(" schema:\n"); + out.push_str(" sql: |\n"); + for line in spec.sql.trim_end().lines() { + if line.is_empty() { + out.push('\n'); + } else { + out.push_str(" "); + out.push_str(line); + out.push('\n'); + } + } + + Ok(out) +} + +fn trimmed_non_empty(value: Option<&str>) -> Option<&str> { + value.map(str::trim).filter(|value| !value.is_empty()) +} + +/// Quote a value as a double-quoted YAML scalar. A JSON string literal is valid +/// YAML, so this safely escapes URLs that contain `:`, `@`, etc. +fn yaml_quote(value: &str) -> String { + serde_json::to_string(value).expect("string serialization should succeed") +} + +#[cfg(test)] +mod tests { + use super::*; + + fn secret_spec() -> AtlasSchemaSpec { + AtlasSchemaSpec { + name: "orders".to_string(), + namespace: None, + database: AtlasDatabaseUrl::SecretKeyRef { + name: "orders-db".to_string(), + key: "url".to_string(), + }, + dev_url: None, + sql: "CREATE TABLE orders (id text PRIMARY KEY);".to_string(), + } + } + + #[test] + fn renders_secret_ref_resource_with_indented_sql() { + let yaml = render_atlas_schema(&secret_spec()).unwrap(); + assert!(yaml.contains("apiVersion: db.atlasgo.io/v1alpha1\n")); + assert!(yaml.contains("kind: AtlasSchema\n")); + assert!(yaml.contains(" name: orders\n")); + assert!( + yaml.contains(" urlFrom:\n secretKeyRef:\n name: orders-db\n key: url\n") + ); + // SQL is a literal block scalar, each line indented under `sql: |`. + assert!(yaml.contains(" sql: |\n CREATE TABLE orders (id text PRIMARY KEY);\n")); + // No namespace / devURL emitted when unset. + assert!(!yaml.contains("namespace:")); + assert!(!yaml.contains("devURL:")); + } + + #[test] + fn namespace_and_dev_url_are_optional_and_quoted() { + let mut spec = secret_spec(); + spec.namespace = Some("data".to_string()); + spec.dev_url = Some("docker://postgres/16/dev".to_string()); + let yaml = render_atlas_schema(&spec).unwrap(); + assert!(yaml.contains(" namespace: data\n")); + assert!(yaml.contains(" devURL: \"docker://postgres/16/dev\"\n")); + } + + #[test] + fn inline_url_is_quoted_to_survive_special_characters() { + let mut spec = secret_spec(); + spec.database = + AtlasDatabaseUrl::Inline("postgres://u:p@host:5432/db?sslmode=disable".to_string()); + let yaml = render_atlas_schema(&spec).unwrap(); + assert!(yaml.contains(" url: \"postgres://u:p@host:5432/db?sslmode=disable\"\n")); + assert!(!yaml.contains("urlFrom:")); + } + + #[test] + fn multi_statement_sql_keeps_every_line_indented() { + let mut spec = secret_spec(); + spec.sql = "CREATE TABLE a (id text);\nCREATE TABLE b (id text);".to_string(); + let yaml = render_atlas_schema(&spec).unwrap(); + assert!(yaml.contains(" CREATE TABLE a (id text);\n CREATE TABLE b (id text);\n")); + } + + #[test] + fn empty_name_or_sql_is_rejected() { + let mut blank_name = secret_spec(); + blank_name.name = " ".to_string(); + assert!(render_atlas_schema(&blank_name).is_err()); + + let mut blank_sql = secret_spec(); + blank_sql.sql = "\n \n".to_string(); + assert!(render_atlas_schema(&blank_sql).is_err()); + } + + #[test] + fn incomplete_secret_ref_is_rejected() { + let mut spec = secret_spec(); + spec.database = AtlasDatabaseUrl::SecretKeyRef { + name: "orders-db".to_string(), + key: " ".to_string(), + }; + assert!(render_atlas_schema(&spec).is_err()); + } +} diff --git a/distributed_tooling/src/lib.rs b/distributed_tooling/src/lib.rs index 4732f12..bffaf6c 100644 --- a/distributed_tooling/src/lib.rs +++ b/distributed_tooling/src/lib.rs @@ -10,9 +10,15 @@ //! A CLI such as `hops-cli` maps its flags to a [`ServiceScaffoldSpec`], calls //! this crate, then decides where to write files, whether to overwrite, and //! whether to run the [`PostCreateAction`]s (e.g. `gh repo create`). +//! +//! The crate also renders standalone deployment artifacts from already-known +//! inputs — see [`render_atlas_schema`], which wraps desired-state schema SQL into +//! an `AtlasSchema` resource for the caller to emit anywhere (e.g. stdout). +mod atlas; mod generate; +pub use atlas::{render_atlas_schema, AtlasDatabaseUrl, AtlasSchemaSpec}; pub use generate::{generate_service_scaffold, package_name}; /// What to scaffold. The pure input to [`generate_service_scaffold`]. From f2d49238614a8be382a1a2f0ddf1f6459254dfc8 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Sat, 6 Jun 2026 14:30:08 -0500 Subject: [PATCH 2/4] =?UTF-8?q?feat:=20distributed=5Fcli=20(dsvc)=20?= =?UTF-8?q?=E2=80=94=20fold=20generation=20in,=20add=20schema=20--format?= =?UTF-8?q?=20atlas?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Make Distributed's service tooling a single in-workspace crate that is both a binary (`dsvc`) and a library, eliminating cross-repo release coordination: - Fold the former `distributed_tooling` crate (pure scaffold + Atlas generation) into `distributed_cli` as internal `generate`/`atlas` modules; the public generation API is re-exported from the crate root. - Add the command surface (`cli` module): scaffold / describe / schema, ported from hops-cli's service adapter, with `run(&ServiceArgs)` as the dispatcher. - `dsvc schema --format atlas` renders an AtlasSchema resource to stdout (flag-configured: --name/--namespace/--db-secret/--db-secret-key/--db-url/ --dev-url); default --format sql is unchanged. - The library exposes `ServiceArgs` + `run` so another CLI (hops) can mount the commands under `hops service` and dispatch — re-exporting, not reimplementing, so a new flag here reaches hops on a plain `cargo update`. - Publish workflow: replace publish-tooling with publish-cli. distributed_tooling 1.5.0 stays on crates.io for the already-merged hops-cli until it migrates to depend on distributed_cli. Implements [[tasks/atlas-operator-schema-gitops]] Co-Authored-By: Claude Opus 4.8 --- .github/workflows/on-v-tag-publish.yaml | 10 +- Cargo.toml | 2 +- distributed_cli/Cargo.toml | 19 + .../src/atlas.rs | 0 distributed_cli/src/cli.rs | 1074 +++++++++++++++++ .../src/generate/github.rs | 0 .../src/generate/gitops.rs | 0 .../src/generate/mod.rs | 0 .../src/generate/names.rs | 0 .../src/generate/service_crate.rs | 0 .../src/lib.rs | 33 +- distributed_cli/src/main.rs | 19 + distributed_tooling/Cargo.toml | 10 - 13 files changed, 1138 insertions(+), 29 deletions(-) create mode 100644 distributed_cli/Cargo.toml rename {distributed_tooling => distributed_cli}/src/atlas.rs (100%) create mode 100644 distributed_cli/src/cli.rs rename {distributed_tooling => distributed_cli}/src/generate/github.rs (100%) rename {distributed_tooling => distributed_cli}/src/generate/gitops.rs (100%) rename {distributed_tooling => distributed_cli}/src/generate/mod.rs (100%) rename {distributed_tooling => distributed_cli}/src/generate/names.rs (100%) rename {distributed_tooling => distributed_cli}/src/generate/service_crate.rs (100%) rename {distributed_tooling => distributed_cli}/src/lib.rs (82%) create mode 100644 distributed_cli/src/main.rs delete mode 100644 distributed_tooling/Cargo.toml diff --git a/.github/workflows/on-v-tag-publish.yaml b/.github/workflows/on-v-tag-publish.yaml index 4dabfd1..501310d 100644 --- a/.github/workflows/on-v-tag-publish.yaml +++ b/.github/workflows/on-v-tag-publish.yaml @@ -17,14 +17,14 @@ jobs: manifest_path: distributed_macros/Cargo.toml cargo_publish_args: "--locked" - # distributed_tooling is a standalone crate (only depends on serde_json), so it - # publishes independently of the macros/core crates. - publish-tooling: + # distributed_cli (the `dsvc` binary + generation library) has no internal + # workspace dependencies, so it publishes independently of the macros/core crates. + publish-cli: uses: unbounded-tech/workflows-rust/.github/workflows/publish.yaml@feat/cargo-publish secrets: crates_io_token: ${{ secrets.CRATES_IO_TOKEN }} with: - manifest_path: distributed_tooling/Cargo.toml + manifest_path: distributed_cli/Cargo.toml cargo_publish_args: "--locked" publish: @@ -37,7 +37,7 @@ jobs: cargo_publish_args: "--locked" release: - needs: [publish, publish-tooling] + needs: [publish, publish-cli] permissions: contents: write uses: unbounded-tech/workflow-simple-release/.github/workflows/workflow.yaml@main diff --git a/Cargo.toml b/Cargo.toml index efd5108..6d08543 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace] -members = ["distributed_macros", "distributed_tooling"] +members = ["distributed_macros", "distributed_cli"] resolver = "2" [workspace.package] diff --git a/distributed_cli/Cargo.toml b/distributed_cli/Cargo.toml new file mode 100644 index 0000000..82a46ae --- /dev/null +++ b/distributed_cli/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "distributed_cli" +description = "The `dsvc` CLI for Distributed services: scaffold projects, describe their manifest, and render schema artifacts (SQL or Atlas Operator resources). Also a library so other CLIs (e.g. hops) can mount its commands." +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true + +[lib] +path = "src/lib.rs" + +[[bin]] +name = "dsvc" +path = "src/main.rs" + +[dependencies] +clap = { version = "4", features = ["derive"] } +serde = { version = "1", features = ["derive"] } +serde_json = "1" diff --git a/distributed_tooling/src/atlas.rs b/distributed_cli/src/atlas.rs similarity index 100% rename from distributed_tooling/src/atlas.rs rename to distributed_cli/src/atlas.rs diff --git a/distributed_cli/src/cli.rs b/distributed_cli/src/cli.rs new file mode 100644 index 0000000..fe68621 --- /dev/null +++ b/distributed_cli/src/cli.rs @@ -0,0 +1,1074 @@ +//! The `dsvc` command surface: clap types plus the [`run`] dispatcher. Generation +//! lives in the crate's `generate`/`atlas` modules; this module maps flags onto +//! those types and owns the filesystem / process side effects (writing files, +//! running `gh`, compiling the manifest harness). +//! +//! `hops` mounts [`ServiceArgs`] under `hops service` and dispatches with [`run`], +//! re-exporting the commands rather than reimplementing them. + +use clap::{Args, Subcommand, ValueEnum}; +use serde::Deserialize; +use std::error::Error; +use std::ffi::OsString; +use std::fs; +use std::path::{Component, Path, PathBuf}; +use std::process::Command; + +use crate::{ + generate_service_scaffold, package_name, render_atlas_schema, AtlasDatabaseUrl, + AtlasSchemaSpec, BusTarget, FileMode, GeneratedFile, GithubRepo, GitopsPromoteTarget, + PostCreateAction, ServiceScaffoldSpec, ServiceTransport, StoreTarget, +}; + +const DISTRIBUTED_MANIFEST_SCHEMA_VERSION: u64 = 1; + +#[derive(Args, Debug)] +pub struct ServiceArgs { + #[command(subcommand)] + pub command: ServiceCommands, +} + +#[derive(Subcommand, Debug)] +pub enum ServiceCommands { + /// Scaffold a new Distributed microservice crate + #[command(alias = "create")] + Scaffold(ScaffoldArgs), + /// Print a service's Distributed project manifest as JSON + Describe(DescribeArgs), + /// Render schema artifacts (SQL or an Atlas Operator resource) from a manifest + Schema(SchemaArgs), +} + +#[derive(Args, Debug)] +pub struct ScaffoldArgs { + /// Service/package name to scaffold + pub name: String, + /// Output directory. Defaults to ./. + #[arg(long)] + pub path: Option, + /// Service framework to scaffold + #[arg(long, value_enum, default_value = "distributed")] + pub framework: Framework, + /// Compatibility alias for scaffold kind, e.g. distributed-microsvc. + #[arg(long)] + pub kind: Option, + /// Runtime transport to scaffold + #[arg(long, value_enum, default_value = "http")] + pub transport: Transport, + /// Compatibility shortcut for --transport http. + #[arg(long)] + pub http: bool, + /// Compatibility shortcut for --transport knative. + #[arg(long)] + pub knative: bool, + /// Model aggregate to scaffold. May be repeated. + #[arg(long)] + pub model: Vec, + /// Generate placeholder read-model modules and register them in distributed_manifest(). + #[arg(long)] + pub read_models: bool, + /// Command handler to scaffold. May be repeated. + #[arg(long)] + pub command: Vec, + /// Event handler to scaffold. May be repeated. + #[arg(long)] + pub event: Vec, + /// Message bus backend to scaffold. + #[arg(long, value_enum)] + pub bus: Option, + /// Generate a Helm deploy chart under .gitops/deploy. + #[arg(long)] + pub gitops: bool, + /// Generate a GitOps promotion chart for Argo CD or Flux. + #[arg(long, value_enum)] + pub gitops_promote: Option, + /// GitHub repository to create and configure with release workflows. + #[arg(long, value_name = "OWNER/REPO")] + pub github: Option, + /// GitOps preview environment repository to promote pull-request previews into. + #[arg(long, value_name = "OWNER/REPO")] + pub github_preview: Option, + /// GitOps permanent environment repository to promote version tags into. + #[arg(long, value_name = "OWNER/REPO")] + pub github_promote: Option, + /// Read-model/schema storage target + #[arg( + long, + alias = "storage", + visible_alias = "storage", + visible_alias = "read-model", + value_enum, + default_value = "postgres" + )] + pub store: Store, + /// Path to the local Distributed crate. + #[arg(long)] + pub distributed_path: Option, + /// Overwrite generated files in an existing directory. + #[arg(long)] + pub force: bool, +} + +#[derive(Args, Debug)] +pub struct DescribeArgs { + /// Service project directory. Defaults to the current directory. + #[arg(long, default_value = ".")] + pub path: PathBuf, + /// Cargo.toml for the target service. Overrides --path. + #[arg(long)] + pub manifest_path: Option, + /// Cargo package to inspect when the manifest belongs to a workspace. + #[arg(long)] + pub package: Option, + /// Comma-delimited feature list for the target service. + #[arg(long, value_delimiter = ',')] + pub features: Vec, + /// Disable default features on the target service dependency. + #[arg(long)] + pub no_default_features: bool, + /// Manifest function to call. Defaults to ::distributed_manifest. + #[arg(long)] + pub entrypoint: Option, + /// Output format. + #[arg(long, value_enum, default_value = "json")] + pub format: ManifestFormat, + /// Path to the local Distributed crate. + #[arg(long)] + pub distributed_path: Option, +} + +#[derive(Args, Debug)] +pub struct SchemaArgs { + /// Service project directory. Defaults to the current directory. + #[arg(long, default_value = ".")] + pub path: PathBuf, + /// Cargo.toml for the target service. Overrides --path. + #[arg(long)] + pub manifest_path: Option, + /// Cargo package to inspect when the manifest belongs to a workspace. + #[arg(long)] + pub package: Option, + /// Comma-delimited feature list for the target service. + #[arg(long, value_delimiter = ',')] + pub features: Vec, + /// Disable default features on the target service dependency. + #[arg(long)] + pub no_default_features: bool, + /// Manifest function to call. Defaults to ::distributed_manifest. + #[arg(long)] + pub entrypoint: Option, + /// SQL dialect to render. + #[arg(long, value_enum, default_value = "postgres")] + pub dialect: SchemaDialect, + /// Output artifact format. + #[arg(long, value_enum, default_value = "sql")] + pub format: SchemaFormat, + /// AtlasSchema metadata.name (required for --format atlas). + #[arg(long)] + pub name: Option, + /// AtlasSchema metadata.namespace (--format atlas). + #[arg(long)] + pub namespace: Option, + /// Kubernetes Secret holding the database URL (--format atlas, GitOps-friendly). + #[arg(long, value_name = "SECRET")] + pub db_secret: Option, + /// Key within --db-secret that holds the database URL. + #[arg(long, default_value = "url")] + pub db_secret_key: String, + /// Inline database URL (--format atlas; prefer --db-secret for GitOps). + #[arg(long)] + pub db_url: Option, + /// Atlas devURL — a scratch database used to plan changes (--format atlas). + #[arg(long)] + pub dev_url: Option, + /// Output file. Defaults to stdout. + #[arg(long, alias = "output", visible_alias = "output")] + pub out: Option, + /// Path to the local Distributed crate. + #[arg(long)] + pub distributed_path: Option, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, ValueEnum)] +pub enum Framework { + Distributed, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, ValueEnum)] +pub enum Transport { + Http, + Knative, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, ValueEnum)] +pub enum GitopsPromote { + Argo, + Flux, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, ValueEnum)] +pub enum Bus { + Rabbitmq, + Kafka, + Psql, + Nats, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, ValueEnum)] +pub enum Store { + Postgres, + Sqlite, + InMemory, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, ValueEnum)] +pub enum ManifestFormat { + Json, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, ValueEnum)] +pub enum SchemaDialect { + Postgres, + Sqlite, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, ValueEnum)] +pub enum SchemaFormat { + /// Raw migration SQL. + Sql, + /// An Atlas Operator `AtlasSchema` resource wrapping the desired-state SQL. + Atlas, +} + +// Map the CLI's clap enums onto the generation spec enums. These exist so +// `--help` / value parsing stay in the command surface while generation stays in +// the `generate`/`atlas` modules. +impl From for ServiceTransport { + fn from(transport: Transport) -> Self { + match transport { + Transport::Http => ServiceTransport::Http, + Transport::Knative => ServiceTransport::Knative, + } + } +} + +impl From for StoreTarget { + fn from(store: Store) -> Self { + match store { + Store::Postgres => StoreTarget::Postgres, + Store::Sqlite => StoreTarget::Sqlite, + Store::InMemory => StoreTarget::InMemory, + } + } +} + +impl From for BusTarget { + fn from(bus: Bus) -> Self { + match bus { + Bus::Rabbitmq => BusTarget::Rabbitmq, + Bus::Kafka => BusTarget::Kafka, + Bus::Psql => BusTarget::Psql, + Bus::Nats => BusTarget::Nats, + } + } +} + +impl From for GitopsPromoteTarget { + fn from(promote: GitopsPromote) -> Self { + match promote { + GitopsPromote::Argo => GitopsPromoteTarget::Argo, + GitopsPromote::Flux => GitopsPromoteTarget::Flux, + } + } +} + +/// Dispatch a parsed service command. The `dsvc` binary and any host CLI (e.g. +/// `hops service`) both call this. +pub fn run(args: &ServiceArgs) -> Result<(), Box> { + match &args.command { + ServiceCommands::Scaffold(scaffold) => run_scaffold(scaffold), + ServiceCommands::Describe(describe) => run_describe(describe), + ServiceCommands::Schema(schema) => run_schema(schema), + } +} + +fn run_scaffold(args: &ScaffoldArgs) -> Result<(), Box> { + validate_scaffold_kind(args.framework, args.kind.as_deref())?; + let transport = if args.http && args.knative { + return Err("--http and --knative cannot be used together".into()); + } else if args.http { + Transport::Http + } else if args.knative { + Transport::Knative + } else { + args.transport + }; + + let github = parse_optional_github_repo(args.github.as_deref(), "--github")?; + let github_preview = + parse_optional_github_repo(args.github_preview.as_deref(), "--github-preview")?; + let github_promote = + parse_optional_github_repo(args.github_promote.as_deref(), "--github-promote")?; + + // The default output directory uses the normalized package name, so derive it + // (and fail fast on an invalid name) before creating any directory. + let package_name = package_name(&args.name)?; + let output_dir = args + .path + .clone() + .unwrap_or_else(|| PathBuf::from(&package_name)); + let output_dir = absolute_path(&output_dir)?; + ensure_output_dir(&output_dir, args.force)?; + + let distributed_path = resolve_distributed_path(args.distributed_path.as_deref(), &output_dir)?; + let distributed_dependency_path = path_for_toml(&relative_path(&output_dir, &distributed_path)); + + let spec = ServiceScaffoldSpec { + name: args.name.clone(), + transport: transport.into(), + store: args.store.into(), + bus: args.bus.map(Into::into), + models: args.model.clone(), + read_models: args.read_models, + commands: args.command.clone(), + events: args.event.clone(), + distributed_dependency_path, + gitops: args.gitops, + gitops_promote: args.gitops_promote.map(Into::into), + github, + github_preview, + github_promote, + }; + + let project = generate_service_scaffold(spec)?; + for file in &project.files { + write_generated_file(&output_dir, file)?; + } + for warning in &project.warnings { + eprintln!("warning: {warning}"); + } + for action in &project.post_create_actions { + match action { + PostCreateAction::EnsureGithubRepository { repo } => ensure_github_repo(repo)?, + } + } + + println!("Scaffolded Distributed service at {}", output_dir.display()); + Ok(()) +} + +fn run_describe(args: &DescribeArgs) -> Result<(), Box> { + match args.format { + ManifestFormat::Json => { + let json = run_manifest_harness( + &HarnessOptions { + path: args.path.clone(), + manifest_path: args.manifest_path.clone(), + package: args.package.clone(), + features: args.features.clone(), + no_default_features: args.no_default_features, + entrypoint: args.entrypoint.clone(), + distributed_path: args.distributed_path.clone(), + }, + HarnessMode::DescribeJson, + )?; + let envelope: serde_json::Value = serde_json::from_str(&json)?; + validate_manifest_json(&envelope)?; + println!("{}", serde_json::to_string_pretty(&envelope)?); + Ok(()) + } + } +} + +fn run_schema(args: &SchemaArgs) -> Result<(), Box> { + let sql = run_manifest_harness( + &HarnessOptions { + path: args.path.clone(), + manifest_path: args.manifest_path.clone(), + package: args.package.clone(), + features: args.features.clone(), + no_default_features: args.no_default_features, + entrypoint: args.entrypoint.clone(), + distributed_path: args.distributed_path.clone(), + }, + HarnessMode::SchemaSql(args.dialect), + )?; + + let content = match args.format { + SchemaFormat::Sql => sql, + SchemaFormat::Atlas => render_atlas_schema(&atlas_spec_from_flags(args, sql)?)?, + }; + + if let Some(out) = &args.out { + if let Some(parent) = out.parent().filter(|parent| !parent.as_os_str().is_empty()) { + fs::create_dir_all(parent)?; + } + fs::write(out, content)?; + } else { + print!("{content}"); + } + Ok(()) +} + +/// Build an [`AtlasSchemaSpec`] from `--format atlas` flags plus the rendered +/// desired-state SQL. The database reference must be given explicitly (a Secret +/// reference for GitOps, or an inline URL for dev). +fn atlas_spec_from_flags( + args: &SchemaArgs, + sql: String, +) -> Result> { + let name = args + .name + .clone() + .ok_or("--name is required for --format atlas")?; + + let database = match (&args.db_url, &args.db_secret) { + (Some(_), Some(_)) => { + return Err("pass either --db-url or --db-secret, not both".into()); + } + (Some(url), None) => AtlasDatabaseUrl::Inline(url.clone()), + (None, Some(secret)) => AtlasDatabaseUrl::SecretKeyRef { + name: secret.clone(), + key: args.db_secret_key.clone(), + }, + (None, None) => { + return Err( + "--format atlas needs a database: pass --db-secret (GitOps) or --db-url (dev)" + .into(), + ); + } + }; + + Ok(AtlasSchemaSpec { + name, + namespace: args.namespace.clone(), + database, + dev_url: args.dev_url.clone(), + sql, + }) +} + +fn validate_scaffold_kind(framework: Framework, kind: Option<&str>) -> Result<(), Box> { + if framework != Framework::Distributed { + return Err("only --framework distributed is supported".into()); + } + + if let Some(kind) = kind { + match kind { + "distributed-microsvc" | "distributed" => {} + _ => { + return Err(format!( + "unsupported service kind `{kind}`; expected distributed-microsvc" + ) + .into()); + } + } + } + + Ok(()) +} + +fn ensure_output_dir(path: &Path, force: bool) -> Result<(), Box> { + if path.exists() { + if !path.is_dir() { + return Err(format!("{} exists and is not a directory", path.display()).into()); + } + if !force && fs::read_dir(path)?.next().is_some() { + return Err(format!( + "{} already exists and is not empty; pass --force to overwrite generated files", + path.display() + ) + .into()); + } + } + fs::create_dir_all(path)?; + Ok(()) +} + +/// Write one generated file under `output_dir`, creating parent directories and +/// honoring the optional executable mode hint. +fn write_generated_file(output_dir: &Path, file: &GeneratedFile) -> Result<(), Box> { + let path = output_dir.join(&file.path); + if let Some(parent) = path.parent() { + fs::create_dir_all(parent)?; + } + fs::write(&path, &file.contents)?; + if file.mode == Some(FileMode::Executable) { + set_executable(&path)?; + } + Ok(()) +} + +#[cfg(unix)] +fn set_executable(path: &Path) -> Result<(), Box> { + use std::os::unix::fs::PermissionsExt; + let mut perms = fs::metadata(path)?.permissions(); + perms.set_mode(perms.mode() | 0o111); + fs::set_permissions(path, perms)?; + Ok(()) +} + +#[cfg(not(unix))] +fn set_executable(_path: &Path) -> Result<(), Box> { + Ok(()) +} + +fn parse_optional_github_repo( + raw: Option<&str>, + flag: &str, +) -> Result, Box> { + raw.map(|value| { + GithubRepo::parse(value) + .map_err(|err| -> Box { format!("{flag}: {err}").into() }) + }) + .transpose() +} + +fn ensure_github_repo(repo: &GithubRepo) -> Result<(), Box> { + let slug = repo.slug(); + let view_output = Command::new("gh") + .args(["repo", "view", &slug, "--json", "nameWithOwner"]) + .output(); + + match view_output { + Ok(output) if output.status.success() => { + println!("GitHub repository {slug} already exists"); + return Ok(()); + } + Ok(_) => {} + Err(err) if err.kind() == std::io::ErrorKind::NotFound => { + return Err( + "GitHub CLI (`gh`) is not installed or not in PATH. Install it before using --github." + .into(), + ); + } + Err(err) => return Err(Box::new(err)), + } + + let output = Command::new("gh") + .args(github_repo_create_args(&slug)) + .output()?; + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + return Err(format!("gh repo create failed: {stderr}").into()); + } + println!("Created GitHub repository {slug}"); + Ok(()) +} + +fn github_repo_create_args(slug: &str) -> Vec<&str> { + vec!["repo", "create", slug, "--private"] +} + +fn validate_manifest_json(envelope: &serde_json::Value) -> Result<(), Box> { + let Some(schema_version) = envelope + .get("schema_version") + .and_then(serde_json::Value::as_u64) + else { + return Err("manifest JSON is missing numeric schema_version".into()); + }; + if schema_version != DISTRIBUTED_MANIFEST_SCHEMA_VERSION { + return Err(format!( + "unsupported Distributed manifest schema version {schema_version}; expected {DISTRIBUTED_MANIFEST_SCHEMA_VERSION}" + ) + .into()); + } + if envelope.get("project").is_none() { + return Err("manifest JSON is missing project".into()); + } + Ok(()) +} + +#[derive(Clone, Debug)] +struct HarnessOptions { + path: PathBuf, + manifest_path: Option, + package: Option, + features: Vec, + no_default_features: bool, + entrypoint: Option, + distributed_path: Option, +} + +#[derive(Clone, Copy, Debug)] +enum HarnessMode { + DescribeJson, + SchemaSql(SchemaDialect), +} + +impl HarnessMode { + fn cache_key(self) -> &'static str { + match self { + HarnessMode::DescribeJson => "describe-json", + HarnessMode::SchemaSql(SchemaDialect::Postgres) => "schema-postgres", + HarnessMode::SchemaSql(SchemaDialect::Sqlite) => "schema-sqlite", + } + } +} + +fn run_manifest_harness( + options: &HarnessOptions, + mode: HarnessMode, +) -> Result> { + let manifest_path = + resolve_target_manifest_path(&options.path, options.manifest_path.as_deref())?; + let package = cargo_package(&manifest_path, options.package.as_deref())?; + let distributed_path = + resolve_distributed_path(options.distributed_path.as_deref(), &package.directory)?; + let crate_ident = package.name.replace('-', "_"); + let entrypoint = options + .entrypoint + .clone() + .map(|entrypoint| qualify_entrypoint(&crate_ident, &entrypoint)) + .unwrap_or_else(|| Ok(format!("{crate_ident}::distributed_manifest")))?; + validate_rust_path(&entrypoint)?; + + let harness_root = package.directory.join("target/dsvc-manifest-harness"); + let harness_dir = harness_root.join(mode.cache_key()); + fs::create_dir_all(harness_dir.join("src"))?; + fs::write( + harness_dir.join("Cargo.toml"), + harness_cargo_toml( + &format!("dsvc-manifest-harness-{}", mode.cache_key()), + &crate_ident, + &package.name, + &package.directory, + &distributed_path, + &options.features, + options.no_default_features, + ), + )?; + fs::write( + harness_dir.join("src/main.rs"), + harness_main_rs(&entrypoint, mode), + )?; + + let manifest_path = harness_dir.join("Cargo.toml"); + let output = Command::new("cargo") + .args([ + "run", + "--quiet", + "--manifest-path", + manifest_path.to_string_lossy().as_ref(), + ]) + .env("CARGO_TARGET_DIR", harness_root.join("target")) + .output()?; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + return Err(format!("manifest harness failed: {stderr}").into()); + } + + Ok(String::from_utf8_lossy(&output.stdout).to_string()) +} + +fn harness_cargo_toml( + harness_package_name: &str, + crate_ident: &str, + package_name: &str, + package_dir: &Path, + distributed_path: &Path, + features: &[String], + no_default_features: bool, +) -> String { + let features = features + .iter() + .map(toml_string) + .collect::>() + .join(", "); + let default_features = if no_default_features { + ", default-features = false" + } else { + "" + }; + + format!( + r#"[package] +name = {harness_package_name} +version = "0.1.0" +edition = "2021" + +[workspace] + +[dependencies] +distributed = {{ path = {distributed_path} }} +serde_json = "1" +{crate_ident} = {{ package = {package_name}, path = {package_dir}{default_features}, features = [{features}] }} +"#, + harness_package_name = toml_string(harness_package_name), + distributed_path = toml_string(path_for_toml(distributed_path)), + package_name = toml_string(package_name), + package_dir = toml_string(path_for_toml(package_dir)), + ) +} + +fn harness_main_rs(entrypoint: &str, mode: HarnessMode) -> String { + match mode { + HarnessMode::DescribeJson => format!( + r#"fn main() {{ + let manifest = {entrypoint}(); + let envelope = distributed::DistributedManifestEnvelope::new(manifest); + println!("{{}}", serde_json::to_string_pretty(&envelope).expect("manifest should serialize")); +}} +"# + ), + HarnessMode::SchemaSql(dialect) => { + let dialect = match dialect { + SchemaDialect::Postgres => "Postgres", + SchemaDialect::Sqlite => "Sqlite", + }; + format!( + r#"fn main() {{ + let manifest = {entrypoint}(); + let envelope = distributed::DistributedManifestEnvelope::new(manifest); + let statements = envelope + .project + .sql_statements(distributed::TableSqlDialect::{dialect}) + .expect("manifest SQL should render"); + if !statements.is_empty() {{ + println!("{{}}", statements.join("\n\n")); + }} +}} +"# + ) + } + } +} + +fn resolve_target_manifest_path( + path: &Path, + manifest_path: Option<&Path>, +) -> Result> { + let manifest = if let Some(manifest_path) = manifest_path { + manifest_path.to_path_buf() + } else if path.is_dir() { + path.join("Cargo.toml") + } else { + path.to_path_buf() + }; + + if !manifest.exists() { + return Err(format!("target manifest not found: {}", manifest.display()).into()); + } + Ok(manifest.canonicalize()?) +} + +#[derive(Clone, Debug)] +struct CargoPackage { + name: String, + directory: PathBuf, +} + +fn cargo_package( + manifest_path: &Path, + package_name: Option<&str>, +) -> Result> { + let output = Command::new("cargo") + .args([ + "metadata", + "--no-deps", + "--format-version", + "1", + "--manifest-path", + manifest_path.to_string_lossy().as_ref(), + ]) + .output()?; + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + return Err(format!("cargo metadata failed: {stderr}").into()); + } + + let metadata: CargoMetadata = serde_json::from_slice(&output.stdout)?; + let selected = if let Some(package_name) = package_name { + metadata + .packages + .into_iter() + .find(|package| package.name == package_name) + .ok_or_else(|| format!("package `{package_name}` was not found in cargo metadata"))? + } else if metadata.packages.len() == 1 { + metadata + .packages + .into_iter() + .next() + .expect("single package should exist") + } else { + let manifest_path = manifest_path.canonicalize()?; + metadata + .packages + .into_iter() + .find(|package| { + Path::new(&package.manifest_path).canonicalize().ok() == Some(manifest_path.clone()) + }) + .ok_or("multiple packages found; pass --package to select one")? + }; + let manifest_path = PathBuf::from(&selected.manifest_path); + let directory = manifest_path + .parent() + .ok_or("cargo package manifest has no parent directory")? + .to_path_buf(); + + Ok(CargoPackage { + name: selected.name, + directory, + }) +} + +#[derive(Debug, Deserialize)] +struct CargoMetadata { + packages: Vec, +} + +#[derive(Debug, Deserialize)] +struct CargoMetadataPackage { + name: String, + manifest_path: String, +} + +fn resolve_distributed_path( + provided: Option<&Path>, + anchor: &Path, +) -> Result> { + if let Some(path) = provided { + return validate_distributed_path(path); + } + if let Ok(path) = std::env::var("DISTRIBUTED_PATH") { + return validate_distributed_path(Path::new(&path)); + } + + let mut roots = Vec::new(); + roots.extend(anchor.ancestors().map(Path::to_path_buf)); + roots.extend(std::env::current_dir()?.ancestors().map(Path::to_path_buf)); + + for root in roots { + for candidate in [root.clone(), root.join("distributed")] { + if candidate.join("Cargo.toml").exists() + && cargo_toml_package_name(&candidate.join("Cargo.toml")).as_deref() + == Some("distributed") + { + return Ok(candidate.canonicalize()?); + } + } + } + + Err("unable to find local Distributed crate; pass --distributed-path".into()) +} + +fn validate_distributed_path(path: &Path) -> Result> { + let path = path.canonicalize()?; + let manifest = path.join("Cargo.toml"); + if !manifest.exists() { + return Err(format!("{} does not contain Cargo.toml", path.display()).into()); + } + if cargo_toml_package_name(&manifest).as_deref() != Some("distributed") { + return Err(format!("{} is not the Distributed crate", path.display()).into()); + } + Ok(path) +} + +fn cargo_toml_package_name(path: &Path) -> Option { + let contents = fs::read_to_string(path).ok()?; + let mut in_package = false; + for line in contents.lines() { + let trimmed = line.trim(); + if trimmed == "[package]" { + in_package = true; + continue; + } + if trimmed.starts_with('[') { + in_package = false; + } + if in_package { + if let Some(value) = trimmed.strip_prefix("name") { + let value = value.trim_start(); + if let Some(value) = value.strip_prefix('=') { + return value.trim().trim_matches('"').to_string().into(); + } + } + } + } + None +} + +fn qualify_entrypoint(crate_ident: &str, entrypoint: &str) -> Result> { + let entrypoint = entrypoint.trim(); + if entrypoint.is_empty() { + return Err("entrypoint cannot be empty".into()); + } + if entrypoint.contains("::") { + Ok(entrypoint.to_string()) + } else { + Ok(format!("{crate_ident}::{entrypoint}")) + } +} + +fn validate_rust_path(path: &str) -> Result<(), Box> { + let valid = path + .split("::") + .all(|segment| !segment.is_empty() && is_rust_ident(segment)); + if valid { + Ok(()) + } else { + Err(format!("invalid Rust entrypoint path `{path}`").into()) + } +} + +fn is_rust_ident(value: &str) -> bool { + let mut chars = value.chars(); + let Some(first) = chars.next() else { + return false; + }; + (first == '_' || first.is_ascii_alphabetic()) + && chars.all(|char| char == '_' || char.is_ascii_alphanumeric()) +} + +fn absolute_path(path: &Path) -> Result> { + if path.is_absolute() { + Ok(path.to_path_buf()) + } else { + Ok(std::env::current_dir()?.join(path)) + } +} + +fn relative_path(from_dir: &Path, to: &Path) -> PathBuf { + let from = path_components(from_dir); + let to = path_components(to); + let common = from + .iter() + .zip(to.iter()) + .take_while(|(left, right)| left == right) + .count(); + let mut relative = PathBuf::new(); + for _ in common..from.len() { + relative.push(".."); + } + for component in &to[common..] { + relative.push(component); + } + if relative.as_os_str().is_empty() { + PathBuf::from(".") + } else { + relative + } +} + +fn path_components(path: &Path) -> Vec { + path.components() + .filter_map(|component| match component { + Component::Normal(value) => Some(value.to_os_string()), + _ => None, + }) + .collect() +} + +fn path_for_toml(path: &Path) -> String { + path.to_string_lossy().replace('\\', "/") +} + +fn toml_string(value: impl AsRef) -> String { + serde_json::to_string(value.as_ref()).expect("string serialization should succeed") +} + +#[cfg(test)] +mod tests { + use super::*; + + fn schema_args() -> SchemaArgs { + SchemaArgs { + path: PathBuf::from("."), + manifest_path: None, + package: None, + features: Vec::new(), + no_default_features: false, + entrypoint: None, + dialect: SchemaDialect::Postgres, + format: SchemaFormat::Atlas, + name: Some("orders".to_string()), + namespace: None, + db_secret: Some("orders-db".to_string()), + db_secret_key: "url".to_string(), + db_url: None, + dev_url: None, + out: None, + distributed_path: None, + } + } + + #[test] + fn github_repo_create_args_are_private() { + assert_eq!( + github_repo_create_args("hops-ops/test-domain"), + vec!["repo", "create", "hops-ops/test-domain", "--private"] + ); + } + + #[test] + fn optional_github_repo_reports_the_flag_on_error() { + let err = parse_optional_github_repo(Some("missing-repo"), "--github") + .expect_err("invalid repo should error"); + assert!(err.to_string().contains("--github")); + assert!(parse_optional_github_repo(None, "--github") + .unwrap() + .is_none()); + let ok = parse_optional_github_repo(Some("hops-ops/test-domain"), "--github") + .unwrap() + .unwrap(); + assert_eq!(ok.slug(), "hops-ops/test-domain"); + } + + #[test] + fn harness_is_standalone_inside_cached_target_directory() { + let cargo_toml = harness_cargo_toml( + "dsvc-manifest-harness-schema-postgres", + "todo_model", + "todo-model", + Path::new("/tmp/todo-model"), + Path::new("/tmp/distributed"), + &[], + false, + ); + + assert!(cargo_toml.contains("\n[workspace]\n")); + assert!(cargo_toml.contains("name = \"dsvc-manifest-harness-schema-postgres\"")); + } + + #[test] + fn atlas_spec_uses_secret_ref_by_default() { + let spec = atlas_spec_from_flags(&schema_args(), "CREATE TABLE orders (id text);".into()) + .expect("secret ref spec"); + assert_eq!( + spec.database, + AtlasDatabaseUrl::SecretKeyRef { + name: "orders-db".to_string(), + key: "url".to_string(), + } + ); + assert_eq!(spec.name, "orders"); + } + + #[test] + fn atlas_inline_url_when_db_url_given() { + let mut args = schema_args(); + args.db_secret = None; + args.db_url = Some("postgres://localhost/orders".to_string()); + let spec = atlas_spec_from_flags(&args, "CREATE TABLE orders (id text);".into()).unwrap(); + assert_eq!( + spec.database, + AtlasDatabaseUrl::Inline("postgres://localhost/orders".to_string()) + ); + } + + #[test] + fn atlas_requires_name_and_a_database() { + let mut no_name = schema_args(); + no_name.name = None; + assert!(atlas_spec_from_flags(&no_name, "sql".into()).is_err()); + + let mut no_db = schema_args(); + no_db.db_secret = None; + assert!(atlas_spec_from_flags(&no_db, "sql".into()).is_err()); + + let mut both = schema_args(); + both.db_url = Some("postgres://localhost/orders".to_string()); + assert!(atlas_spec_from_flags(&both, "sql".into()).is_err()); + } +} diff --git a/distributed_tooling/src/generate/github.rs b/distributed_cli/src/generate/github.rs similarity index 100% rename from distributed_tooling/src/generate/github.rs rename to distributed_cli/src/generate/github.rs diff --git a/distributed_tooling/src/generate/gitops.rs b/distributed_cli/src/generate/gitops.rs similarity index 100% rename from distributed_tooling/src/generate/gitops.rs rename to distributed_cli/src/generate/gitops.rs diff --git a/distributed_tooling/src/generate/mod.rs b/distributed_cli/src/generate/mod.rs similarity index 100% rename from distributed_tooling/src/generate/mod.rs rename to distributed_cli/src/generate/mod.rs diff --git a/distributed_tooling/src/generate/names.rs b/distributed_cli/src/generate/names.rs similarity index 100% rename from distributed_tooling/src/generate/names.rs rename to distributed_cli/src/generate/names.rs diff --git a/distributed_tooling/src/generate/service_crate.rs b/distributed_cli/src/generate/service_crate.rs similarity index 100% rename from distributed_tooling/src/generate/service_crate.rs rename to distributed_cli/src/generate/service_crate.rs diff --git a/distributed_tooling/src/lib.rs b/distributed_cli/src/lib.rs similarity index 82% rename from distributed_tooling/src/lib.rs rename to distributed_cli/src/lib.rs index bffaf6c..851e974 100644 --- a/distributed_tooling/src/lib.rs +++ b/distributed_cli/src/lib.rs @@ -1,24 +1,31 @@ -//! Deterministic service-scaffold generation for Distributed services. +//! The `dsvc` CLI for Distributed services — both a binary and a library. //! -//! This crate owns the *pure* generation rules for a Distributed service project: -//! Cargo layout, Rust source templates, manifest wiring, read-model/handler -//! defaults, GitOps/Knative inference, and GitHub workflow contents. It performs -//! **no** filesystem, network, or CLI side effects — [`generate_service_scaffold`] -//! takes a [`ServiceScaffoldSpec`] and returns a [`GeneratedProject`] describing -//! the files to write and any follow-up actions to perform. +//! It bundles two things in one crate so there is no cross-repo coordination: //! -//! A CLI such as `hops-cli` maps its flags to a [`ServiceScaffoldSpec`], calls -//! this crate, then decides where to write files, whether to overwrite, and -//! whether to run the [`PostCreateAction`]s (e.g. `gh repo create`). +//! - **Pure generation** (the [`generate`]/[`atlas`] modules): the rules for a +//! Distributed service project — Cargo layout, Rust source templates, manifest +//! wiring, GitOps/Knative inference, GitHub workflows, and Atlas schema +//! resources. These perform no I/O — [`generate_service_scaffold`] takes a +//! [`ServiceScaffoldSpec`] and returns a [`GeneratedProject`]; +//! [`render_atlas_schema`] wraps desired-state SQL into an `AtlasSchema`. +//! - **The command surface** (the [`cli`] module): the clap types and [`run`] +//! dispatcher that own the filesystem / process side effects (writing files, +//! running `gh`, compiling the manifest harness). //! -//! The crate also renders standalone deployment artifacts from already-known -//! inputs — see [`render_atlas_schema`], which wraps desired-state schema SQL into -//! an `AtlasSchema` resource for the caller to emit anywhere (e.g. stdout). +//! The `dsvc` binary parses [`ServiceArgs`] and calls [`run`]. Another CLI (e.g. +//! `hops`) can depend on this crate, mount [`ServiceArgs`] under its own +//! subcommand, and dispatch with [`run`] — re-exporting the commands rather than +//! reimplementing them. mod atlas; +mod cli; mod generate; pub use atlas::{render_atlas_schema, AtlasDatabaseUrl, AtlasSchemaSpec}; +pub use cli::{ + run, Bus, DescribeArgs, Framework, GitopsPromote, ManifestFormat, ScaffoldArgs, SchemaArgs, + SchemaDialect, SchemaFormat, ServiceArgs, ServiceCommands, Store, Transport, +}; pub use generate::{generate_service_scaffold, package_name}; /// What to scaffold. The pure input to [`generate_service_scaffold`]. diff --git a/distributed_cli/src/main.rs b/distributed_cli/src/main.rs new file mode 100644 index 0000000..c35e961 --- /dev/null +++ b/distributed_cli/src/main.rs @@ -0,0 +1,19 @@ +use clap::Parser; +use distributed_cli::ServiceArgs; + +/// The `dsvc` CLI: scaffold Distributed services, describe their manifest, and +/// render schema artifacts (SQL or Atlas Operator resources). +#[derive(Parser, Debug)] +#[command(name = "dsvc", version, about, long_about = None)] +struct Cli { + #[command(flatten)] + args: ServiceArgs, +} + +fn main() { + let cli = Cli::parse(); + if let Err(err) = distributed_cli::run(&cli.args) { + eprintln!("error: {err}"); + std::process::exit(1); + } +} diff --git a/distributed_tooling/Cargo.toml b/distributed_tooling/Cargo.toml deleted file mode 100644 index 0e5d1c3..0000000 --- a/distributed_tooling/Cargo.toml +++ /dev/null @@ -1,10 +0,0 @@ -[package] -name = "distributed_tooling" -description = "Deterministic service-scaffold and artifact generation for Distributed services. Pure (no filesystem, network, or CLI): a ServiceScaffoldSpec in, a GeneratedProject out." -version.workspace = true -edition.workspace = true -license.workspace = true -repository.workspace = true - -[dependencies] -serde_json = "1" From 172c4a6b0a17675d46db1f99939604631654ff6a Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Sat, 6 Jun 2026 15:51:20 -0500 Subject: [PATCH 3/4] test: distributed_cli integration tests + CI job MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Drive the real `dsvc` binary end-to-end: - cli_scaffold.rs: `dsvc scaffold` to a temp dir, assert the generated tree (fast; pure generation + filesystem; runs in normal `cargo test`). - cli_manifest.rs: `#[ignore]`d harness e2e — `dsvc describe`, `schema --dialect postgres`, and `schema --format atlas` against a committed `orders-service` fixture (a standalone crate with its own `[workspace]` and a `#[derive(ReadModel)]` registered in `distributed_manifest()`). Ignored by default because they compile the fixture via nested cargo. - integration-distributed-cli.yaml: reusable workflow running `cargo test -p distributed_cli -- --include-ignored`; referenced from the push-to-main pipeline and gating version-and-tag. Implements [[tasks/distributed-cli-integration-tests]] Co-Authored-By: Claude Opus 4.8 --- .../integration-distributed-cli.yaml | 23 ++++++ .../on-push-main-version-and-tag.yaml | 5 +- distributed_cli/tests/cli_manifest.rs | 70 +++++++++++++++++++ distributed_cli/tests/cli_scaffold.rs | 53 ++++++++++++++ .../tests/fixtures/orders-service/.gitignore | 2 + .../tests/fixtures/orders-service/Cargo.toml | 16 +++++ .../tests/fixtures/orders-service/src/lib.rs | 19 +++++ 7 files changed, 187 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/integration-distributed-cli.yaml create mode 100644 distributed_cli/tests/cli_manifest.rs create mode 100644 distributed_cli/tests/cli_scaffold.rs create mode 100644 distributed_cli/tests/fixtures/orders-service/.gitignore create mode 100644 distributed_cli/tests/fixtures/orders-service/Cargo.toml create mode 100644 distributed_cli/tests/fixtures/orders-service/src/lib.rs diff --git a/.github/workflows/integration-distributed-cli.yaml b/.github/workflows/integration-distributed-cli.yaml new file mode 100644 index 0000000..08d454d --- /dev/null +++ b/.github/workflows/integration-distributed-cli.yaml @@ -0,0 +1,23 @@ +name: distributed_cli Integration Tests + +# Reusable workflow: referenced via `uses: ./.github/workflows/integration-distributed-cli.yaml`. +# Runs the `dsvc` integration tests, including the `#[ignore]`d manifest-harness +# e2e tests (describe/schema) that compile a fixture service via nested cargo. +on: + workflow_call: + +jobs: + distributed-cli: + name: distributed_cli Integration Tests + runs-on: ubuntu-latest + env: + CARGO_TERM_COLOR: always + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 + with: + persist-credentials: false + - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 + with: + toolchain: stable + - name: Run distributed_cli integration tests (incl. harness e2e) + run: cargo test -p distributed_cli --verbose -- --include-ignored diff --git a/.github/workflows/on-push-main-version-and-tag.yaml b/.github/workflows/on-push-main-version-and-tag.yaml index 8b17678..c15edf8 100644 --- a/.github/workflows/on-push-main-version-and-tag.yaml +++ b/.github/workflows/on-push-main-version-and-tag.yaml @@ -29,11 +29,14 @@ jobs: kafka: uses: ./.github/workflows/integration-kafka.yaml + distributed-cli: + uses: ./.github/workflows/integration-distributed-cli.yaml + # This uses commit logs and tags from git to determine the next version number and create a tag for the release. # Some commits such as chore: will not trigger a version bump and tag; this is by design. version-and-tag: name: Version and Tag - needs: [quality, postgres, nats, rabbitmq, kafka] + needs: [quality, postgres, nats, rabbitmq, kafka, distributed-cli] uses: unbounded-tech/workflow-vnext-tag/.github/workflows/workflow.yaml@v1.21.3 secrets: DEPLOY_KEY: ${{ secrets.DEPLOY_KEY }} diff --git a/distributed_cli/tests/cli_manifest.rs b/distributed_cli/tests/cli_manifest.rs new file mode 100644 index 0000000..ebd0977 --- /dev/null +++ b/distributed_cli/tests/cli_manifest.rs @@ -0,0 +1,70 @@ +//! End-to-end tests for the manifest-harness commands (`describe` / `schema`). +//! These compile the `orders-service` fixture via a nested `cargo` build, so they +//! are `#[ignore]`d by default and run explicitly in the integration CI job +//! (`cargo test -p distributed_cli --include-ignored`). + +use std::path::{Path, PathBuf}; +use std::process::Command; + +fn distributed_root() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .expect("distributed_cli has a parent directory") + .to_path_buf() +} + +fn fixture_manifest() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/orders-service/Cargo.toml") +} + +/// Run `dsvc ` against the fixture, returning stdout. Always passes +/// `--manifest-path` and `--distributed-path` so resolution is deterministic. +fn dsvc(args: &[&str]) -> String { + let root = distributed_root(); + let manifest = fixture_manifest(); + let output = Command::new(env!("CARGO_BIN_EXE_dsvc")) + .args(args) + .args(["--manifest-path", manifest.to_str().unwrap()]) + .args(["--distributed-path", root.to_str().unwrap()]) + .output() + .expect("dsvc should run"); + assert!( + output.status.success(), + "dsvc {args:?} failed:\n{}", + String::from_utf8_lossy(&output.stderr) + ); + String::from_utf8_lossy(&output.stdout).into_owned() +} + +#[test] +#[ignore = "compiles the fixture via the manifest harness; run in the integration job"] +fn describe_emits_manifest_json() { + let json = dsvc(&["describe"]); + assert!(json.contains("\"schema_version\""), "json: {json}"); + assert!(json.contains("\"orders\""), "json: {json}"); +} + +#[test] +#[ignore = "compiles the fixture via the manifest harness; run in the integration job"] +fn schema_renders_postgres_sql() { + let sql = dsvc(&["schema", "--dialect", "postgres"]); + assert!(sql.contains("CREATE TABLE"), "sql: {sql}"); + assert!(sql.contains("orders"), "sql: {sql}"); +} + +#[test] +#[ignore = "compiles the fixture via the manifest harness; run in the integration job"] +fn schema_renders_atlas_resource() { + let yaml = dsvc(&[ + "schema", + "--format", + "atlas", + "--name", + "orders", + "--db-secret", + "orders-db", + ]); + assert!(yaml.contains("kind: AtlasSchema"), "yaml: {yaml}"); + assert!(yaml.contains("secretKeyRef"), "yaml: {yaml}"); + assert!(yaml.contains("CREATE TABLE"), "yaml: {yaml}"); +} diff --git a/distributed_cli/tests/cli_scaffold.rs b/distributed_cli/tests/cli_scaffold.rs new file mode 100644 index 0000000..0b9327d --- /dev/null +++ b/distributed_cli/tests/cli_scaffold.rs @@ -0,0 +1,53 @@ +//! Integration test for `dsvc scaffold`: drive the real binary and assert the +//! generated project tree. Pure generation + filesystem, so it is fast and needs +//! no nested compilation. + +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::Command; + +/// Repo root (the `distributed` crate) — `distributed_cli`'s parent. +fn distributed_root() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .expect("distributed_cli has a parent directory") + .to_path_buf() +} + +#[test] +fn scaffold_generates_a_service_tree() { + let out_dir = Path::new(env!("CARGO_TARGET_TMPDIR")).join("scaffold-orders"); + let _ = fs::remove_dir_all(&out_dir); + + let status = Command::new(env!("CARGO_BIN_EXE_dsvc")) + .args([ + "scaffold", + "orders", + "--path", + out_dir.to_str().unwrap(), + "--store", + "postgres", + "--gitops", + "--distributed-path", + distributed_root().to_str().unwrap(), + ]) + .status() + .expect("dsvc should run"); + assert!(status.success(), "dsvc scaffold failed"); + + for expected in [ + "Cargo.toml", + "src/lib.rs", + "src/main.rs", + "src/service.rs", + ".gitops/deploy/Chart.yaml", + ] { + assert!( + out_dir.join(expected).exists(), + "missing generated file: {expected}" + ); + } + + let cargo = fs::read_to_string(out_dir.join("Cargo.toml")).unwrap(); + assert!(cargo.contains("\"postgres\""), "Cargo.toml: {cargo}"); +} diff --git a/distributed_cli/tests/fixtures/orders-service/.gitignore b/distributed_cli/tests/fixtures/orders-service/.gitignore new file mode 100644 index 0000000..4fffb2f --- /dev/null +++ b/distributed_cli/tests/fixtures/orders-service/.gitignore @@ -0,0 +1,2 @@ +/target +/Cargo.lock diff --git a/distributed_cli/tests/fixtures/orders-service/Cargo.toml b/distributed_cli/tests/fixtures/orders-service/Cargo.toml new file mode 100644 index 0000000..16ff564 --- /dev/null +++ b/distributed_cli/tests/fixtures/orders-service/Cargo.toml @@ -0,0 +1,16 @@ +# Standalone fixture compiled by the `dsvc` manifest harness in integration tests. +# Its own `[workspace]` decouples it from the repo workspace. +[package] +name = "orders-service" +version = "0.0.0" +edition = "2021" +publish = false + +[workspace] + +[lib] +path = "src/lib.rs" + +[dependencies] +distributed = { path = "../../../.." } +serde = { version = "1", features = ["derive"] } diff --git a/distributed_cli/tests/fixtures/orders-service/src/lib.rs b/distributed_cli/tests/fixtures/orders-service/src/lib.rs new file mode 100644 index 0000000..e82eb09 --- /dev/null +++ b/distributed_cli/tests/fixtures/orders-service/src/lib.rs @@ -0,0 +1,19 @@ +//! Minimal Distributed service fixture for `dsvc` manifest-harness integration +//! tests: one read model (→ an `orders` table) registered in the project manifest. + +use distributed::{DistributedProjectManifest, ReadModel}; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize, ReadModel)] +#[table("orders")] +pub struct OrderView { + #[id("order_id")] + pub order_id: String, + pub status: String, +} + +/// The entrypoint `dsvc describe`/`dsvc schema` call by default +/// (`::distributed_manifest`). +pub fn distributed_manifest() -> DistributedProjectManifest { + DistributedProjectManifest::new("orders").read_model::() +} From 32c197f2f74e6838a46b5b21138e4fbcb8099a4d Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Sat, 6 Jun 2026 16:11:19 -0500 Subject: [PATCH 4/4] fix: validate Kubernetes name format for AtlasSchema name/namespace MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit render_atlas_schema now rejects names that aren't RFC-1123 labels (lowercase letters, digits, hyphens; no leading/trailing hyphen) for both metadata.name and metadata.namespace, instead of only checking non-empty. This fails at generation with a clear message rather than emitting YAML the API server would reject — and guards against characters (newlines, colons, quotes) that would break the document itself. Addresses CodeRabbit review on PR #74. Implements [[tasks/atlas-operator-schema-gitops]] Co-Authored-By: Claude Opus 4.8 --- distributed_cli/src/atlas.rs | 61 ++++++++++++++++++++++++++++++++++-- 1 file changed, 58 insertions(+), 3 deletions(-) diff --git a/distributed_cli/src/atlas.rs b/distributed_cli/src/atlas.rs index d33a1d2..d90ba34 100644 --- a/distributed_cli/src/atlas.rs +++ b/distributed_cli/src/atlas.rs @@ -49,10 +49,11 @@ pub struct AtlasSchemaSpec { /// Returns an error for an empty name or empty schema SQL (nothing to apply), or /// an incompletely specified database reference. pub fn render_atlas_schema(spec: &AtlasSchemaSpec) -> Result { - let name = spec.name.trim(); - if name.is_empty() { - return Err(ScaffoldError::new("AtlasSchema name must not be empty")); + validate_k8s_name(&spec.name, "AtlasSchema name")?; + if let Some(namespace) = trimmed_non_empty(spec.namespace.as_deref()) { + validate_k8s_name(namespace, "AtlasSchema namespace")?; } + let name = spec.name.trim(); if spec.sql.trim().is_empty() { return Err(ScaffoldError::new( "AtlasSchema has no schema SQL to apply (no tables registered?)", @@ -117,6 +118,32 @@ fn trimmed_non_empty(value: Option<&str>) -> Option<&str> { value.map(str::trim).filter(|value| !value.is_empty()) } +/// Validate a Kubernetes object name (RFC 1123 label: lowercase ASCII letters, +/// digits, and `-`, not starting or ending with `-`). Fails at generation time +/// with a clear message rather than emitting YAML the API server would reject — +/// and rejects names with characters (newlines, colons, quotes) that would also +/// break the document itself. +fn validate_k8s_name(value: &str, field: &str) -> Result<(), ScaffoldError> { + let name = value.trim(); + if name.is_empty() { + return Err(ScaffoldError::new(format!("{field} must not be empty"))); + } + if !name + .chars() + .all(|c| c.is_ascii_lowercase() || c.is_ascii_digit() || c == '-') + { + return Err(ScaffoldError::new(format!( + "{field} `{name}` must contain only lowercase letters, digits, and hyphens" + ))); + } + if name.starts_with('-') || name.ends_with('-') { + return Err(ScaffoldError::new(format!( + "{field} `{name}` must not start or end with a hyphen" + ))); + } + Ok(()) +} + /// Quote a value as a double-quoted YAML scalar. A JSON string literal is valid /// YAML, so this safely escapes URLs that contain `:`, `@`, etc. fn yaml_quote(value: &str) -> String { @@ -195,6 +222,34 @@ mod tests { assert!(render_atlas_schema(&blank_sql).is_err()); } + #[test] + fn invalid_kubernetes_names_are_rejected() { + for bad in [ + "Orders", + "orders_db", + "orders.db", + "-orders", + "orders-", + "a b", + ] { + let mut spec = secret_spec(); + spec.name = bad.to_string(); + assert!( + render_atlas_schema(&spec).is_err(), + "expected `{bad}` to be rejected" + ); + } + + let mut bad_ns = secret_spec(); + bad_ns.namespace = Some("Data".to_string()); + assert!(render_atlas_schema(&bad_ns).is_err()); + + // A valid RFC-1123 name still renders. + let mut ok = secret_spec(); + ok.name = "orders-2".to_string(); + assert!(render_atlas_schema(&ok).is_ok()); + } + #[test] fn incomplete_secret_ref_is_rejected() { let mut spec = secret_spec();