From 654d8f408c546ef594dd6f628b14438a14350a10 Mon Sep 17 00:00:00 2001 From: Jeff Repanich Date: Sat, 25 Jul 2026 12:33:45 -0400 Subject: [PATCH] feat: add generic repository validation commands --- Cargo.lock | 82 +++++++ Cargo.toml | 1 + README.md | 35 +++ src/main.rs | 13 + src/repository_checks.rs | 509 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 640 insertions(+) create mode 100644 src/repository_checks.rs diff --git a/Cargo.lock b/Cargo.lock index dd44b58..dce6f58 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -169,6 +169,7 @@ dependencies = [ "regex", "serde", "serde_json", + "toml", "walkdir", ] @@ -205,12 +206,24 @@ dependencies = [ "memchr", ] +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + [[package]] name = "find-msvc-tools" version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + [[package]] name = "heck" version = "0.5.0" @@ -241,6 +254,16 @@ dependencies = [ "cc", ] +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown", +] + [[package]] name = "is_terminal_polyfill" version = "1.70.2" @@ -413,6 +436,15 @@ dependencies = [ "zmij", ] +[[package]] +name = "serde_spanned" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" +dependencies = [ + "serde", +] + [[package]] name = "shlex" version = "1.3.0" @@ -436,6 +468,47 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "toml" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "toml_write", + "winnow", +] + +[[package]] +name = "toml_write" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" + [[package]] name = "unicode-ident" version = "1.0.24" @@ -580,6 +653,15 @@ dependencies = [ "windows-link", ] +[[package]] +name = "winnow" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" +dependencies = [ + "memchr", +] + [[package]] name = "zmij" version = "1.0.21" diff --git a/Cargo.toml b/Cargo.toml index 3ec8436..3d22b6b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,4 +14,5 @@ csv = "1" regex = "1" serde = { version = "1", features = ["derive"] } serde_json = "1" +toml = "0.8" walkdir = "2" diff --git a/README.md b/README.md index 2b4dd90..c21ff75 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,10 @@ Standalone Rust command-line tool with child commands for cntryl. - `validate-tests`: checks Rust tests for naming and AAA conventions. - `generate-inventory`: scans tests and benchmarks and writes an inventory report. - `summarize-benchmarks`: collects benchmark results, compares them to baseline, and writes a report. +- `validate-docs`: validates configured Markdown inventory, links, anchors, and policy text. +- `validate-benchmarks`: validates Cargo benchmark targets against documentation and workflow coverage. +- `check-module-sizes`: checks production Rust module sizes with configured thresholds and allowlists. +- `test-watchdog`: runs integration tests one at a time with per-test timeouts. ## Install @@ -36,6 +40,10 @@ Run them from the repository you want to inspect after install: cntryl-tools validate-tests cntryl-tools generate-inventory cntryl-tools summarize-benchmarks +cntryl-tools validate-docs +cntryl-tools validate-benchmarks +cntryl-tools check-module-sizes +cntryl-tools test-watchdog --suite --timeout 60 ``` `validate-tests` runs against the current directory by default, so run it from the repo you want to check. @@ -54,6 +62,33 @@ If you are already in the repo you want to inspect, just run the installed comma cntryl-tools validate-tests cntryl-tools generate-inventory cntryl-tools summarize-benchmarks +cntryl-tools validate-docs +cntryl-tools validate-benchmarks +cntryl-tools check-module-sizes +cntryl-tools test-watchdog --suite --timeout 60 +``` + +## Repository policy + +Checks use `.cntryl/repository.toml` when present. The file keeps project policy +out of the shared binary: + +```toml +[docs] +required = ["README.md", "docs/README.md"] +forbidden_paths = ["docs/archive/old.md"] +forbidden_text = ["placeholder text"] + +[benchmarks] +documentation = ["docs/development/benchmarks.md"] +workflow = ".github/workflows/bench.yml" +not_pr_gate_phrase = "not a pull-request performance gate" + +[module_sizes] +warn_lines = 1200 +max_lines = 1600 +allowlist = ["src/large_provider.rs"] +legacy_allowlist = [] ``` `summarize-benchmarks` also accepts `--product-name` and `--report-title` if you want to override the default report branding. diff --git a/src/main.rs b/src/main.rs index b37c066..175f583 100644 --- a/src/main.rs +++ b/src/main.rs @@ -11,6 +11,7 @@ mod generate_inventory; mod model; #[path = "summarize_benchmarks/report.rs"] mod report; +mod repository_checks; mod summarize_benchmarks; #[path = "summarize_benchmarks/sweep.rs"] mod sweep; @@ -34,6 +35,14 @@ enum Commands { GenerateInventory(generate_inventory::GenerateInventoryArgs), #[command(name = "summarize-benchmarks")] SummarizeBenchmarks(summarize_benchmarks::SummarizeBenchmarksArgs), + #[command(name = "validate-docs")] + ValidateDocs(repository_checks::ValidateDocsArgs), + #[command(name = "validate-benchmarks")] + ValidateBenchmarks(repository_checks::ValidateBenchmarksArgs), + #[command(name = "check-module-sizes")] + CheckModuleSizes(repository_checks::CheckModuleSizesArgs), + #[command(name = "test-watchdog")] + TestWatchdog(repository_checks::TestWatchdogArgs), } fn main() { @@ -42,6 +51,10 @@ fn main() { Commands::ValidateTests(args) => validate_tests::run(args), Commands::GenerateInventory(args) => generate_inventory::run(&args), Commands::SummarizeBenchmarks(args) => summarize_benchmarks::run(args), + Commands::ValidateDocs(args) => repository_checks::validate_docs(args), + Commands::ValidateBenchmarks(args) => repository_checks::validate_benchmarks(args), + Commands::CheckModuleSizes(args) => repository_checks::check_module_sizes(args), + Commands::TestWatchdog(args) => repository_checks::test_watchdog(args), } .unwrap_or_else(|error| { eprintln!("error: {error:#}"); diff --git a/src/repository_checks.rs b/src/repository_checks.rs new file mode 100644 index 0000000..9c064f7 --- /dev/null +++ b/src/repository_checks.rs @@ -0,0 +1,509 @@ +//! Generic repository-quality checks used by Cntryl repositories. + +use std::collections::{BTreeMap, BTreeSet}; +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::thread; +use std::time::{Duration, Instant}; + +use anyhow::{bail, Context, Result}; +use clap::Args; +use regex::Regex; +use serde::Deserialize; +use serde_json::Value; +use walkdir::WalkDir; + +fn root(path: &Path) -> Result { + path.canonicalize() + .with_context(|| format!("failed to resolve repository root {}", path.display())) +} + +fn read_config Deserialize<'de> + Default>( + root: &Path, + path: &Option, +) -> Result { + let path = path + .clone() + .unwrap_or_else(|| PathBuf::from(".cntryl/repository.toml")); + let path = root.join(path); + if !path.is_file() { + return Ok(T::default()); + } + toml::from_str( + &fs::read_to_string(&path).with_context(|| format!("failed to read {}", path.display()))?, + ) + .with_context(|| format!("failed to parse {}", path.display())) +} + +#[derive(Debug, Deserialize, Default)] +struct RepositoryConfig { + #[serde(default)] + docs: DocsConfig, + #[serde(default)] + benchmarks: BenchmarkConfig, + #[serde(default)] + module_sizes: ModuleSizeConfig, +} + +#[derive(Debug, Args, Clone)] +pub struct ValidateDocsArgs { + #[arg(long, default_value = ".")] + pub root: PathBuf, + #[arg(long)] + pub config: Option, +} + +#[derive(Debug, Deserialize, Default)] +struct DocsConfig { + #[serde(default)] + required: Vec, + #[serde(default)] + forbidden_paths: Vec, + #[serde(default)] + forbidden_text: Vec, +} + +pub fn validate_docs(args: ValidateDocsArgs) -> Result { + let root = root(&args.root)?; + let config = read_config::(&root, &args.config)?.docs; + let markdown: Vec = WalkDir::new(&root) + .into_iter() + .filter_entry(|entry| { + entry.file_name() != "target" && !entry.file_name().to_string_lossy().starts_with('.') + }) + .filter_map(Result::ok) + .filter(|entry| { + entry.file_type().is_file() && entry.path().extension().is_some_and(|ext| ext == "md") + }) + .map(|entry| entry.into_path()) + .collect(); + let names: BTreeSet = markdown.iter().map(|path| relative(&root, path)).collect(); + let mut errors = Vec::new(); + for path in &config.required { + if !names.contains(path) { + errors.push(format!("missing required document: {path}")); + } + } + for path in &config.forbidden_paths { + if names.contains(path) { + errors.push(format!("forbidden document is present: {path}")); + } + } + let mut headings = BTreeMap::new(); + for path in &markdown { + let text = fs::read_to_string(path) + .with_context(|| format!("failed to read {}", path.display()))?; + let name = relative(&root, path); + headings.insert(name.clone(), heading_ids(&text)); + for marker in &config.forbidden_text { + if text.contains(marker) { + errors.push(format!("{name}: forbidden documentation text {marker:?}")); + } + } + for (target, anchor) in markdown_links(&text) { + if target.is_empty() || target.contains("://") || target.starts_with("mailto:") { + continue; + } + let target_path = path.parent().unwrap_or(&root).join(&target); + let target_path = target_path.canonicalize().unwrap_or(target_path); + if !target_path.is_file() + && !(target_path.is_dir() && target_path.join("README.md").is_file()) + { + errors.push(format!("{name}: broken local link {target}")); + } + if let Some(anchor) = anchor { + let target_name = relative(&root, &target_path); + if !headings + .get(&target_name) + .is_some_and(|ids| ids.contains(&slug(&anchor))) + { + errors.push(format!("{name}: broken anchor {target}#{anchor}")); + } + } + } + } + if errors.is_empty() { + println!("validated {} Markdown documents", markdown.len()); + return Ok(0); + } + for error in errors { + eprintln!("error: {error}"); + } + Ok(1) +} + +fn relative(root: &Path, path: &Path) -> String { + path.strip_prefix(root) + .unwrap_or(path) + .to_string_lossy() + .replace('\\', "/") +} + +fn slug(value: &str) -> String { + value + .chars() + .map(|ch| ch.to_ascii_lowercase()) + .filter(|ch| ch.is_ascii_alphanumeric() || *ch == ' ' || *ch == '-') + .collect::() + .split_whitespace() + .collect::>() + .join("-") +} + +fn heading_ids(text: &str) -> BTreeSet { + text.lines() + .filter_map(|line| line.strip_prefix('#')) + .filter(|line| line.starts_with('#') || line.starts_with(' ')) + .map(|line| slug(line.trim_matches('#').trim())) + .collect() +} + +fn markdown_links(text: &str) -> Vec<(String, Option)> { + let Ok(re) = Regex::new(r"\[[^\]]*\]\(([^)#\s]*)(?:#([^)]*))?\)") else { + return Vec::new(); + }; + re.captures_iter(text) + .map(|capture| { + ( + capture[1].to_string(), + capture.get(2).map(|m| m.as_str().to_string()), + ) + }) + .collect() +} + +#[derive(Debug, Args, Clone)] +pub struct ValidateBenchmarksArgs { + #[arg(long, default_value = ".")] + pub root: PathBuf, + #[arg(long)] + pub config: Option, +} + +#[derive(Debug, Deserialize)] +struct BenchmarkConfig { + #[serde(default = "default_benchmark_docs")] + documentation: Vec, + #[serde(default = "default_benchmark_workflow")] + workflow: String, + #[serde(default)] + not_pr_gate_phrase: Option, +} + +impl Default for BenchmarkConfig { + fn default() -> Self { + Self { + documentation: default_benchmark_docs(), + workflow: default_benchmark_workflow(), + not_pr_gate_phrase: None, + } + } +} +fn default_benchmark_docs() -> Vec { + vec![ + "docs/development/benchmarks.md".into(), + "docs/development/performance-targets.md".into(), + "docs/operations/performance-tuning.md".into(), + ] +} +fn default_benchmark_workflow() -> String { + ".github/workflows/bench.yml".into() +} + +pub fn validate_benchmarks(args: ValidateBenchmarksArgs) -> Result { + let root = root(&args.root)?; + let config = read_config::(&root, &args.config)?.benchmarks; + let metadata: Value = serde_json::from_slice( + &Command::new("cargo") + .args(["metadata", "--no-deps", "--format-version", "1"]) + .current_dir(&root) + .output() + .context("failed to run cargo metadata")? + .stdout, + ) + .context("invalid cargo metadata")?; + let package = metadata["packages"] + .as_array() + .and_then(|packages| { + packages.iter().find(|package| { + package["manifest_path"] + .as_str() + .is_some_and(|path| Path::new(path).parent() == Some(root.as_path())) + }) + }) + .context("repository package not found in cargo metadata")?; + let registered: BTreeSet = package["targets"] + .as_array() + .unwrap_or(&Vec::new()) + .iter() + .filter(|target| { + target["kind"] + .as_array() + .is_some_and(|kinds| kinds.iter().any(|kind| kind == "bench")) + }) + .filter_map(|target| target["name"].as_str().map(str::to_owned)) + .collect(); + let bench_re = Regex::new(r#"cargo\s+bench\s+--bench\s+['\"`]?([A-Za-z0-9_.*-]+)"#)?; + let mut errors = Vec::new(); + let mut docs = String::new(); + for path in &config.documentation { + let text = fs::read_to_string(root.join(path)) + .with_context(|| format!("failed to read {path}"))?; + for capture in bench_re.captures_iter(&text) { + let target = &capture[1]; + if !registered.contains(target) { + errors.push(format!("{path} advertises unregistered benchmark {target}")); + } + } + if text.contains("--save-baseline") || text.contains("--baseline") { + errors.push(format!( + "{path} advertises obsolete Criterion baseline flags" + )); + } + docs.push_str(&text); + } + let workflow = fs::read_to_string(root.join(&config.workflow)) + .with_context(|| format!("failed to read {}", config.workflow))?; + for target in registered { + if !bench_re.captures_iter(&workflow).any(|capture| { + capture[1] == target + || capture[1].contains('*') && target.starts_with(capture[1].trim_end_matches('*')) + }) { + errors.push(format!("benchmark workflow does not execute {target}")); + } + } + if !workflow.contains("workflow_dispatch:") { + errors.push("benchmark workflow has no manual trigger".into()); + } + if workflow.contains("pull_request:") { + errors.push("benchmark workflow must not be a pull-request gate".into()); + } + if let Some(phrase) = config.not_pr_gate_phrase { + if !docs.contains(&phrase) { + errors.push(format!( + "benchmark docs do not contain required phrase {phrase:?}" + )); + } + } + if errors.is_empty() { + println!("benchmark contract matches Cargo metadata and workflow coverage"); + return Ok(0); + } + for error in errors { + eprintln!("error: {error}"); + } + Ok(1) +} + +#[derive(Debug, Args, Clone)] +pub struct CheckModuleSizesArgs { + #[arg(long, default_value = ".")] + pub root: PathBuf, + #[arg(long)] + pub config: Option, +} + +#[derive(Debug, Deserialize)] +struct ModuleSizeConfig { + #[serde(default = "default_warn")] + warn_lines: usize, + #[serde(default = "default_max")] + max_lines: usize, + #[serde(default)] + allowlist: Vec, + #[serde(default)] + legacy_allowlist: Vec, +} +impl Default for ModuleSizeConfig { + fn default() -> Self { + Self { + warn_lines: default_warn(), + max_lines: default_max(), + allowlist: Vec::new(), + legacy_allowlist: Vec::new(), + } + } +} +fn default_warn() -> usize { + 1_200 +} +fn default_max() -> usize { + 1_600 +} + +pub fn check_module_sizes(args: CheckModuleSizesArgs) -> Result { + let root = root(&args.root)?; + let config = read_config::(&root, &args.config)?.module_sizes; + let src = root.join("src"); + let mut failures = Vec::new(); + for entry in WalkDir::new(&src) + .into_iter() + .filter_map(Result::ok) + .filter(|e| { + e.file_type().is_file() + && e.path().extension().is_some_and(|ext| ext == "rs") + && e.file_name() != "tests.rs" + }) + { + let path = entry.path(); + let count = production_lines(&fs::read_to_string(path)?); + let name = relative(&root, path); + if count > config.warn_lines { + println!( + "{count:5} {name}{}", + if config.allowlist.iter().any(|allowed| allowed == &name) { + " [allowlist]" + } else { + "" + } + ); + } + if count > config.max_lines + && !config.allowlist.iter().any(|allowed| allowed == &name) + && !config + .legacy_allowlist + .iter() + .any(|allowed| allowed == &name) + { + failures.push(format!("{count:5} {name}")); + } + } + if failures.is_empty() { + return Ok(0); + } + eprintln!( + "modules exceed configured maximum of {} lines:", + config.max_lines + ); + for failure in failures { + eprintln!(" {failure}"); + } + Ok(1) +} + +fn production_lines(text: &str) -> usize { + let mut count = 0; + let mut skip = false; + let mut depth = 0i32; + for line in text.lines() { + let trimmed = line.trim(); + if !skip && trimmed.contains("#[cfg(test)]") { + skip = true; + continue; + } + if skip { + depth += line.matches('{').count() as i32; + depth -= line.matches('}').count() as i32; + if depth <= 0 && (line.contains('{') || line.contains(';')) { + skip = false; + } + } else if !trimmed.is_empty() && !trimmed.starts_with("//") { + count += 1; + } + } + count +} + +#[derive(Debug, Args, Clone)] +pub struct TestWatchdogArgs { + #[arg(long)] + pub suite: Option, + #[arg(long)] + pub all: bool, + #[arg(long)] + pub pattern: Option, + #[arg(long, default_value_t = 60)] + pub timeout: u64, + #[arg(long)] + pub continue_on_fail: bool, + #[arg(long, default_value = ".")] + pub root: PathBuf, +} + +pub fn test_watchdog(args: TestWatchdogArgs) -> Result { + if args.suite.is_none() && !args.all { + bail!("provide --suite or --all"); + } + let root = root(&args.root)?; + let suites = if let Some(suite) = args.suite { + vec![suite] + } else { + fs::read_dir(root.join("tests"))? + .filter_map(Result::ok) + .filter_map(|entry| { + entry.path().file_stem().and_then(|stem| { + entry + .path() + .extension() + .is_some_and(|ext| ext == "rs") + .then(|| stem.to_string_lossy().into_owned()) + }) + }) + .collect() + }; + let mut failed = false; + for suite in suites { + let output = Command::new("cargo") + .args(["test", "--test", &suite, "--", "--list"]) + .current_dir(&root) + .output() + .with_context(|| format!("failed to list tests in {suite}"))?; + if !output.status.success() { + eprint!("{}", String::from_utf8_lossy(&output.stdout)); + return Ok(2); + } + for line in String::from_utf8_lossy(&output.stdout) + .lines() + .filter_map(|line| line.strip_suffix(": test")) + { + if args + .pattern + .as_ref() + .is_some_and(|pattern| !format!("{suite}::{line}").contains(pattern)) + { + continue; + } + println!("=== RUN {suite}::{line} (timeout {}s) ===", args.timeout); + let status = run_test(&root, &suite, line, args.timeout)?; + if status != 0 { + failed = true; + if !args.continue_on_fail { + return Ok(if status == 124 { 124 } else { 2 }); + } + } + } + } + Ok(if failed { 2 } else { 0 }) +} + +fn run_test(root: &Path, suite: &str, name: &str, timeout: u64) -> Result { + let mut child = Command::new("cargo") + .args([ + "test", + "--test", + suite, + name, + "--", + "--exact", + "--nocapture", + "--test-threads=1", + ]) + .current_dir(root) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()) + .spawn()?; + let start = Instant::now(); + loop { + if let Some(status) = child.try_wait()? { + return Ok(status.code().unwrap_or(1)); + } + if start.elapsed() >= Duration::from_secs(timeout) { + child.kill()?; + let _ = child.wait(); + eprintln!("=== TIMEOUT {suite}::{name} ==="); + return Ok(124); + } + thread::sleep(Duration::from_millis(100)); + } +}