diff --git a/crates/prek/src/cli/cache_gc.rs b/crates/prek/src/cli/cache_gc.rs index bd47c40a2..5e6e4a766 100644 --- a/crates/prek/src/cli/cache_gc.rs +++ b/crates/prek/src/cli/cache_gc.rs @@ -25,6 +25,7 @@ use crate::store::{CacheBucket, REPO_MARKER, Store, ToolBucket}; #[derive(Debug, Copy, Clone, Eq, PartialEq)] enum RemovalKind { Repos, + RepoSources, HookEnvs, Tools, CacheEntries, @@ -36,6 +37,7 @@ impl RemovalKind { if count > 1 { match self { RemovalKind::Repos => "repos", + RemovalKind::RepoSources => "repo sources", RemovalKind::HookEnvs => "hook envs", RemovalKind::Tools => "tools", RemovalKind::CacheEntries => "cache entries", @@ -44,6 +46,7 @@ impl RemovalKind { } else { match self { RemovalKind::Repos => "repo", + RemovalKind::RepoSources => "repo source", RemovalKind::HookEnvs => "hook env", RemovalKind::Tools => "tool", RemovalKind::CacheEntries => "cache entry", @@ -159,6 +162,7 @@ pub(crate) async fn cache_gc( let mut kept_configs: FxHashSet<&Path> = FxHashSet::default(); let mut used_repo_keys: FxHashSet = FxHashSet::default(); + let mut used_repo_source_keys: FxHashSet = FxHashSet::default(); let mut used_hook_env_dirs: FxHashSet = FxHashSet::default(); let mut used_tools: FxHashSet = FxHashSet::default(); let mut used_tool_versions: FxHashMap> = FxHashMap::default(); @@ -182,7 +186,7 @@ pub(crate) async fn cache_gc( continue; } err => { - warn!(path = %config_path.display(), %err, "Failed to parse config, skipping for GC"); + warn!(path = %config_path.display(), %err, "Failed to load config, skipping for GC"); kept_configs.insert(config_path); continue; } @@ -204,6 +208,7 @@ pub(crate) async fn cache_gc( if let Some(key) = key { used_repo_keys.insert(key); } + used_repo_source_keys.insert(Store::repo_source_key(remote.source())); } } } @@ -251,6 +256,15 @@ pub(crate) async fn cache_gc( verbose, )?; + // Sweep repo-sources/; the hidden lock directory is ignored by the sweeper. + let removed_repo_sources = sweep_dir_by_name( + RemovalKind::RepoSources, + &store.repo_sources_dir(), + &used_repo_source_keys, + dry_run, + verbose, + )?; + // Sweep hooks/ let removed_hooks = sweep_dir_by_name( RemovalKind::HookEnvs, @@ -288,15 +302,15 @@ pub(crate) async fn cache_gc( verbose, )?; - // Seep scratch/, as it is only temporary data. if !dry_run { - let _ = fs_err::remove_dir_all(store.scratch_path()); + clear_directory(&store.scratch_path())?; } // Keep recent recovery patches, but clear out stale ones that are unlikely to be useful. let removed_patches = sweep_stale_patch_files(&store.patches_dir(), dry_run, verbose)?; let mut removed = RemovalSummary::default(); removed += &removed_repos; + removed += &removed_repo_sources; removed += &removed_hooks; removed += &removed_tools; removed += &removed_cache; @@ -318,6 +332,7 @@ pub(crate) async fn cache_gc( if verbose { print_removed_details(printer, verb, &removed_repos)?; + print_removed_details(printer, verb, &removed_repo_sources)?; print_removed_details(printer, verb, &removed_hooks)?; print_removed_details(printer, verb, &removed_tools)?; print_removed_details(printer, verb, &removed_cache)?; @@ -328,6 +343,18 @@ pub(crate) async fn cache_gc( Ok(ExitStatus::Success) } +fn clear_directory(path: &Path) -> std::io::Result<()> { + for entry in fs_err::read_dir(path)? { + let entry = entry?; + if entry.file_type()?.is_dir() { + fs_err::remove_dir_all(entry.path())?; + } else { + fs_err::remove_file(entry.path())?; + } + } + Ok(()) +} + fn print_removed_details(printer: Printer, verb: &str, removal: &Removal) -> Result<()> { if removal.count == 0 { return Ok(()); diff --git a/crates/prek/src/cli/try_repo.rs b/crates/prek/src/cli/try_repo.rs index e1ca20b2f..20d9a35bd 100644 --- a/crates/prek/src/cli/try_repo.rs +++ b/crates/prek/src/cli/try_repo.rs @@ -29,77 +29,15 @@ async fn get_head_rev(repo: &Path) -> Result { Ok(head_rev) } -async fn clone_and_commit(repo_path: &Path, head_rev: &str, tmp_dir: &Path) -> Result { - let shadow = tmp_dir.join("shadow-repo"); - git::git_cmd()? - .arg("clone") - .arg(repo_path) - .arg(&shadow) - .output() - .await?; - git::git_cmd()? - .arg("checkout") - .arg(head_rev) - .arg("-b") - .arg("_prek_tmp") - .current_dir(&shadow) - .output() - .await?; - - let index_path = shadow.join(".git/index"); - let objects_path = shadow.join(".git/objects"); - - let staged_files = git::get_staged_files(repo_path).await?; - if !staged_files.is_empty() { - git::git_cmd()? - .arg("add") - .arg("--") - .file_args(&staged_files) - .current_dir(repo_path) - .env("GIT_INDEX_FILE", &index_path) - .env("GIT_OBJECT_DIRECTORY", &objects_path) - .output() - .await?; - } - - let mut add_u_cmd = git::git_cmd()?; - add_u_cmd - .arg("add") - .arg("--update") // Update tracked files - .current_dir(repo_path) - .env("GIT_INDEX_FILE", &index_path) - .env("GIT_OBJECT_DIRECTORY", &objects_path) - .output() - .await?; - - git::git_cmd()? - .arg("commit") - .arg("-m") - .arg("Temporary commit by prek try-repo") - .arg("--no-gpg-sign") - .arg("--no-edit") - .arg("--no-verify") - .current_dir(&shadow) - .env("GIT_AUTHOR_NAME", "prek test") - .env("GIT_AUTHOR_EMAIL", "test@example.com") - .env("GIT_COMMITTER_NAME", "prek test") - .env("GIT_COMMITTER_EMAIL", "test@example.com") - .output() - .await?; - - Ok(shadow) -} - struct PreparedRepo<'a> { - runtime_source: Cow<'a, str>, - display_source: Option<&'a str>, + source: Cow<'a, str>, rev: String, } async fn prepare_repo<'a>( + store: &Store, repo: &'a str, rev: Option<&str>, - tmp_dir: &Path, ) -> Result> { let repo_path = Path::new(repo); let is_local = repo_path.is_dir(); @@ -117,8 +55,7 @@ async fn prepare_repo<'a>( // If rev is provided, use it directly. if let Some(rev) = rev { return Ok(PreparedRepo { - runtime_source, - display_source: is_local.then_some(repo), + source: runtime_source, rev: rev.to_string(), }); } @@ -143,20 +80,25 @@ async fn prepare_repo<'a>( .to_string() }; - // If repo is a local repo with uncommitted changes, create a shadow repo to commit the changes. + // Persist a deterministic synthetic commit in the shared source. The logical source remains + // the canonical local path, so identical dirty trees get the same repo and environment keys. if is_local && git::has_diff("HEAD", repo_path).await? { warn_user!("Creating temporary repo with uncommitted changes..."); - let shadow = clone_and_commit(repo_path, &head_rev, tmp_dir).await?; - let head_rev = get_head_rev(&shadow).await?; + let source = store.repo_source_path(runtime_source.as_ref()); + let _source_lock = store.repo_source_lock(runtime_source.as_ref()).await?; + git::ensure_bare_repo(runtime_source.as_ref(), &source).await?; + let head_rev = + git::fetch_repo_source_revision(&source, &head_rev, git::TerminalPrompt::Disabled) + .await?; + let snapshot = + git::create_repo_snapshot(&source, repo_path, &head_rev, &store.scratch_path()).await?; Ok(PreparedRepo { - runtime_source: Cow::Owned(shadow.to_string_lossy().into_owned()), - display_source: None, - rev: head_rev, + source: runtime_source, + rev: snapshot, }) } else { Ok(PreparedRepo { - runtime_source, - display_source: is_local.then_some(repo), + source: runtime_source, rev: head_rev, }) } @@ -202,41 +144,46 @@ pub(crate) async fn try_repo( } let store = Store::from_settings()?; - let tmp_dir = TempDir::with_prefix_in("try-repo-", store.scratch_path())?; - - let prepared = prepare_repo(&repo, rev.as_deref(), tmp_dir.path()) - .await - .context("Failed to determine repository and revision")?; - - let store = Store::from_path(tmp_dir.path()).init()?; - let repo_config = config::RemoteRepo::new( - prepared.runtime_source.to_string(), - prepared.rev.clone(), - vec![], - ); - let repo_clone_path = store.clone_repo(&repo_config, None).await?; - let selectors = Selectors::load(&run_args.includes, &run_args.skips, GIT_ROOT.as_ref()?)?; - let manifest = - config::read_manifest(&repo_clone_path.join(prek_consts::PRE_COMMIT_HOOKS_YAML))?; - - let hooks = manifest - .hooks - .into_iter() - .filter(|hook| selectors.matches_hook_id(&hook.id)) - .map(|hook| hook.id) - .collect::>(); + let (_tmp_dir, prepared, hooks, config_str, config_file) = { + let _lock = store.lock_async().await?; + // `cache gc` clears the contents of the store scratch directory after taking the store + // lock. Keep the active generated config in the system temporary directory so it survives + // between this preparation lock and the lock acquired by `run`. + let tmp_dir = TempDir::with_prefix("try-repo-")?; + let prepared = prepare_repo(&store, &repo, rev.as_deref()) + .await + .context("Failed to determine repository and revision")?; + let repo_config = + config::RemoteRepo::new(prepared.source.to_string(), prepared.rev.clone(), vec![]); + let repo_clone_path = store.clone_repo(&repo_config, None).await?; + let manifest = + config::read_manifest(&repo_clone_path.join(prek_consts::PRE_COMMIT_HOOKS_YAML))?; + let hooks = manifest + .hooks + .into_iter() + .filter(|hook| selectors.matches_hook_id(&hook.id)) + .map(|hook| hook.id) + .collect::>(); + + let config_str = render_repo_config_toml(&prepared.source, &prepared.rev, &hooks); + let config_file = tmp_dir.path().join(PREK_TOML); + fs_err::tokio::write(&config_file, &config_str).await?; + // Make the new source/checkout visible to GC before releasing the store lock. `run` also + // tracks this path, but a concurrent `cache gc` must not sweep a dirty synthetic commit in + // the interval between preparation and hook initialization. + store.track_configs(std::iter::once(config_file.as_path()))?; + + (tmp_dir, prepared, hooks, config_str, config_file) + }; // The scratch config needs the resolved source, while the displayed config should preserve // the user's path so it remains meaningful when copied into their project. - let config_str = render_repo_config_toml(&prepared.runtime_source, &prepared.rev, &hooks); - let config_file = tmp_dir.path().join(PREK_TOML); - fs_err::tokio::write(&config_file, &config_str).await?; - - let display_config_str = match prepared.display_source { - Some(source) => Cow::Owned(render_repo_config_toml(source, &prepared.rev, &hooks)), - None => Cow::Borrowed(config_str.as_str()), + let display_config_str = if prepared.source.as_ref() == repo { + Cow::Borrowed(config_str.as_str()) + } else { + Cow::Owned(render_repo_config_toml(&repo, &prepared.rev, &hooks)) }; writeln!( diff --git a/crates/prek/src/cli/update/mod.rs b/crates/prek/src/cli/update/mod.rs index 7df4f200c..74ff686d9 100644 --- a/crates/prek/src/cli/update/mod.rs +++ b/crates/prek/src/cli/update/mod.rs @@ -418,10 +418,20 @@ pub(crate) async fn update( filesystem: Option, printer: Printer, ) -> Result { + // Repository sources are shared with normal hook initialization and try-repo. Keep the + // command-level store invariant while per-source locks below serialize individual fetches. + let _store_lock = store.lock_async().await?; + let workspace_root = Workspace::find_root(config.as_deref(), &CWD)?; // TODO: support selectors? let selectors = Selectors::default(); let workspace = Workspace::discover(store, workspace_root, config, Some(&selectors), true)?; + store.track_configs( + workspace + .projects() + .iter() + .map(|project| project.config_file()), + )?; let tag_filters = TagFilters::new(include_tag, exclude_tag, repo_include_tag, repo_exclude_tag)?; @@ -450,7 +460,8 @@ pub(crate) async fn update( .first() .map_or(repo_source.source, |target| target.repo); let progress = reporter.on_update_start(display_repo); - let result = evaluate_repo_source(repo_source, bleeding_edge, &tag_filters).await; + let result = + evaluate_repo_source(store, repo_source, bleeding_edge, &tag_filters).await; reporter.on_update_complete(progress); result }) diff --git a/crates/prek/src/cli/update/repository.rs b/crates/prek/src/cli/update/repository.rs index 3a6cdae5c..4453d3a25 100644 --- a/crates/prek/src/cli/update/repository.rs +++ b/crates/prek/src/cli/update/repository.rs @@ -2,10 +2,11 @@ use std::borrow::Cow; use std::cmp::Ordering; use std::path::Path; use std::process::Stdio; +use std::str; use std::sync::OnceLock; use std::time::{SystemTime, UNIX_EPOCH}; -use anyhow::Result; +use anyhow::{Context, Result}; use itertools::Itertools; use prek_consts::PRE_COMMIT_HOOKS_YAML; use prek_consts::env_vars::EnvVars; @@ -16,22 +17,10 @@ use tracing::{debug, trace}; use crate::cli::update::{CommitPresence, RevisionSelection, SkippedDowngrade, TagTimestamp}; use crate::{config, git}; -/// Initializes a temporary git repo and fetches the remote HEAD plus tags. +/// Ensures the shared bare source exists and refreshes the remote HEAD plus tags. pub(super) async fn setup_and_fetch_repo(repo_url: &str, repo_path: &Path) -> Result<()> { - git::init_repo(repo_url, repo_path).await?; - git::git_cmd()? - .arg("fetch") - .arg("origin") - .arg("HEAD") - .arg("--quiet") - .arg("--filter=blob:none") - .arg("--tags") - .current_dir(repo_path) - .remove_git_envs() - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .status() - .await?; + git::ensure_bare_repo(repo_url, repo_path).await?; + git::fetch_repo_source_for_update(repo_path).await?; Ok(()) } @@ -50,13 +39,14 @@ pub(super) async fn resolve_revision_to_commit(repo_path: &Path, rev: &str) -> R Ok(String::from_utf8_lossy(&output.stdout).trim().to_string()) } -/// Returns whether a pinned commit SHA is already present in the refs fetched for `prek update`. +/// Returns whether a pinned commit SHA is reachable from the refs fetched for `prek update`. /// -/// `prek update` fetches only `origin/HEAD` and tags, using `--filter=blob:none`. That filter -/// still downloads commits and trees reachable from those refs, but omits blobs. We intentionally -/// use `git --no-lazy-fetch cat-file -e` here instead of `rev-parse`: in a partial clone, -/// `rev-parse` may lazily fetch a missing commit from the promisor remote on demand. On GitHub, -/// that can make a fork-only "impostor commit" appear to belong to the parent repository. +/// The source cache is persistent and may also contain commits fetched for normal hook execution. +/// Object existence alone therefore does not prove that a commit belongs to the current update +/// view. After the object probe, require reachability from the freshly fetched `FETCH_HEAD` or +/// current tag refs, ignoring accumulated objects and prek-owned pin refs. +/// `--no-lazy-fetch` also guarantees that probing an older partial/promisor cache cannot retrieve +/// an arbitrary SHA from the remote as a side effect. /// /// `prek update` only selects updates from tags, or from `HEAD` in `--bleeding-edge` mode. It /// does not normally update to arbitrary branches, so we currently fetch only those refs here. @@ -90,7 +80,13 @@ pub(super) async fn is_commit_present(repo_path: &Path, commit: &str) -> Result< if output.status.success() { let _ = GIT_SUPPORTS_NO_LAZY_FETCH.set(true); - return Ok(CommitPresence::Present); + return Ok( + if commit_reachable_from_update_refs(repo_path, commit).await? { + CommitPresence::Present + } else { + CommitPresence::Absent + }, + ); } if no_lazy_fetch_unsupported(&output.stderr) { @@ -102,6 +98,43 @@ pub(super) async fn is_commit_present(repo_path: &Path, commit: &str) -> Result< Ok(CommitPresence::Absent) } +async fn commit_reachable_from_update_refs(repo_path: &Path, commit: &str) -> Result { + let output = git::git_cmd()? + .arg("merge-base") + .arg("--is-ancestor") + .arg(commit) + .arg("FETCH_HEAD") + .check(false) + .current_dir(repo_path) + .remove_git_envs() + .output() + .await?; + + match output.status.code() { + Some(0) => return Ok(true), + Some(1) => {} + _ => { + anyhow::bail!( + "Failed to check whether commit `{commit}` is reachable from FETCH_HEAD: {}", + String::from_utf8_lossy(&output.stderr).trim() + ); + } + } + + let output = git::git_cmd()? + .arg("for-each-ref") + .arg(format!("--contains={commit}")) + .arg("--format=%(refname)") + .arg("refs/tags") + .check(true) + .current_dir(repo_path) + .remove_git_envs() + .output() + .await?; + + Ok(!output.stdout.is_empty()) +} + pub(super) fn no_lazy_fetch_unsupported(stderr: &[u8]) -> bool { let stderr = String::from_utf8_lossy(stderr); stderr.contains("--no-lazy-fetch") && stderr.contains("unknown option") @@ -369,38 +402,27 @@ fn levenshtein_distance(a: &str, b: &str) -> usize { row[a_len] } -/// Checks out the candidate manifest and verifies all configured hook ids still exist. -pub(super) async fn checkout_and_validate_manifest( +/// Reads the candidate manifest from the bare source and verifies configured hook ids still exist. +pub(super) async fn validate_manifest( repo_path: &Path, rev: &str, required_hook_ids: &[&str], ) -> Result<()> { - if cfg!(windows) { - git::git_cmd()? - .arg("show") - .arg(format!("{rev}:{PRE_COMMIT_HOOKS_YAML}")) - .current_dir(repo_path) - .remove_git_envs() - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .status() - .await?; - } - - git::git_cmd()? - .arg("checkout") - .arg("--quiet") - .arg(rev) - .arg("--") - .arg(PRE_COMMIT_HOOKS_YAML) + let output = git::git_cmd()? + .arg("show") + .arg(format!("{rev}:{PRE_COMMIT_HOOKS_YAML}")) .current_dir(repo_path) .remove_git_envs() - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .status() + .check(true) + .output() .await?; - let manifest = config::read_manifest(&repo_path.join(PRE_COMMIT_HOOKS_YAML))?; + let content = str::from_utf8(&output.stdout).with_context(|| { + format!("Manifest `{PRE_COMMIT_HOOKS_YAML}` at rev `{rev}` is not UTF-8") + })?; + let manifest: config::Manifest = serde_saphyr::from_str(content).with_context(|| { + format!("Failed to parse manifest `{PRE_COMMIT_HOOKS_YAML}` at rev `{rev}`") + })?; let new_hook_ids = manifest .hooks .into_iter() diff --git a/crates/prek/src/cli/update/source.rs b/crates/prek/src/cli/update/source.rs index 3c229c14e..fb5466964 100644 --- a/crates/prek/src/cli/update/source.rs +++ b/crates/prek/src/cli/update/source.rs @@ -7,9 +7,9 @@ use tracing::{debug, trace, warn}; use crate::cli::update::config::read_frozen_refs; use crate::cli::update::repository::{ - checkout_and_validate_manifest, get_tags_pointing_at_revision, is_commit_present, - list_tag_metadata, resolve_revision_to_commit, select_best_tag, select_update_revision, - setup_and_fetch_repo, + get_tags_pointing_at_revision, is_commit_present, list_tag_metadata, + resolve_revision_to_commit, select_best_tag, select_update_revision, setup_and_fetch_repo, + validate_manifest, }; use crate::cli::update::{ CommitPresence, FrozenMismatch, FrozenMismatchAction, FrozenMismatchReason, RepoSource, @@ -19,6 +19,7 @@ use crate::cli::update::{ use crate::config::{Repo, looks_like_sha}; use crate::fs::Simplified; use crate::settings::{FilesystemOptions, UpdateSettings}; +use crate::store::Store; use crate::workspace::Workspace; /// Identifies repo usages that can share one update evaluation. @@ -206,61 +207,63 @@ async fn collect_frozen_mismatches<'a>( /// Fetches a remote repository once, then evaluates all configured revisions that use it. pub(super) async fn evaluate_repo_source<'a>( + store: &Store, repo_source: &'a RepoSource<'a>, bleeding_edge: bool, tag_filters: &TagFilters, ) -> Result>> { - let tmp_dir = tempfile::tempdir()?; - let repo_path = tmp_dir.path(); + let repo_path = store.repo_source_path(repo_source.source); let result = async { + // Keep FETCH_HEAD and the fetched tag set stable until every target that shares this + // source has been evaluated. Other prek processes may fetch different revisions into the + // same bare repository. + let _source_lock = store.repo_source_lock(repo_source.source).await?; trace!( - "Cloning repository `{}` to `{}`", + "Fetching repository `{}` to `{}`", repo_source.source, repo_path.display() ); - setup_and_fetch_repo(repo_source.source, repo_path).await?; - let metadata = list_tag_metadata(repo_path).await?; + setup_and_fetch_repo(repo_source.source, &repo_path).await?; + let tag_timestamps = list_tag_metadata(&repo_path).await?; - anyhow::Ok(metadata) + let mut updates = Vec::with_capacity(repo_source.targets.len()); + for target in &repo_source.targets { + let update_tag_timestamps = tag_filters + .filter(target.repo, &tag_timestamps) + .into_iter() + .cloned() + .collect::>(); + let result = evaluate_repo_target( + &repo_path, + target, + bleeding_edge, + &tag_timestamps, + &update_tag_timestamps, + ) + .await; + + updates.push(RepoUpdate { target, result }); + } + + anyhow::Ok(updates) } .await; - let tag_timestamps = match result { - Ok(metadata) => metadata, + match result { + Ok(updates) => Ok(updates), Err(e) => { let error = format!("{e:#}"); - return Ok(repo_source + Ok(repo_source .targets .iter() .map(|target| RepoUpdate { target, result: Err(anyhow::anyhow!(error.clone())), }) - .collect()); + .collect()) } - }; - - let mut updates = Vec::with_capacity(repo_source.targets.len()); - for target in &repo_source.targets { - let update_tag_timestamps = tag_filters - .filter(target.repo, &tag_timestamps) - .into_iter() - .cloned() - .collect::>(); - let result = evaluate_repo_target( - repo_path, - target, - bleeding_edge, - &tag_timestamps, - &update_tag_timestamps, - ) - .await; - - updates.push(RepoUpdate { target, result }); } - - Ok(updates) } /// Resolves one configured repo target within an already-fetched remote repository. @@ -324,7 +327,7 @@ async fn evaluate_repo_target<'a>( (rev, None) }; - checkout_and_validate_manifest(repo_path, &rev, &target.required_hook_ids).await?; + validate_manifest(repo_path, &rev, &target.required_hook_ids).await?; Ok(ResolvedRepoUpdate { revision: Revision { rev, frozen }, diff --git a/crates/prek/src/git.rs b/crates/prek/src/git.rs index 596495c90..1a18c5af9 100644 --- a/crates/prek/src/git.rs +++ b/crates/prek/src/git.rs @@ -431,29 +431,88 @@ pub(crate) fn get_root() -> Result { path_from_git_bytes(output.stdout.trim_ascii()).map_err(Error::from) } -pub(crate) async fn init_repo(url: &str, path: &Path) -> Result<(), Error> { - git_cmd()? - // Unset `extensions.objectFormat` if set, just follow what hash the remote uses. - .arg("-c") - .arg("init.defaultObjectFormat=") - .arg("init") - .arg("--template=") - .arg(path) - .remove_git_envs() - .check(true) - .output() - .await?; +/// Ensure a shared bare source repository exists and points at its upstream. +pub(crate) async fn ensure_bare_repo(repo: &str, source: &Path) -> Result<(), Error> { + let valid = if source.join("HEAD").try_exists()? { + let output = git_cmd()? + .arg("--git-dir") + .arg(source) + .arg("rev-parse") + .arg("--is-bare-repository") + .remove_git_envs() + .check(false) + .output() + .await?; + output.status.success() && output.stdout.trim_ascii() == b"true" + } else { + false + }; - git_cmd()? - .current_dir(path) + if !valid { + if source.try_exists()? { + let metadata = fs_err::tokio::symlink_metadata(source).await?; + if metadata.is_dir() { + fs_err::tokio::remove_dir_all(source).await?; + } else { + fs_err::tokio::remove_file(source).await?; + } + } + if let Some(parent) = source.parent() { + fs_err::tokio::create_dir_all(parent).await?; + } + + git_cmd()? + // Unset `extensions.objectFormat` if set, just follow what hash the remote uses. + .arg("-c") + .arg("init.defaultObjectFormat=") + .arg("init") + .arg("--bare") + .arg("--template=") + .arg(source) + .remove_git_envs() + .check(true) + .output() + .await?; + + git_cmd()? + .arg("--git-dir") + .arg(source) + .arg("remote") + .arg("add") + .arg("origin") + .arg(repo) + .remove_git_envs() + .check(true) + .output() + .await?; + + return Ok(()); + } + + let output = git_cmd()? + .arg("--git-dir") + .arg(source) .arg("remote") - .arg("add") + .arg("set-url") .arg("origin") - .arg(url) + .arg(repo) .remove_git_envs() - .check(true) + .check(false) .output() .await?; + if !output.status.success() { + git_cmd()? + .arg("--git-dir") + .arg(source) + .arg("remote") + .arg("add") + .arg("origin") + .arg(repo) + .remove_git_envs() + .check(true) + .output() + .await?; + } Ok(()) } @@ -501,14 +560,108 @@ pub(crate) fn is_auth_error(err: &Error) -> bool { .any(|needle| error.contains(needle)) } -async fn shallow_clone( +fn is_full_object_id(rev: &str) -> bool { + matches!(rev.len(), 40 | 64) && rev.as_bytes().iter().all(u8::is_ascii_hexdigit) +} + +async fn try_resolve_repo_source_commit(source: &Path, rev: &str) -> Result, Error> { + let output = git_cmd()? + .arg("--git-dir") + .arg(source) + .arg("rev-parse") + .arg(format!("{rev}^{{commit}}")) + .remove_git_envs() + .check(false) + .output() + .await?; + if output.status.success() { + Ok(Some( + String::from_utf8_lossy(&output.stdout) + .trim_ascii() + .to_string(), + )) + } else { + Ok(None) + } +} + +async fn resolve_repo_source_commit(source: &Path, rev: &str) -> Result { + if let Some(commit) = try_resolve_repo_source_commit(source, rev).await? { + return Ok(commit); + } + + let remote_rev = format!("refs/remotes/origin/{rev}"); + if let Some(commit) = try_resolve_repo_source_commit(source, &remote_rev).await? { + return Ok(commit); + } + + let output = git_cmd()? + .arg("--git-dir") + .arg(source) + .arg("rev-parse") + .arg(format!("{rev}^{{commit}}")) + .remove_git_envs() + .check(true) + .output() + .await?; + Ok(String::from_utf8_lossy(&output.stdout) + .trim_ascii() + .to_string()) +} + +async fn pin_repo_source_commit(source: &Path, commit: &str) -> Result<(), Error> { + git_cmd()? + .arg("--git-dir") + .arg(source) + .arg("update-ref") + .arg(repo_source_pin(commit)) + .arg(commit) + .remove_git_envs() + .check(true) + .output() + .await?; + Ok(()) +} + +fn repo_source_pin(commit: &str) -> String { + format!("refs/heads/prek/{commit}") +} + +async fn repo_source_commit_is_pinned(source: &Path, commit: &str) -> Result { + Ok(git_cmd()? + .arg("--git-dir") + .arg(source) + .args(["show-ref", "--verify", "--quiet"]) + .arg(repo_source_pin(commit)) + .remove_git_envs() + .check(false) + .output() + .await? + .status + .success()) +} + +async fn repo_source_is_partial(source: &Path) -> Result { + let output = git_cmd()? + .arg("--git-dir") + .arg(source) + .args(["config", "--bool", "--get", "remote.origin.promisor"]) + .remove_git_envs() + .check(false) + .output() + .await?; + Ok(output.status.success() && output.stdout.trim_ascii() == b"true") +} + +async fn fetch_repo_source_shallow( + source: &Path, rev: &str, - path: &Path, terminal_prompt: TerminalPrompt, ) -> Result<(), Error> { git_cmd()? - .current_dir(path) .hidden_args(["-c", "protocol.version=2"]) + .arg("--git-dir") + .arg(source) .arg("fetch") .arg("origin") .arg(rev) @@ -519,30 +672,212 @@ async fn shallow_clone( .check(true) .output() .await?; + Ok(()) +} - git_cmd()? - .current_dir(path) - .arg("checkout") - .arg("FETCH_HEAD") +async fn materialize_repo_source_revision( + source: &Path, + rev: &str, + terminal_prompt: TerminalPrompt, +) -> Result<(), Error> { + // Walk the snapshot's tree, rather than the commit, so history is excluded. Fetching all + // promised objects reported by `--missing=print` in one request avoids both one-fetch-per-blob + // lazy loading and the shallow-boundary changes caused by another depth-one commit fetch. + let output = git_cmd()? + .arg("--git-dir") + .arg(source) + .args([ + "rev-list", + "--objects", + "--missing=print", + "--no-object-names", + ]) + .arg(format!("{rev}^{{tree}}")) + .remove_git_envs() + .check(true) + .output() + .await?; + let missing = str::from_utf8(&output.stdout)? + .lines() + .filter_map(|line| line.strip_prefix('?')) + .collect::>(); + if missing.is_empty() { + return Ok(()); + } + + let mut fetch = git_cmd()?; + fetch + .hidden_args(["-c", "fetch.negotiationAlgorithm=noop"]) + .arg("--git-dir") + .arg(source) + .arg("fetch") + .arg("origin") + .args([ + "--no-tags", + "--no-write-fetch-head", + "--recurse-submodules=no", + "--stdin", + ]) .remove_git_envs() - .env(EnvVars::PREK_INTERNAL__SKIP_POST_CHECKOUT, "1") + .env(EnvVars::LC_ALL, "C") + .env(EnvVars::GIT_TERMINAL_PROMPT, terminal_prompt.env_value()) + .stdin(Stdio::piped()) + .stdout(Stdio::null()); + let mut child = fetch.spawn()?; + let mut stdin = child.stdin.take().expect("failed to open stdin"); + for object in missing { + stdin.write_all(object.as_bytes()).await?; + stdin.write_all(b"\n").await?; + } + stdin.shutdown().await?; + drop(stdin); + fetch.check_status(child.wait().await?)?; + Ok(()) +} + +async fn fetch_repo_source_full( + source: &Path, + terminal_prompt: TerminalPrompt, +) -> Result<(), Error> { + let is_shallow = source.join("shallow").try_exists()?; + let mut cmd = git_cmd()?; + cmd.hidden_args(["--git-dir"]) + .hidden_args([source.as_os_str()]) + .arg("fetch") + .arg("origin") + .hidden_args([ + "+refs/heads/*:refs/remotes/origin/*", + "+refs/tags/*:refs/tags/*", + "--prune", + ]); + if is_shallow { + cmd.hidden_args(["--unshallow"]); + } else { + cmd.hidden_args(["--update-shallow"]); + } + cmd.remove_git_envs() .env(EnvVars::LC_ALL, "C") .env(EnvVars::GIT_TERMINAL_PROMPT, terminal_prompt.env_value()) .check(true) .output() .await?; + Ok(()) +} - update_submodules(path, terminal_prompt, true).await?; +/// Fetch a revision into a shared bare source and return its exact commit. +/// +/// A source has two object-completeness levels: refs fetched by `prek update` have a complete +/// commit graph but may omit blobs, while `refs/heads/prek/` certifies that the commit's +/// depth-one tree and blobs are materialized for a derived checkout. Object existence alone is +/// therefore not a checkout-readiness check once the source becomes a partial repository. +pub(crate) async fn fetch_repo_source_revision( + source: &Path, + rev: &str, + terminal_prompt: TerminalPrompt, +) -> Result { + if is_full_object_id(rev) + && let Some(commit) = try_resolve_repo_source_commit(source, rev).await? + { + if repo_source_commit_is_pinned(source, &commit).await? { + return Ok(commit); + } + if repo_source_is_partial(source).await? { + materialize_repo_source_revision(source, &commit, terminal_prompt).await?; + } + pin_repo_source_commit(source, &commit).await?; + return Ok(commit); + } - Ok(()) + let commit = match fetch_repo_source_shallow(source, rev, terminal_prompt).await { + Ok(()) => { + let commit = resolve_repo_source_commit(source, "FETCH_HEAD").await?; + if repo_source_is_partial(source).await? { + materialize_repo_source_revision(source, &commit, terminal_prompt).await?; + } + commit + } + Err(err) => { + if is_auth_error(&err) { + warn!( + ?err, + "Failed to fetch repo source due to authentication error" + ); + return Err(err); + } + + warn!( + ?err, + "Failed to fetch repo source revision, falling back to full fetch" + ); + fetch_repo_source_full(source, terminal_prompt).await?; + let commit = resolve_repo_source_commit(source, rev).await?; + if repo_source_is_partial(source).await? { + materialize_repo_source_revision(source, &commit, terminal_prompt).await?; + } + commit + } + }; + + // Create the pin only after the revision is complete; checkout treats this ref as its readiness + // marker as well as the branch exposed to the depth-one transport clone. + pin_repo_source_commit(source, &commit).await?; + Ok(commit) } -async fn full_clone(rev: &str, path: &Path, terminal_prompt: TerminalPrompt) -> Result<(), Error> { - git_cmd()? - .current_dir(path) +/// Refresh the default branch and tags used by `prek update`. +/// +/// Update needs the complete commit graph for ancestry checks, but not historical file contents. +/// A blobless fetch can coexist with complete pinned snapshots already added by normal runs. If a +/// run added a depth-one boundary, `--unshallow` repairs the graph before update inspects ancestry. +pub(crate) async fn fetch_repo_source_for_update(source: &Path) -> Result<(), Error> { + let is_shallow = source.join("shallow").try_exists()?; + let mut cmd = git_cmd()?; + cmd.hidden_args(["-c", "protocol.version=2"]) + .arg("--git-dir") + .arg(source) .arg("fetch") + .arg("--filter=blob:none") .arg("origin") - .arg("--tags") + .arg("HEAD") + .arg("+refs/tags/*:refs/tags/*") + .arg("--prune"); + if is_shallow { + cmd.arg("--unshallow"); + } else { + cmd.arg("--update-shallow"); + } + cmd.remove_git_envs() + .env(EnvVars::LC_ALL, "C") + .check(true) + .output() + .await?; + Ok(()) +} + +/// Create an independent checkout from a shared bare source. +pub(crate) async fn checkout_repo_from_source( + source: &Path, + repo: &str, + revision: &str, + target: &Path, + terminal_prompt: TerminalPrompt, +) -> Result<(), Error> { + git_cmd()? + .arg("clone") + .arg("--no-checkout") + // Only the pin's depth-one closure is guaranteed complete. Force the transport path so Git + // honors `--depth=1` for a local source; a normal local clone ignores depth and could copy + // blobless history from the update view into a non-promisor checkout. + .arg("--no-local") + .arg("--depth=1") + .arg("--single-branch") + .arg("--no-tags") + .arg("--branch") + .arg(format!("prek/{revision}")) + .arg("--template=") + .arg("--origin=origin") + .arg(source) + .arg(target) .remove_git_envs() .env(EnvVars::LC_ALL, "C") .env(EnvVars::GIT_TERMINAL_PROMPT, terminal_prompt.env_value()) @@ -551,9 +886,21 @@ async fn full_clone(rev: &str, path: &Path, terminal_prompt: TerminalPrompt) -> .await?; git_cmd()? - .current_dir(path) + .current_dir(target) + .arg("remote") + .arg("set-url") + .arg("origin") + .arg(repo) + .remove_git_envs() + .check(true) + .output() + .await?; + + git_cmd()? + .current_dir(target) .arg("checkout") - .arg(rev) + .arg("--quiet") + .arg(revision) .remove_git_envs() .env(EnvVars::PREK_INTERNAL__SKIP_POST_CHECKOUT, "1") .env(EnvVars::LC_ALL, "C") @@ -562,11 +909,108 @@ async fn full_clone(rev: &str, path: &Path, terminal_prompt: TerminalPrompt) -> .output() .await?; - update_submodules(path, terminal_prompt, false).await?; + if let Err(err) = update_submodules(target, terminal_prompt, true).await { + if is_auth_error(&err) { + return Err(err); + } + warn!( + ?err, + "Failed to shallow clone submodules, falling back to full clones" + ); + update_submodules(target, terminal_prompt, false).await?; + } Ok(()) } +/// Snapshot tracked and staged local changes as a deterministic commit in a bare source. +pub(crate) async fn create_repo_snapshot( + source: &Path, + worktree: &Path, + head_commit: &str, + scratch: &Path, +) -> Result { + let temp = tempfile::tempdir_in(scratch)?; + let index = temp.path().join("index"); + let objects = source.join("objects"); + + git_cmd()? + .current_dir(worktree) + .arg("read-tree") + .arg(head_commit) + .remove_git_envs() + .env("GIT_INDEX_FILE", &index) + .env("GIT_OBJECT_DIRECTORY", &objects) + .check(true) + .output() + .await?; + + let staged_files = get_staged_files(worktree).await?; + if !staged_files.is_empty() { + git_cmd()? + .current_dir(worktree) + .arg("add") + .arg("--") + .file_args(&staged_files) + .remove_git_envs() + .env("GIT_INDEX_FILE", &index) + .env("GIT_OBJECT_DIRECTORY", &objects) + .check(true) + .output() + .await?; + } + + git_cmd()? + .current_dir(worktree) + .arg("add") + .arg("--update") + .remove_git_envs() + .env("GIT_INDEX_FILE", &index) + .env("GIT_OBJECT_DIRECTORY", &objects) + .check(true) + .output() + .await?; + + let tree = git_cmd()? + .current_dir(worktree) + .arg("write-tree") + .remove_git_envs() + .env("GIT_INDEX_FILE", &index) + .env("GIT_OBJECT_DIRECTORY", &objects) + .check(true) + .output() + .await?; + let tree = String::from_utf8_lossy(&tree.stdout) + .trim_ascii() + .to_string(); + + let commit = git_cmd()? + .current_dir(worktree) + .arg("commit-tree") + .arg(&tree) + .arg("-p") + .arg(head_commit) + .arg("-m") + .arg("Temporary commit by prek try-repo") + .remove_git_envs() + .env("GIT_OBJECT_DIRECTORY", &objects) + .env("GIT_AUTHOR_NAME", "prek try-repo") + .env("GIT_AUTHOR_EMAIL", "try-repo@prek.dev") + .env("GIT_COMMITTER_NAME", "prek try-repo") + .env("GIT_COMMITTER_EMAIL", "try-repo@prek.dev") + .env("GIT_AUTHOR_DATE", "2000-01-01T00:00:00Z") + .env("GIT_COMMITTER_DATE", "2000-01-01T00:00:00Z") + .check(true) + .output() + .await?; + let commit = String::from_utf8_lossy(&commit.stdout) + .trim_ascii() + .to_string(); + pin_repo_source_commit(source, &commit).await?; + + Ok(commit) +} + async fn update_submodules( path: &Path, terminal_prompt: TerminalPrompt, @@ -618,35 +1062,6 @@ async fn should_update_submodules(path: &Path) -> Result { .any(|entry| entry.starts_with(b"160000 "))) } -async fn clone_repo_attempt( - rev: &str, - path: &Path, - terminal_prompt: TerminalPrompt, -) -> Result<(), Error> { - if let Err(err) = shallow_clone(rev, path, terminal_prompt).await { - if is_auth_error(&err) { - warn!(?err, "Failed to shallow clone due to authentication error"); - return Err(err); - } - - warn!(?err, "Failed to shallow clone, falling back to full clone"); - return full_clone(rev, path, terminal_prompt).await; - } - - Ok(()) -} - -/// Clone a repository into an initialized destination with the requested terminal prompt mode. -pub(crate) async fn clone_repo( - url: &str, - rev: &str, - path: &Path, - terminal_prompt: TerminalPrompt, -) -> Result<(), Error> { - init_repo(url, path).await?; - clone_repo_attempt(rev, path, terminal_prompt).await -} - async fn get_config_value(scope: Option<&str>, key: &str) -> Result>, Error> { let mut cmd = git_cmd()?; cmd.arg("config").arg("--includes"); @@ -934,7 +1349,8 @@ mod tests { #[cfg(unix)] use super::zsplit; use super::{ - Error, GIT, TerminalPrompt, full_clone, init_repo, shared_repository_file_mode, + Error, GIT, TerminalPrompt, checkout_repo_from_source, ensure_bare_repo, + fetch_repo_source_for_update, fetch_repo_source_revision, shared_repository_file_mode, should_update_submodules, update_submodules, }; use assert_cmd::assert::OutputAssertExt; @@ -948,6 +1364,34 @@ mod tests { command.assert().success(); } + fn git_stdout(path: &Path, args: &[&str]) -> String { + let output = Command::new(GIT.as_ref().unwrap()) + .current_dir(path) + .args(args) + .output() + .unwrap(); + assert!(output.status.success()); + String::from_utf8_lossy(&output.stdout).trim().to_string() + } + + fn commit_all(path: &Path, message: &str) { + run_git(path, &["add", "--all"]); + run_git( + path, + &[ + "-c", + "commit.gpgsign=false", + "-c", + "user.name=prek", + "-c", + "user.email=prek@example.com", + "commit", + "-m", + message, + ], + ); + } + #[tokio::test] async fn should_update_submodules_when_gitmodules_exists() { let tmp = tempfile::tempdir().unwrap(); @@ -958,40 +1402,243 @@ mod tests { } #[tokio::test] - async fn full_clone_skips_submodule_update_when_repo_has_no_submodules() { + async fn repo_source_update_refreshes_moved_and_deleted_tags() { let remote = tempfile::tempdir().unwrap(); run_git(remote.path(), &["init"]); - fs_err::write(remote.path().join("file.txt"), "content\n").unwrap(); - run_git(remote.path(), &["add", "."]); run_git( remote.path(), &[ + "-c", + "commit.gpgsign=false", "-c", "user.name=prek", "-c", "user.email=prek@example.com", "commit", + "--allow-empty", "-m", - "initial commit", + "first", ], ); - let output = Command::new(GIT.as_ref().unwrap()) + let first = Command::new(GIT.as_ref().unwrap()) .current_dir(remote.path()) - .arg("rev-parse") - .arg("HEAD") + .args(["rev-parse", "HEAD"]) .output() .unwrap(); - assert!(output.status.success()); - let rev = String::from_utf8_lossy(&output.stdout); + let first = String::from_utf8_lossy(&first.stdout).trim().to_string(); + run_git(remote.path(), &["-c", "tag.gpgsign=false", "tag", "v1"]); - let clone = tempfile::tempdir().unwrap(); - init_repo(remote.path().to_str().unwrap(), clone.path()) + let source_root = tempfile::tempdir().unwrap(); + let source = source_root.path().join("source.git"); + ensure_bare_repo(remote.path().to_str().unwrap(), &source) .await .unwrap(); + fetch_repo_source_for_update(&source).await.unwrap(); + let fetched_first = Command::new(GIT.as_ref().unwrap()) + .current_dir(&source) + .args(["rev-parse", "refs/tags/v1"]) + .output() + .unwrap(); + assert_eq!(String::from_utf8_lossy(&fetched_first.stdout).trim(), first); + + run_git( + remote.path(), + &[ + "-c", + "commit.gpgsign=false", + "-c", + "user.name=prek", + "-c", + "user.email=prek@example.com", + "commit", + "--allow-empty", + "-m", + "second", + ], + ); + let second = Command::new(GIT.as_ref().unwrap()) + .current_dir(remote.path()) + .args(["rev-parse", "HEAD"]) + .output() + .unwrap(); + let second = String::from_utf8_lossy(&second.stdout).trim().to_string(); + run_git( + remote.path(), + &["-c", "tag.gpgsign=false", "tag", "-f", "v1"], + ); + run_git(remote.path(), &["-c", "tag.gpgsign=false", "tag", "v2"]); - full_clone(rev.trim(), clone.path(), TerminalPrompt::Disabled) + fetch_repo_source_for_update(&source).await.unwrap(); + let moved = Command::new(GIT.as_ref().unwrap()) + .current_dir(&source) + .args(["rev-parse", "refs/tags/v1"]) + .output() + .unwrap(); + assert_eq!(String::from_utf8_lossy(&moved.stdout).trim(), second); + + run_git(remote.path(), &["tag", "--delete", "v2"]); + fetch_repo_source_for_update(&source).await.unwrap(); + let deleted = Command::new(GIT.as_ref().unwrap()) + .current_dir(&source) + .args(["show-ref", "--verify", "--quiet", "refs/tags/v2"]) + .status() + .unwrap(); + assert!(!deleted.success()); + } + + #[tokio::test] + async fn repo_source_full_fetches_unshallow_existing_sources() { + let remote = tempfile::tempdir().unwrap(); + run_git(remote.path(), &["init"]); + run_git( + remote.path(), + &[ + "-c", + "commit.gpgsign=false", + "-c", + "user.name=prek", + "-c", + "user.email=prek@example.com", + "commit", + "--allow-empty", + "-m", + "first", + ], + ); + let first = Command::new(GIT.as_ref().unwrap()) + .current_dir(remote.path()) + .args(["rev-parse", "HEAD"]) + .output() + .unwrap(); + let first = String::from_utf8_lossy(&first.stdout).trim().to_string(); + let first_short = &first[..7]; + run_git( + remote.path(), + &[ + "-c", + "commit.gpgsign=false", + "-c", + "user.name=prek", + "-c", + "user.email=prek@example.com", + "commit", + "--allow-empty", + "-m", + "second", + ], + ); + + let source_root = tempfile::tempdir().unwrap(); + let update_source = source_root.path().join("update.git"); + ensure_bare_repo(remote.path().to_str().unwrap(), &update_source) + .await + .unwrap(); + fetch_repo_source_revision(&update_source, "HEAD", TerminalPrompt::Disabled) .await .unwrap(); + assert!(update_source.join("shallow").is_file()); + + fetch_repo_source_for_update(&update_source).await.unwrap(); + assert!(!update_source.join("shallow").exists()); + Command::new(GIT.as_ref().unwrap()) + .args([ + "--git-dir", + update_source.to_str().unwrap(), + "cat-file", + "-e", + ]) + .arg(format!("{first}^{{commit}}")) + .assert() + .success(); + + let fallback_source = source_root.path().join("fallback.git"); + ensure_bare_repo(remote.path().to_str().unwrap(), &fallback_source) + .await + .unwrap(); + fetch_repo_source_revision(&fallback_source, "HEAD", TerminalPrompt::Disabled) + .await + .unwrap(); + assert!(fallback_source.join("shallow").is_file()); + + let resolved = + fetch_repo_source_revision(&fallback_source, first_short, TerminalPrompt::Disabled) + .await + .unwrap(); + assert_eq!(resolved, first); + assert!(!fallback_source.join("shallow").exists()); + + let cached = + fetch_repo_source_revision(&fallback_source, &resolved, TerminalPrompt::Disabled) + .await + .unwrap(); + assert_eq!(cached, first); + } + + #[cfg(unix)] + #[tokio::test] + async fn repo_source_materializes_only_the_checkout_snapshot_after_update() { + let remote = tempfile::tempdir().unwrap(); + run_git(remote.path(), &["init"]); + fs_err::write(remote.path().join("old.py"), "print('old')\n").unwrap(); + commit_all(remote.path(), "old hook"); + let old_commit = git_stdout(remote.path(), &["rev-parse", "HEAD"]); + let old_blob = git_stdout(remote.path(), &["rev-parse", "HEAD:old.py"]); + + fs_err::remove_file(remote.path().join("old.py")).unwrap(); + fs_err::write(remote.path().join("hook.py"), "print('hello')\n").unwrap(); + commit_all(remote.path(), "hook"); + // Local upload-pack disables filtering by default, unlike the hosted remotes this models. + run_git(remote.path(), &["config", "uploadpack.allowFilter", "true"]); + let repo = format!("file://{}", remote.path().display()); + + let root = tempfile::tempdir().unwrap(); + let source = root.path().join("source.git"); + ensure_bare_repo(&repo, &source).await.unwrap(); + fetch_repo_source_for_update(&source).await.unwrap(); + let commit = git_stdout(&source, &["rev-parse", "FETCH_HEAD"]); + + let resolved = fetch_repo_source_revision(&source, &commit, TerminalPrompt::Disabled) + .await + .unwrap(); + assert_eq!(resolved, commit); + assert!( + !source.join("shallow").exists(), + "materializing a known commit must preserve update's complete graph" + ); + let old_snapshot = Command::new(GIT.as_ref().unwrap()) + .args([ + "--git-dir", + source.to_str().unwrap(), + "rev-list", + "--objects", + "--missing=print", + "--no-object-names", + ]) + .arg(format!("{old_commit}^{{tree}}")) + .output() + .unwrap(); + let missing_old_blob = format!("?{old_blob}"); + assert!( + String::from_utf8_lossy(&old_snapshot.stdout) + .lines() + .any(|line| line == missing_old_blob), + "materializing HEAD must not fetch blobs used only by its history" + ); + + let checkout = root.path().join("checkout"); + checkout_repo_from_source(&source, &repo, &commit, &checkout, TerminalPrompt::Disabled) + .await + .unwrap(); + assert_eq!( + fs_err::read_to_string(checkout.join("hook.py")).unwrap(), + "print('hello')\n" + ); + assert_eq!(git_stdout(&checkout, &["rev-list", "--count", "HEAD"]), "1"); + Command::new(GIT.as_ref().unwrap()) + .current_dir(&checkout) + .args(["fsck", "--full"]) + .assert() + .success(); } #[tokio::test] diff --git a/crates/prek/src/store.rs b/crates/prek/src/store.rs index 11a0d3885..1b16c1da8 100644 --- a/crates/prek/src/store.rs +++ b/crates/prek/src/store.rs @@ -95,6 +95,8 @@ impl Store { /// Initialize the store. pub(crate) fn init(self) -> Result { fs_err::create_dir_all(&self.path)?; + fs_err::create_dir_all(self.repo_sources_dir())?; + fs_err::create_dir_all(self.repo_source_locks_dir())?; fs_err::create_dir_all(self.repos_dir())?; fs_err::create_dir_all(self.hooks_dir())?; fs_err::create_dir_all(self.scratch_path())?; @@ -116,13 +118,25 @@ impl Store { terminal_prompt: TerminalPrompt, ) -> Result { let temp = tempfile::tempdir_in(self.scratch_path())?; + let source = self.repo_source_path(repo.source()); + let _source_lock = self.repo_source_lock(repo.source()).await?; debug!( + source = %source.display(), target = %temp.path().display(), %repo, ?terminal_prompt, - "Cloning repo" + "Preparing repo checkout" ); - git::clone_repo(repo.source(), &repo.rev, temp.path(), terminal_prompt).await?; + git::ensure_bare_repo(repo.source(), &source).await?; + let revision = git::fetch_repo_source_revision(&source, &repo.rev, terminal_prompt).await?; + git::checkout_repo_from_source( + &source, + repo.source(), + &revision, + temp.path(), + terminal_prompt, + ) + .await?; Ok(temp) } @@ -289,11 +303,28 @@ impl Store { LockedFile::acquire(self.path.join(".lock"), "store").await } + pub(crate) async fn repo_source_lock( + &self, + source: &str, + ) -> Result { + LockedFile::acquire( + self.repo_source_locks_dir() + .join(format!("{}.lock", Self::repo_source_key(source))), + format!("repo source `{source}`"), + ) + .await + } + /// Returns the path to where a remote repo would be stored. pub(crate) fn repo_path(&self, repo: &RemoteRepo) -> PathBuf { self.repos_dir().join(Self::repo_key(repo)) } + /// Returns the path to the shared bare source for a remote repo. + pub(crate) fn repo_source_path(&self, source: &str) -> PathBuf { + self.repo_sources_dir().join(Self::repo_source_key(source)) + } + /// Returns the store key (directory name) for a remote repo. pub(crate) fn repo_key(repo: &RemoteRepo) -> String { let mut hasher = SeaHasher::new(); @@ -302,6 +333,21 @@ impl Store { to_hex(hasher.finish()) } + /// Returns the store key for the shared bare source of a remote repo. + pub(crate) fn repo_source_key(source: &str) -> String { + let mut hasher = SeaHasher::new(); + source.hash(&mut hasher); + to_hex(hasher.finish()) + } + + pub(crate) fn repo_sources_dir(&self) -> PathBuf { + self.path.join("repo-sources") + } + + fn repo_source_locks_dir(&self) -> PathBuf { + self.repo_sources_dir().join(".locks") + } + pub(crate) fn repos_dir(&self) -> PathBuf { self.path.join("repos") } diff --git a/crates/prek/tests/cache.rs b/crates/prek/tests/cache.rs index e6e635058..148b54f5c 100644 --- a/crates/prek/tests/cache.rs +++ b/crates/prek/tests/cache.rs @@ -278,6 +278,7 @@ fn cache_gc_removes_unreferenced_entries() -> anyhow::Result<()> { // Add a few obviously-unused entries. home.child("repos/unused-repo").create_dir_all()?; + home.child("repo-sources/unused-source").create_dir_all()?; home.child("hooks/unused-hook-env").create_dir_all()?; home.child("tools/node").create_dir_all()?; home.child("cache/go").create_dir_all()?; @@ -295,13 +296,15 @@ fn cache_gc_removes_unreferenced_entries() -> anyhow::Result<()> { success: true exit_code: 0 ----- stdout ----- - Removed 1 repo, 2 hook envs, 1 tool, 1 cache entry ([SIZE]) + Removed 1 repo, 1 repo source, 2 hook envs, 1 tool, 1 cache entry ([SIZE]) ----- stderr ----- "#); home.child("repos/unused-repo") .assert(predicates::path::missing()); + home.child("repo-sources/unused-source") + .assert(predicates::path::missing()); home.child("hooks/unused-hook-env") .assert(predicates::path::missing()); home.child("tools/node").assert(predicates::path::missing()); @@ -365,6 +368,11 @@ fn cache_gc_keeps_relative_remote_repo() -> anyhow::Result<()> { .transpose()? .expect("expected the relative remote repo to be cached") .path(); + let cached_source = fs_err::read_dir(context.home_dir().child("repo-sources").path())? + .filter_map(Result::ok) + .find(|entry| !entry.file_name().to_string_lossy().starts_with('.')) + .expect("expected the relative remote repo source to be cached") + .path(); repos_dir.child("unused-repo").create_dir_all()?; cmd_snapshot!(context.filters(), context.command().args(["cache", "gc"]), @r" @@ -377,6 +385,10 @@ fn cache_gc_keeps_relative_remote_repo() -> anyhow::Result<()> { "); assert!(cached_repo.is_dir(), "cache GC removed the configured repo"); + assert!( + cached_source.is_dir(), + "cache GC removed the configured repo source" + ); repos_dir .child("unused-repo") .assert(predicates::path::missing()); @@ -888,8 +900,10 @@ fn cache_gc_drops_missing_tracked_config() -> anyhow::Result<()> { let tracked: Vec = serde_json::from_str(&content)?; assert!(tracked.is_empty()); - // Scratch is always cleared. Patch directories remain unless they contain stale patch files. - home.child("scratch").assert(predicates::path::missing()); + // Scratch contents are cleared; patch directories remain unless they contain stale files. + home.child("scratch").assert(predicates::path::is_dir()); + home.child("scratch/some-temp") + .assert(predicates::path::missing()); home.child("patches").assert(predicates::path::is_dir()); Ok(()) diff --git a/crates/prek/tests/run.rs b/crates/prek/tests/run.rs index e455912a3..6a94f5599 100644 --- a/crates/prek/tests/run.rs +++ b/crates/prek/tests/run.rs @@ -15,6 +15,14 @@ use crate::common::{TestContext, cmd_snapshot, git_cmd}; mod common; +fn non_hidden_directory_count(root: &Path) -> Result { + Ok(fs_err::read_dir(root)? + .filter_map(std::result::Result::ok) + .filter(|entry| !entry.file_name().to_string_lossy().starts_with('.')) + .filter(|entry| entry.file_type().is_ok_and(|kind| kind.is_dir())) + .count()) +} + #[test] fn run_basic() -> Result<()> { let context = TestContext::new(); @@ -358,6 +366,15 @@ fn same_repo() -> Result<()> { ----- stderr ----- "); + assert_eq!( + non_hidden_directory_count(context.home_dir().child("repo-sources").path())?, + 1 + ); + assert_eq!( + non_hidden_directory_count(context.home_dir().child("repos").path())?, + 2 + ); + Ok(()) } diff --git a/crates/prek/tests/try_repo.rs b/crates/prek/tests/try_repo.rs index c4f3260fe..97441dce9 100644 --- a/crates/prek/tests/try_repo.rs +++ b/crates/prek/tests/try_repo.rs @@ -8,6 +8,14 @@ use crate::common::{TestContext, cmd_snapshot, git_cmd}; use assert_fs::fixture::ChildPath; use prek_consts::PRE_COMMIT_HOOKS_YAML; +fn non_hidden_directory_count(root: &ChildPath) -> Result { + Ok(fs_err::read_dir(root.path())? + .filter_map(std::result::Result::ok) + .filter(|entry| !entry.file_name().to_string_lossy().starts_with('.')) + .filter(|entry| entry.file_type().is_ok_and(|kind| kind.is_dir())) + .count()) +} + fn create_hook_repo(context: &TestContext, repo_name: &str) -> Result { let repo_dir = context.home_dir().child(format!("test-repos/{repo_name}")); repo_dir.create_dir_all()?; @@ -308,11 +316,7 @@ fn try_repo_uncommitted_changes() -> Result<()> { context.git_add("."); let mut filters = context.filters(); - filters.extend([ - (r"try-repo-[^/\\]+", "[REPO]"), - (r"[a-f0-9]{40}", "[COMMIT_SHA]"), - ("'", "\""), - ]); + filters.extend([(r"[a-f0-9]{40}", "[COMMIT_SHA]"), ("'", "\"")]); cmd_snapshot!(filters, context.try_repo().arg(&repo_path), @r#" success: true @@ -320,7 +324,7 @@ fn try_repo_uncommitted_changes() -> Result<()> { ----- stdout ----- Using generated `prek.toml`: [[repos]] - repo = "[HOME]/scratch/[REPO]/shadow-repo" + repo = "[HOME]/test-repos/try-repo-uncommitted" rev = "[COMMIT_SHA]" hooks = [ { id = "uncommitted-hook" }, @@ -332,6 +336,16 @@ fn try_repo_uncommitted_changes() -> Result<()> { warning: Creating temporary repo with uncommitted changes... "#); + context.try_repo().arg(&repo_path).assert().success(); + assert_eq!( + non_hidden_directory_count(&context.home_dir().child("repo-sources"))?, + 1 + ); + assert_eq!( + non_hidden_directory_count(&context.home_dir().child("repos"))?, + 1 + ); + Ok(()) } diff --git a/crates/prek/tests/update.rs b/crates/prek/tests/update.rs index 83875e51f..b6b7d6d39 100644 --- a/crates/prek/tests/update.rs +++ b/crates/prek/tests/update.rs @@ -4,6 +4,7 @@ use assert_fs::fixture::ChildPath; use assert_fs::prelude::*; use insta::assert_snapshot; use prek_consts::{PRE_COMMIT_CONFIG_YAML, PREK_TOML}; +use std::path::PathBuf; use crate::common::{TestContext, cmd_snapshot, git_cmd}; @@ -158,6 +159,12 @@ fn update_basic() -> Result<()> { - id: test-hook ", repo_path}); context.git_add("."); + // An existing tracking file disables workspace-cache bootstrapping. `update` must register + // this config itself so its persistent repo source remains live during cache GC. + context + .home_dir() + .child("config-tracking.json") + .write_str("[]")?; let filters = context.filters(); @@ -184,6 +191,20 @@ fn update_basic() -> Result<()> { } ); + let tracked: Vec = serde_json::from_str(&fs_err::read_to_string( + context.home_dir().child("config-tracking.json").path(), + )?)?; + assert_eq!( + tracked, + vec![ + context + .work_dir() + .child(PRE_COMMIT_CONFIG_YAML) + .path() + .to_path_buf() + ] + ); + Ok(()) } @@ -1726,6 +1747,10 @@ fn update_warns_for_branch_only_pinned_commit_with_frozen_comment() -> Result<() context.git_add("."); + // Populate the persistent repo source with this branch-only commit through the normal run + // path. Update must not mistake an accumulated object for a commit reachable from HEAD/tags. + context.run().arg("--all-files").assert().success(); + let filters = context .filters() .into_iter()