Skip to content

Reuse remote repository sources #2315

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
j178 wants to merge 1 commit into
base: master
Choose a base branch
from
Draft

Reuse remote repository sources #2315

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 30 additions & 3 deletions crates/prek/src/cli/cache_gc.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ use crate::store::{CacheBucket, REPO_MARKER, Store, ToolBucket};
#[derive(Debug, Copy, Clone, Eq, PartialEq)]
enum RemovalKind {
Repos,
RepoSources,
HookEnvs,
Tools,
CacheEntries,
Expand All @@ -36,6 +37,7 @@ impl RemovalKind {
if count > 1 {
match self {
RemovalKind::Repos => "repos",
RemovalKind::RepoSources => "repo sources",
RemovalKind::HookEnvs => "hook envs",
RemovalKind::Tools => "tools",
RemovalKind::CacheEntries => "cache entries",
Expand All @@ -44,6 +46,7 @@ impl RemovalKind {
} else {
match self {
RemovalKind::Repos => "repo",
RemovalKind::RepoSources => "repo source",
RemovalKind::HookEnvs => "hook env",
RemovalKind::Tools => "tool",
RemovalKind::CacheEntries => "cache entry",
Expand Down Expand Up @@ -159,6 +162,7 @@ pub(crate) async fn cache_gc(

let mut kept_configs: FxHashSet<&Path> = FxHashSet::default();
let mut used_repo_keys: FxHashSet<String> = FxHashSet::default();
let mut used_repo_source_keys: FxHashSet<String> = FxHashSet::default();
let mut used_hook_env_dirs: FxHashSet<String> = FxHashSet::default();
let mut used_tools: FxHashSet<ToolBucket> = FxHashSet::default();
let mut used_tool_versions: FxHashMap<ToolBucket, FxHashSet<String>> = FxHashMap::default();
Expand All @@ -182,7 +186,7 @@ pub(crate) async fn cache_gc(
continue;
}
err => {
warn!(path = %config_path.display(), %err, "Failed to parse config, skipping for GC");
warn!(path = %config_path.display(), %err, "Failed to load config, skipping for GC");
kept_configs.insert(config_path);
continue;
}
Expand All @@ -204,6 +208,7 @@ pub(crate) async fn cache_gc(
if let Some(key) = key {
used_repo_keys.insert(key);
}
used_repo_source_keys.insert(Store::repo_source_key(remote.source()));
}
}
}
Expand Down Expand Up @@ -251,6 +256,15 @@ pub(crate) async fn cache_gc(
verbose,
)?;

// Sweep repo-sources/<hash(repo)>; the hidden lock directory is ignored by the sweeper.
let removed_repo_sources = sweep_dir_by_name(
RemovalKind::RepoSources,
&store.repo_sources_dir(),
&used_repo_source_keys,
dry_run,
verbose,
)?;

// Sweep hooks/<hash>
let removed_hooks = sweep_dir_by_name(
RemovalKind::HookEnvs,
Expand Down Expand Up @@ -288,15 +302,15 @@ pub(crate) async fn cache_gc(
verbose,
)?;

// Seep scratch/, as it is only temporary data.
if !dry_run {
let _ = fs_err::remove_dir_all(store.scratch_path());
clear_directory(&store.scratch_path())?;
}
// Keep recent recovery patches, but clear out stale ones that are unlikely to be useful.
let removed_patches = sweep_stale_patch_files(&store.patches_dir(), dry_run, verbose)?;

let mut removed = RemovalSummary::default();
removed += &removed_repos;
removed += &removed_repo_sources;
removed += &removed_hooks;
removed += &removed_tools;
removed += &removed_cache;
Expand All @@ -318,6 +332,7 @@ pub(crate) async fn cache_gc(

if verbose {
print_removed_details(printer, verb, &removed_repos)?;
print_removed_details(printer, verb, &removed_repo_sources)?;
print_removed_details(printer, verb, &removed_hooks)?;
print_removed_details(printer, verb, &removed_tools)?;
print_removed_details(printer, verb, &removed_cache)?;
Expand All @@ -328,6 +343,18 @@ pub(crate) async fn cache_gc(
Ok(ExitStatus::Success)
}

fn clear_directory(path: &Path) -> std::io::Result<()> {
for entry in fs_err::read_dir(path)? {
let entry = entry?;
if entry.file_type()?.is_dir() {
fs_err::remove_dir_all(entry.path())?;
} else {
fs_err::remove_file(entry.path())?;
}
}
Ok(())
}

fn print_removed_details(printer: Printer, verb: &str, removal: &Removal) -> Result<()> {
if removal.count == 0 {
return Ok(());
Expand Down
155 changes: 51 additions & 104 deletions crates/prek/src/cli/try_repo.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,77 +29,15 @@ async fn get_head_rev(repo: &Path) -> Result<String> {
Ok(head_rev)
}

async fn clone_and_commit(repo_path: &Path, head_rev: &str, tmp_dir: &Path) -> Result<PathBuf> {
let shadow = tmp_dir.join("shadow-repo");
git::git_cmd()?
.arg("clone")
.arg(repo_path)
.arg(&shadow)
.output()
.await?;
git::git_cmd()?
.arg("checkout")
.arg(head_rev)
.arg("-b")
.arg("_prek_tmp")
.current_dir(&shadow)
.output()
.await?;

let index_path = shadow.join(".git/index");
let objects_path = shadow.join(".git/objects");

let staged_files = git::get_staged_files(repo_path).await?;
if !staged_files.is_empty() {
git::git_cmd()?
.arg("add")
.arg("--")
.file_args(&staged_files)
.current_dir(repo_path)
.env("GIT_INDEX_FILE", &index_path)
.env("GIT_OBJECT_DIRECTORY", &objects_path)
.output()
.await?;
}

let mut add_u_cmd = git::git_cmd()?;
add_u_cmd
.arg("add")
.arg("--update") // Update tracked files
.current_dir(repo_path)
.env("GIT_INDEX_FILE", &index_path)
.env("GIT_OBJECT_DIRECTORY", &objects_path)
.output()
.await?;

git::git_cmd()?
.arg("commit")
.arg("-m")
.arg("Temporary commit by prek try-repo")
.arg("--no-gpg-sign")
.arg("--no-edit")
.arg("--no-verify")
.current_dir(&shadow)
.env("GIT_AUTHOR_NAME", "prek test")
.env("GIT_AUTHOR_EMAIL", "[email protected]")
.env("GIT_COMMITTER_NAME", "prek test")
.env("GIT_COMMITTER_EMAIL", "[email protected]")
.output()
.await?;

Ok(shadow)
}

struct PreparedRepo<'a> {
runtime_source: Cow<'a, str>,
display_source: Option<&'a str>,
source: Cow<'a, str>,
rev: String,
}

async fn prepare_repo<'a>(
store: &Store,
repo: &'a str,
rev: Option<&str>,
tmp_dir: &Path,
) -> Result<PreparedRepo<'a>> {
let repo_path = Path::new(repo);
let is_local = repo_path.is_dir();
Expand All @@ -117,8 +55,7 @@ async fn prepare_repo<'a>(
// If rev is provided, use it directly.
if let Some(rev) = rev {
return Ok(PreparedRepo {
runtime_source,
display_source: is_local.then_some(repo),
source: runtime_source,
rev: rev.to_string(),
});
}
Expand All @@ -143,20 +80,25 @@ async fn prepare_repo<'a>(
.to_string()
};

// If repo is a local repo with uncommitted changes, create a shadow repo to commit the changes.
// Persist a deterministic synthetic commit in the shared source. The logical source remains
// the canonical local path, so identical dirty trees get the same repo and environment keys.
if is_local && git::has_diff("HEAD", repo_path).await? {
warn_user!("Creating temporary repo with uncommitted changes...");
let shadow = clone_and_commit(repo_path, &head_rev, tmp_dir).await?;
let head_rev = get_head_rev(&shadow).await?;
let source = store.repo_source_path(runtime_source.as_ref());
let _source_lock = store.repo_source_lock(runtime_source.as_ref()).await?;
git::ensure_bare_repo(runtime_source.as_ref(), &source).await?;
let head_rev =
git::fetch_repo_source_revision(&source, &head_rev, git::TerminalPrompt::Disabled)
.await?;
let snapshot =
git::create_repo_snapshot(&source, repo_path, &head_rev, &store.scratch_path()).await?;
Ok(PreparedRepo {
runtime_source: Cow::Owned(shadow.to_string_lossy().into_owned()),
display_source: None,
rev: head_rev,
source: runtime_source,
rev: snapshot,
})
} else {
Ok(PreparedRepo {
runtime_source,
display_source: is_local.then_some(repo),
source: runtime_source,
rev: head_rev,
})
}
Expand Down Expand Up @@ -202,41 +144,46 @@ pub(crate) async fn try_repo(
}

let store = Store::from_settings()?;
let tmp_dir = TempDir::with_prefix_in("try-repo-", store.scratch_path())?;

let prepared = prepare_repo(&repo, rev.as_deref(), tmp_dir.path())
.await
.context("Failed to determine repository and revision")?;

let store = Store::from_path(tmp_dir.path()).init()?;
let repo_config = config::RemoteRepo::new(
prepared.runtime_source.to_string(),
prepared.rev.clone(),
vec![],
);
let repo_clone_path = store.clone_repo(&repo_config, None).await?;

let selectors = Selectors::load(&run_args.includes, &run_args.skips, GIT_ROOT.as_ref()?)?;

let manifest =
config::read_manifest(&repo_clone_path.join(prek_consts::PRE_COMMIT_HOOKS_YAML))?;

let hooks = manifest
.hooks
.into_iter()
.filter(|hook| selectors.matches_hook_id(&hook.id))
.map(|hook| hook.id)
.collect::<Vec<_>>();
let (_tmp_dir, prepared, hooks, config_str, config_file) = {
let _lock = store.lock_async().await?;
// `cache gc` clears the contents of the store scratch directory after taking the store
// lock. Keep the active generated config in the system temporary directory so it survives
// between this preparation lock and the lock acquired by `run`.
let tmp_dir = TempDir::with_prefix("try-repo-")?;
let prepared = prepare_repo(&store, &repo, rev.as_deref())
.await
.context("Failed to determine repository and revision")?;
let repo_config =
config::RemoteRepo::new(prepared.source.to_string(), prepared.rev.clone(), vec![]);
let repo_clone_path = store.clone_repo(&repo_config, None).await?;
let manifest =
config::read_manifest(&repo_clone_path.join(prek_consts::PRE_COMMIT_HOOKS_YAML))?;
let hooks = manifest
.hooks
.into_iter()
.filter(|hook| selectors.matches_hook_id(&hook.id))
.map(|hook| hook.id)
.collect::<Vec<_>>();

let config_str = render_repo_config_toml(&prepared.source, &prepared.rev, &hooks);
let config_file = tmp_dir.path().join(PREK_TOML);
fs_err::tokio::write(&config_file, &config_str).await?;
// Make the new source/checkout visible to GC before releasing the store lock. `run` also
// tracks this path, but a concurrent `cache gc` must not sweep a dirty synthetic commit in
// the interval between preparation and hook initialization.
store.track_configs(std::iter::once(config_file.as_path()))?;

(tmp_dir, prepared, hooks, config_str, config_file)
};

// The scratch config needs the resolved source, while the displayed config should preserve
// the user's path so it remains meaningful when copied into their project.
let config_str = render_repo_config_toml(&prepared.runtime_source, &prepared.rev, &hooks);
let config_file = tmp_dir.path().join(PREK_TOML);
fs_err::tokio::write(&config_file, &config_str).await?;

let display_config_str = match prepared.display_source {
Some(source) => Cow::Owned(render_repo_config_toml(source, &prepared.rev, &hooks)),
None => Cow::Borrowed(config_str.as_str()),
let display_config_str = if prepared.source.as_ref() == repo {
Cow::Borrowed(config_str.as_str())
} else {
Cow::Owned(render_repo_config_toml(&repo, &prepared.rev, &hooks))
};

writeln!(
Expand Down
13 changes: 12 additions & 1 deletion crates/prek/src/cli/update/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -418,10 +418,20 @@ pub(crate) async fn update(
filesystem: Option<FilesystemOptions>,
printer: Printer,
) -> Result<ExitStatus> {
// Repository sources are shared with normal hook initialization and try-repo. Keep the
// command-level store invariant while per-source locks below serialize individual fetches.
let _store_lock = store.lock_async().await?;

let workspace_root = Workspace::find_root(config.as_deref(), &CWD)?;
// TODO: support selectors?
let selectors = Selectors::default();
let workspace = Workspace::discover(store, workspace_root, config, Some(&selectors), true)?;
store.track_configs(
workspace
.projects()
.iter()
.map(|project| project.config_file()),
)?;

let tag_filters =
TagFilters::new(include_tag, exclude_tag, repo_include_tag, repo_exclude_tag)?;
Expand Down Expand Up @@ -450,7 +460,8 @@ pub(crate) async fn update(
.first()
.map_or(repo_source.source, |target| target.repo);
let progress = reporter.on_update_start(display_repo);
let result = evaluate_repo_source(repo_source, bleeding_edge, &tag_filters).await;
let result =
evaluate_repo_source(store, repo_source, bleeding_edge, &tag_filters).await;
reporter.on_update_complete(progress);
result
})
Expand Down
Loading
Loading