InferredCopyFrom: consider basename match

Summary:
This is another relatively cheap copy-detection heuristic, it roughly goes as:
* extract all the basenames of the new/changed files and all the top-level directories they are in from the current changeset
* search within the same those directories for files with the same basenames
* if any pair has the same `content_id`, record them as copy

This makes this type depends on BSSM where we query for files using basenames.

Differential Revision: D75554932

fbshipit-source-id: b3c40abbb6017e21d6a90a74b90e8d7b6d1c7e29

Author: Liu Yang

eden/mononoke/derived_data/BUCK

@@ -578,6 +578,9 @@ rust_library(
         "fbsource//third-party/rust:anyhow",
         "fbsource//third-party/rust:async-trait",
         "fbsource//third-party/rust:futures",
+        "fbsource//third-party/rust:itertools",
+        "fbsource//third-party/rust:vec1",
+        ":basename_suffix_skeleton_manifest_v3",
         ":fsnodes",
         "//common/rust/shed/cloned:cloned",
         "//eden/mononoke/blobstore:blobstore",

eden/mononoke/derived_data/inferred_copy_from/Cargo.toml

@@ -13,15 +13,18 @@ path = "lib.rs"
 [dependencies]
 anyhow = "1.0.95"
 async-trait = "0.1.86"
+basename_suffix_skeleton_manifest_v3 = { version = "0.1.0", path = "../basename_suffix_skeleton_manifest_v3" }
 blobstore = { version = "0.1.0", path = "../../blobstore" }
 cloned = { version = "0.1.0", git = "https://github.com/facebookexperimental/rust-shed.git", branch = "main" }
 context = { version = "0.1.0", path = "../../server/context" }
 derived_data_manager = { version = "0.1.0", path = "../manager" }
 derived_data_service_if = { version = "0.1.0", path = "../remote/if" }
 fsnodes = { version = "0.1.0", path = "../fsnodes" }
 futures = { version = "0.3.30", features = ["async-await", "compat"] }
+itertools = "0.14.0"
 manifest = { version = "0.1.0", path = "../../manifest" }
 mononoke_types = { version = "0.1.0", path = "../../mononoke_types" }
+vec1 = { version = "1", features = ["serde"] }
 
 [dev-dependencies]
 facet = { version = "0.1.0", git = "https://github.com/facebookexperimental/rust-shed.git", branch = "main" }

eden/mononoke/derived_data/inferred_copy_from/derive.rs

@@ -6,15 +6,21 @@
  */
 
 use std::collections::HashMap;
+use std::collections::HashSet;
 
 use anyhow::Context;
+use anyhow::Error;
 use anyhow::Result;
+use basename_suffix_skeleton_manifest_v3::RootBssmV3DirectoryId;
 use cloned::cloned;
 use context::CoreContext;
 use derived_data_manager::DerivationContext;
 use fsnodes::RootFsnodeId;
+use futures::Stream;
+use futures::StreamExt;
 use futures::future::try_join_all;
 use futures::stream::TryStreamExt;
+use itertools::EitherOrBoth;
 use manifest::ManifestOps;
 use mononoke_types::BonsaiChangeset;
 use mononoke_types::ChangesetId;
@@ -23,6 +29,9 @@ use mononoke_types::MPath;
 use mononoke_types::NonRootMPath;
 use mononoke_types::inferred_copy_from::InferredCopyFrom;
 use mononoke_types::inferred_copy_from::InferredCopyFromEntry;
+use vec1::Vec1;
+
+const BASENAME_MATCH_MAX_CANDIDATES: usize = 10_000;
 
 // It's possible to have multiple source files that match,
 // pick the one with the smallest path
@@ -59,6 +68,26 @@ async fn get_content_to_paths_from_changeset(
     Ok(content_to_paths)
 }
 
+async fn get_matched_paths_by_basenames_from_changeset(
+    ctx: &CoreContext,
+    derivation_ctx: &DerivationContext,
+    cs_id: ChangesetId,
+    basenames: Vec<String>,
+    path_prefixes: Vec<MPath>,
+) -> Result<impl Stream<Item = Result<MPath, Error>>> {
+    derivation_ctx
+        .fetch_dependency::<RootBssmV3DirectoryId>(ctx, cs_id)
+        .await?
+        .find_files_filter_basenames(
+            ctx,
+            derivation_ctx.blobstore().clone(),
+            path_prefixes,
+            EitherOrBoth::Left(Vec1::try_from_vec(basenames)?),
+            None,
+        )
+        .await
+}
+
 // Find exact renames by comparing the content of deleted vs new/changed files
 // in the current changeset. If they have the same content, the path pair is
 // a rename.
@@ -123,15 +152,104 @@ async fn find_exact_renames(
     Ok(renames)
 }
 
+// Infer copies by matching basenames between new/changed files in the
+// current changeset and other files in the same repo (with some constraints).
+// If the basenames match and the content are the same, the path pair is a copy.
+async fn find_basename_matched_copies(
+    ctx: &CoreContext,
+    derivation_ctx: &DerivationContext,
+    bonsai: &BonsaiChangeset,
+    paths_to_ignore: &HashSet<MPath>,
+) -> Result<Vec<(MPath, InferredCopyFromEntry)>> {
+    let mut content_to_paths = HashMap::new();
+    let mut basenames = HashSet::new();
+    let mut path_prefixes = HashSet::new();
+    for (path, file_change) in bonsai.simplified_file_changes() {
+        if !paths_to_ignore.contains(path.into()) {
+            if let Some(fc) = file_change {
+                content_to_paths
+                    .entry(fc.content_id())
+                    .or_insert(vec![])
+                    .push(path.clone());
+
+                basenames.insert(path.basename().to_string());
+                // Restrict search to any of the touched top-level directory
+                if let Some(path_prefix) = path.take_prefix_components(1)? {
+                    path_prefixes.insert(MPath::from(path_prefix));
+                }
+            }
+        }
+    }
+    if basenames.is_empty() {
+        return Ok(vec![]);
+    }
+
+    let basenames_vec = basenames.into_iter().collect::<Vec<_>>();
+    let path_prefixes_vec = path_prefixes.into_iter().collect::<Vec<_>>();
+    let mut content_to_matched_paths = HashMap::new();
+
+    for parent_cs_id in bonsai.parents() {
+        content_to_matched_paths.extend(
+            get_matched_paths_by_basenames_from_changeset(
+                ctx,
+                derivation_ctx,
+                parent_cs_id,
+                basenames_vec.clone(),
+                path_prefixes_vec.clone(),
+            )
+            .await?
+            .try_filter_map(async move |path| Ok(path.into_optional_non_root_path()))
+            .take(BASENAME_MATCH_MAX_CANDIDATES)
+            .try_chunks(100)
+            .try_fold(HashMap::new(), |mut acc, paths| async move {
+                let hashmap =
+                    get_content_to_paths_from_changeset(ctx, derivation_ctx, parent_cs_id, paths)
+                        .await;
+                if let Ok(hashmap) = hashmap {
+                    acc.extend(hashmap.into_iter());
+                }
+                Ok(acc)
+            })
+            .await?
+            .into_iter(),
+        );
+    }
+
+    let mut copies = vec![];
+    for (content_id, paths) in content_to_paths {
+        if let Some(matched_paths) = content_to_matched_paths.get(&content_id) {
+            let from = pick_source_from_candidates(matched_paths).unwrap();
+            for path in paths {
+                copies.push((
+                    MPath::from(path),
+                    InferredCopyFromEntry {
+                        from_csid: from.0,
+                        from_path: from.1.clone(),
+                    },
+                ));
+            }
+        }
+    }
+    Ok(copies)
+}
+
+// TODO: add more cases
+// Ref: https://github.com/git/git/blob/master/diffcore-rename.c
 pub(crate) async fn derive_impl(
     ctx: &CoreContext,
     derivation_ctx: &DerivationContext,
     bonsai: &BonsaiChangeset,
 ) -> Result<Option<InferredCopyFrom>> {
-    // TODO: add more cases
-    // Ref: https://github.com/git/git/blob/master/diffcore-rename.c
-    let entries = find_exact_renames(ctx, derivation_ctx, bonsai).await?;
+    let mut resolved_paths = HashSet::new();
+
+    let exact_renames = find_exact_renames(ctx, derivation_ctx, bonsai).await?;
+    resolved_paths.extend(exact_renames.iter().map(|(path, _)| path.clone()));
+
+    let basename_matched_copies =
+        find_basename_matched_copies(ctx, derivation_ctx, bonsai, &resolved_paths).await?;
+    resolved_paths.extend(basename_matched_copies.iter().map(|(path, _)| path.clone()));
 
+    let entries = [exact_renames, basename_matched_copies].concat();
     if entries.is_empty() {
         Ok(None)
     } else {

eden/mononoke/derived_data/inferred_copy_from/mapping.rs

@@ -12,6 +12,7 @@ use anyhow::Error;
 use anyhow::Result;
 use anyhow::anyhow;
 use async_trait::async_trait;
+use basename_suffix_skeleton_manifest_v3::RootBssmV3DirectoryId;
 use blobstore::BlobstoreGetData;
 use blobstore::Storable;
 use context::CoreContext;
@@ -73,7 +74,7 @@ impl RootInferredCopyFromId {
 impl BonsaiDerivable for RootInferredCopyFromId {
     const VARIANT: DerivableType = DerivableType::InferredCopyFrom;
 
-    type Dependencies = dependencies![RootFsnodeId];
+    type Dependencies = dependencies![RootFsnodeId, RootBssmV3DirectoryId];
     type PredecessorDependencies = dependencies![];
 
     async fn derive_single(

eden/mononoke/derived_data/inferred_copy_from/tests.rs

@@ -71,6 +71,39 @@ async fn init_repo(ctx: &CoreContext) -> Result<(Repo, HashMap<&'static str, Cha
             .await?,
     );
 
+    changesets.insert(
+        "d",
+        CreateCommitContext::new(ctx, &repo, vec![changesets["c"]])
+            .add_file("path/to/basename1", "aabbcc\n")
+            .add_file("path/to/basename2", "ddeeff\n")
+            .set_author_date(DateTime::from_timestamp(1000, 0)?)
+            .commit()
+            .await?,
+    );
+    changesets.insert(
+        "e",
+        CreateCommitContext::new(ctx, &repo, vec![changesets["d"]])
+            // Inferred copies:
+            // d:path/to/basename1 -> path/basename1
+            // d:path/to/basename2 -> path/basename2
+            // d:path/to/basename2 -> another/path/basename2
+            .add_file("path/basename1", "aabbcc\n")
+            .add_file("path/basename2", "ddeeff\n")
+            .add_file("another/path/basename2", "ddeeff\n")
+            .set_author_date(DateTime::from_timestamp(1000, 0)?)
+            .commit()
+            .await?,
+    );
+    changesets.insert(
+        "f",
+        CreateCommitContext::new(ctx, &repo, vec![changesets["d"]])
+            // Not detected due to the directory constraint.
+            .add_file("another/path/basename2", "ddeeff\n")
+            .set_author_date(DateTime::from_timestamp(1000, 0)?)
+            .commit()
+            .await?,
+    );
+
     Ok((repo, changesets))
 }
 
@@ -126,6 +159,44 @@ async fn derive_single_test(fb: FacebookInit) -> Result<()> {
     )
     .await?;
 
+    assert_entries(
+        &ctx,
+        &repo,
+        repo_ctx.changeset(changesets["e"]).await?.unwrap().id(),
+        &[
+            (
+                MPath::new("another/path/basename2")?,
+                InferredCopyFromEntry {
+                    from_csid: changesets["d"],
+                    from_path: MPath::new("path/to/basename2")?,
+                },
+            ),
+            (
+                MPath::new("path/basename1")?,
+                InferredCopyFromEntry {
+                    from_csid: changesets["d"],
+                    from_path: MPath::new("path/to/basename1")?,
+                },
+            ),
+            (
+                MPath::new("path/basename2")?,
+                InferredCopyFromEntry {
+                    from_csid: changesets["d"],
+                    from_path: MPath::new("path/to/basename2")?,
+                },
+            ),
+        ],
+    )
+    .await?;
+
+    assert_entries(
+        &ctx,
+        &repo,
+        repo_ctx.changeset(changesets["f"]).await?.unwrap().id(),
+        &[],
+    )
+    .await?;
+
     Ok(())
 }