Skip to content

Commit b0c979a

Browse files
Prashant Palmeta-codesync[bot]
authored andcommitted
Add diagnostic logging to shard setup and repo facet initialization
Summary: TW task tsp_global/mononoke/mononoke_git_server/59 had 9 shards stuck in Setup state for 1+ hour with no errors logged. Investigation revealed that repo_factory.build() uses futures::try_join! to build ~30 facets concurrently — all must complete for build() to return. SQL-backed facets have 10s query timeouts but non-SQL facets (blobstore fetches, Scribe subscriptions) have no timeout. A single stuck facet blocks the entire build() and keeps the shard in InProgress state indefinitely with no diagnostic output. This diff adds start/finish logging to pinpoint which facet is stuck: - Setup task: logs completion time and success/failure - WBC: logs after create_subscription and after coordinator spawn - Preloaded commit graph: adds blobstore_key to Started/Finished logs for repo identification - Suspect facets in repo_factory: start/finish logging for bonsai_tag_mapping, git_ref_content_mapping, commit_graph, cgdm_changeset_divider, hook_manager Reviewed By: RajivTS Differential Revision: D102354775 fbshipit-source-id: f075dee72d31aa989910abcd2be69bda56b363fd
1 parent e0cd737 commit b0c979a

4 files changed

Lines changed: 260 additions & 159 deletions

File tree

  • eden/mononoke
    • cmdlib/sharding/src
    • repo_attributes
      • bookmarks/warm_bookmarks_cache/src
      • commit_graph/preloaded_commit_graph_storage/src
    • repo_factory/src

eden/mononoke/cmdlib/sharding/src/facebook.rs

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -133,10 +133,24 @@ impl RepoSetupProcess {
133133
setup_handle: runtime_handle.spawn({
134134
let repo_shard = repo_shard.clone();
135135
async move {
136-
// Create guard that will decrement counter on drop
137136
let _guard = ConcurrentSetupGuard::new(fb);
138-
139-
setup_job.setup(&repo_shard).await
137+
let start = time::Instant::now();
138+
let result = setup_job.setup(&repo_shard).await;
139+
let elapsed_s = start.elapsed().as_secs_f64();
140+
match &result {
141+
Ok(_) => info!(
142+
shard = %repo_shard,
143+
elapsed_s,
144+
"Setup for shard completed"
145+
),
146+
Err(e) => error!(
147+
shard = %repo_shard,
148+
elapsed_s,
149+
error = %format!("{:#}", e),
150+
"Setup for shard failed"
151+
),
152+
}
153+
result
140154
}
141155
}),
142156
shard,

eden/mononoke/repo_attributes/bookmarks/warm_bookmarks_cache/src/lib.rs

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -648,6 +648,10 @@ impl WarmBookmarksCache {
648648
.create_subscription(ctx, Freshness::MaybeStale)
649649
.await
650650
.context("Error creating bookmarks subscription")?;
651+
tracing::debug!(
652+
repo = %repo_identity.name(),
653+
"WBC: bookmarks subscription created successfully"
654+
);
651655

652656
let bookmarks_to_watch = init_bookmarks(
653657
ctx,
@@ -678,6 +682,11 @@ impl WarmBookmarksCache {
678682
notify_sync_complete.clone(),
679683
);
680684

685+
tracing::debug!(
686+
repo = %repo_identity.name(),
687+
"WBC: Bookmarks Coordinator Spawned"
688+
);
689+
681690
Ok(Self {
682691
bookmarks: bookmarks_to_watch,
683692
terminate: Some(sender),

eden/mononoke/repo_attributes/commit_graph/preloaded_commit_graph_storage/src/lib.rs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -310,7 +310,7 @@ impl Loader<PreloadedEdges> for PreloadedEdgesLoader {
310310
mononoke::spawn_task({
311311
cloned!(self.ctx, self.blobstore_without_cache, self.blobstore_key);
312312
async move {
313-
info!("Started preloading commit graph");
313+
info!(blobstore_key = %blobstore_key, "Started preloading commit graph");
314314
let maybe_bytes = blobstore_without_cache.get(&ctx, &blobstore_key).await?;
315315
match maybe_bytes {
316316
Some(bytes) => {
@@ -319,6 +319,7 @@ impl Loader<PreloadedEdges> for PreloadedEdgesLoader {
319319
tokio::task::spawn_blocking(move || deserialize_preloaded_edges(bytes))
320320
.await??;
321321
info!(
322+
blobstore_key = %blobstore_key,
322323
"Finished preloading commit graph ({} changesets)",
323324
preloaded_edges.cs_id_to_edges.len()
324325
);

0 commit comments

Comments
 (0)