Skip to content

Commit 681274e

Browse files
committed
fix(batch): NASC parallel — pre-check GPS validity + track skips
1 parent be24434 commit 681274e

1 file changed

Lines changed: 19 additions & 8 deletions

File tree

scripts/batch_processing/run_nasc_parallel.py

Lines changed: 19 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -132,10 +132,18 @@ def _compute_one_nasc(args: tuple[str, str, str, str, int]) -> tuple[str, str, b
132132

133133
if not has_depth:
134134
ds.close()
135-
return (day_key, category, False, "No depth variable")
135+
return (day_key, category, False, "No depth variable — skipped")
136136
if not (has_lat and has_lon):
137137
ds.close()
138-
return (day_key, category, False, "No lat/lon variables")
138+
return (day_key, category, False, "No lat/lon variables — skipped")
139+
140+
# Check for all-NaN GPS (common in some pulse modes where GPS
141+
# wasn't merged during denoising)
142+
lat_var = ds["latitude"] if "latitude" in ds.data_vars else ds.coords["latitude"]
143+
n_valid = int(np.count_nonzero(~np.isnan(lat_var.values.ravel())))
144+
if n_valid == 0:
145+
ds.close()
146+
return (day_key, category, False, "All lat/lon are NaN — skipped")
139147

140148
wlog.info("Computing NASC (range_bin=%s, dist_bin=%s)...", NASC_RANGE_BIN, NASC_DIST_BIN)
141149

@@ -275,6 +283,7 @@ def main() -> None:
275283

276284
completed = 0
277285
failed = 0
286+
skipped = 0
278287
t_start = time.time()
279288

280289
with ProcessPoolExecutor(max_workers=args.workers, mp_context=ctx) as executor:
@@ -289,26 +298,28 @@ def main() -> None:
289298
rday, rcat, success, msg = future.result()
290299
if success:
291300
completed += 1
301+
elif "skipped" in msg.lower():
302+
skipped += 1
292303
else:
293304
failed += 1
294305
elapsed = time.time() - t_start
295-
rate = completed / elapsed if elapsed > 0 else 0
296-
remaining = len(tasks) - completed - failed
306+
rate = (completed + skipped + failed) / elapsed if elapsed > 0 else 0
307+
remaining = len(tasks) - completed - failed - skipped
297308
eta_s = remaining / rate if rate > 0 else 0
298309
eta_m = eta_s / 60
299310

300311
log.info(
301-
"[%d/%d done, %d failed] %s/%s: %s (ETA: %.0f min)",
302-
completed, len(tasks), failed, rday, rcat, msg, eta_m,
312+
"[%d done, %d skip, %d fail / %d] %s/%s: %s (ETA: %.0f min)",
313+
completed, skipped, failed, len(tasks), rday, rcat, msg, eta_m,
303314
)
304315
except Exception as e:
305316
failed += 1
306317
log.error("[%d/%d] %s/%s EXCEPTION: %s", completed, len(tasks), day_key, category, e)
307318

308319
total_time = time.time() - t_start
309320
log.info(
310-
"NASC parallel complete: %d/%d succeeded, %d failed in %.1f min",
311-
completed, len(tasks), failed, total_time / 60,
321+
"NASC parallel complete: %d succeeded, %d skipped, %d failed (of %d) in %.1f min",
322+
completed, skipped, failed, len(tasks), total_time / 60,
312323
)
313324

314325

0 commit comments

Comments
 (0)