From 75001c07fc8db5b091329aa770f443da8d04f026 Mon Sep 17 00:00:00 2001 From: Thomas Applencourt Date: Fri, 19 Jun 2026 10:55:39 -0500 Subject: [PATCH 1/9] Fix rank sometime not traced (LTTNG_UST_REGISTER_TIMEOUT) (#508) Co-authored-by: Thomas Applencourt --- xprof/xprof.rb.in | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/xprof/xprof.rb.in b/xprof/xprof.rb.in index 9b4804dd..8c2be45c 100755 --- a/xprof/xprof.rb.in +++ b/xprof/xprof.rb.in @@ -917,9 +917,14 @@ def all_env_tracers(usr_binary) h["LTTNG_UST_#{name.upcase}_PROFILE"] = 1 if OPTIONS[:profile] h["LTTNG_UST_#{name.upcase}_VERBOSE"] = 1 if LOGGER.level <= Logger::DEBUG end - # Only support blocking. + # Only support blocking, so lttng doesn't drop messages h['LTTNG_UST_ALLOW_BLOCKING'] = 1 + # Each tracee rank tries to register simultaneously. + # With the default (3s) timeout, ranks that lose the registration race give up + # and run untraced. Block until registration completes instead. + h['LTTNG_UST_REGISTER_TIMEOUT'] = -1 + # Customization if OPTIONS[:'backend-names'].include?('ze') h['LTTNG_UST_ZE_PARANOID_DRIFT'] = 1 if OPTIONS[:profile] From a7a7e616e6c49cae1212663c596fc668f7669318 Mon Sep 17 00:00:00 2001 From: Thomas Applencourt Date: Fri, 19 Jun 2026 14:40:58 -0500 Subject: [PATCH 2/9] xprof: default sync daemon to fs to work around MPI sync bug (#509) The MPI sync daemon currently hits an MPI bug. Make the FS daemon the default when THAPI_SYNC_DAEMON is unset, instead of preferring the MPI daemon whenever its binary is present. The MPI daemon stays available explicitly via THAPI_SYNC_DAEMON=mpi. Co-authored-by: Thomas Applencourt Co-authored-by: Claude Opus 4.8 (1M context) --- xprof/xprof.rb.in | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/xprof/xprof.rb.in b/xprof/xprof.rb.in index 8c2be45c..1738f3bc 100755 --- a/xprof/xprof.rb.in +++ b/xprof/xprof.rb.in @@ -437,20 +437,16 @@ class SyncDaemon end SYNC_DAEMON_MPI_PATH - when 'fs' + when 'fs', nil + # FS is the default (daemon_type nil): it avoids the current + # MPI sync-daemon bug. The MPI daemon is still available + # explicitly via THAPI_SYNC_DAEMON=mpi. SYNC_DAEMON_FS_PATH - when nil - if File.exist?(SYNC_DAEMON_MPI_PATH) - SYNC_DAEMON_MPI_PATH - else - LOGGER.warn("No #{SYNC_DAEMON_MPI_PATH} binary. Fallback to #{SYNC_DAEMON_FS_PATH}") - SYNC_DAEMON_FS_PATH - end else raise "Error: THAPI_SYNC_DAEMON=#{daemon_type} is not supported. Allowed: [mpi,fs]" end - spawn(daemon_path, log: "Initialize SyncDaemon #{daemon_type}") + spawn(daemon_path, log: "Initialize SyncDaemon #{daemon_type || 'fs (default)'}") end def local_barrier(name) From 4c6ff1ddcee57f5408c6705aeb3aca9e7d965052 Mon Sep 17 00:00:00 2001 From: Thomas Applencourt Date: Fri, 19 Jun 2026 14:59:39 -0500 Subject: [PATCH 3/9] Tracer ze universal (#505) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ze: add THAPI_REPORT_INJECTED_EVENTS counter When the env var is set, _get_profiling_event's successful ZE_EVENT_CREATE_PTR calls increment a global counter; _lib_cleanup prints the total to stderr ('THAPI: injected events: N') at process exit. Useful for the bats infra to lock in baseline per-test injection counts: any change to the v2 path (e.g. the upcoming lazy fallback for user-event reuse) that inflates injection beyond the baseline fails its regression check. Both the counter increment and the stderr print are gated by the env var read at init time so the steady-state hot path pays nothing. * btx_zeinterval: per-hEvent metadata FIFO with rotation A single ze_event_handle_t can be the signal event of multiple Appends on the same cmdlist — the tracer's lazy capture inserts a per-Append Query for each occurrence, producing one event_profiling_results tracepoint per use. Previously eventToBtxDesct[hEvent] held a single tuple, so the second event_profiling overwrote the first, and all subsequent results attributed to the last kernel paired with that handle (typical symptom: 4 Appends signaling one event tallied as 4 calls of the last kernel, 0 calls of the other three). Switch the map's value type to std::deque: - event_profiling pushes_back metadata for that hEvent. - event_profiling_result pops_front, attributes to that metadata, then push_backs the same entry. Rotation keeps the deque shape stable across cmdlist resubmits — N Appends produce N deque entries at build time; each Execute generates N results that cycle through those N entries in FIFO order. - zeCommandQueueExecuteCommandLists's per-cl walk updates every entry in the deque (not just the first). New regression test interval_profiling_shared_event covers the bug directly: 4 Barriers signaling the same hEvent, 4 results — expected output has 4 distinct device-side ts (post-fix); pre-fix would have collapsed all 4 to the last entry's ts. * btx_zeinterval: pop-and-discard FIFO + clear kernelToDesct on destroy The per-hEvent FIFO deque previously rotated (pop_front + push_back) on every event_profiling_result, intended to let a single set of build-time pushes serve M*N results across M cmdlist resubmits. But for the common case (1 push, 1 pop), rotation leaves stale entries in the deque, so the next push reads the wrong metadata. Symptom: 3 distinct kernel runs attributed as (busy_a=2, busy_b=1) instead of (busy_a=1, busy_b=1, busy_c=1). Switch to pop-and-discard: each Append pushes once, each result pops once. Resubmit-without-rebuild of the same cmdlist is no longer supported here; it is a deferred case on the tracer side as well. Also add zeKernelDestroy_entry to erase kernelToDesct[hKernel]. Drivers recycle freed kernel handle addresses; without the erase, the old entry silently survives until overwritten by the next Create. Defensive even though the rotation fix is what unblocks the busy-timing reproducer. * ze: universal per-Append AppendQueryKernelTimestamps scheme Every profiled Append (Launch*, Memcpy*, Barrier, SignalEvent, ...) is rewritten to: inj_ev = inject_event(cl) // fresh wrapper from per-context pool user_signal = hSignalEvent // saved (may be NULL) AppendQueryKernelTimestamps(cl, 1, &inj_ev, slab, &slot_off, hSignalEvent=user_signal, waitEvents=1 &inj_ev) The Query waits on inj_ev (so it runs after the user op completes) and signals the user's original event (so user wait semantics are preserved). Per-cl slot list records (inj_ev, attribution_event, slab_offset). On sync (queue / event / fence / cl-host) the slabs of every tracked cl are drained — each slot emits one event_profiling_results to the attribution event (user's, or inj_ev if user passed NULL). Identical for in-order / OOO and regular / immediate cmdlists: 1 injected event per Append, 1 extra API call per Append. No host-side KT read; no per-event-handle bookkeeping. Cross-cmdlist event reuse, mid-chain re-signal, and shared signal events all work because each Append has its own slab slot independent of which user event it attributes to. Resubmit of the same regular cmdlist without rebuild between Executes is deferred: the slab slots get overwritten by the second Execute. Reset followed by re-Append works (slot list resets on Reset). This commit: - adds the slab pool + slot list to _ze_command_list_obj_data - adds _universal_record_append, _cl_drain, _on_sync_drain_{cl,all} - swaps the model profiling_prologue/epilogue to the universal path - wires drain hooks to sync APIs (queue/event/fence/cl-host) - removes the legacy per-Append injected-event-attached-to-cl path: _on_destroy_event, _on_reset_event, _unregister_ze_event, _dump_and_reset_our_event, _profile_event_results, _event_cleanup, _on_execute_command_lists, cl_data->events, _ZE_EXECUTED, _ZE_IMMEDIATE_CMD, _ze_events hash, FIND/ADD_ZE_EVENT macros - drops zeEventDestroy/zeEventHostReset/zeCommandQueueExecuteCommandLists model hooks (no consumers left) The injected wrapper still goes through the per-context PUT_ZE_EVENT free pool so the underlying ze_event_handle is reused — only the slot-bookkeeping is freshly allocated per Append. 35/46 reproducer tests pass (was 24/46 with the prior v2 partial path). The 11 remaining failures are all in the deferred resubmit-without- rebuild class. * ze: dependency-tracking drain Replaces the previous "drain everything + status-gate" approach with explicit happens-before tracking. Sync points walk the dependency graph from the synced anchor and drain only the slots reachable. Per-slot bookkeeping gains: waits[] user wait events copied at Append, stable across Executes preds[] pointers to predecessor slots, computed at instantiate live per-run: true between instantiate and drain Per-cl bookkeeping gains: in_flight_q queue of the most recent un-drained Execute (NULL if none) mtx serializes Execute prologue (force-sync + instantiate) is_immediate, is_in_order Global: _ze_latest: hash mapping event_handle -> most recent slot whose attr matches (used to resolve happens-before edges) Algorithm: Append: record slot, copy user waits, for immediate cls instantiate inline Execute: if in_flight_q set, force-sync that queue + drain_cl, then instantiate every slot for the new run, stamp in_flight_q Sync(ev): drain latest[ev] (recurses on preds) Sync(q): drain every cl whose in_flight_q == q Sync(cl): drain that cl Drain reads the slab slot, emits event_profiling_results attributed to the user's signal (or to our injected event if user passed NULL), clears live + preds, removes from latest[] if still the head. Build- time fields (inj, attr, off, waits) stay so the next Execute re-instantiates without re-Appending. zeCommandListReset/Destroy and zeContextDestroy hooks dropped: the user must Sync before any of those, which means our slots are already drained. zeFenceHostSynchronize hook deferred. Removed status-gating, HostReset(inj) (driver does not un-signal KERNEL_TIMESTAMP events anyway — see tests/bugs/host_reset_kts_event), slot survival across drain (replaced by per-run live + dep walk). On the matrix: 28/40 pass, 12 fail. The 12 are all TALLY MISMATCH: tracer emits the right number of event_profiling_results (verified via the new bats helper that runs iprof -t and iprof -j separately), but the tally collapses duplicates with the same hEvent. * btx_zeinterval: per-event ring with cursor + new-phase reset A single hEvent can be the signal event of N Appends in one cl build phase, and the cl can be re-Executed M times, producing M*N result events. Previous attempts (single-entry overwrite, FIFO deque + rotate, FIFO + pop_front) each handled some cases and broke others. The model that fits all four scenarios in the matrix is a ring with a cursor: push: if cursor > 0 (we consumed results since the last push), the prior ring belongs to a finished build phase — clear and reset cursor. Then append the new metadata. result: read entries[cursor], advance cursor, wrap to 0 on overflow. The wrap handles resubmit (1 push, N results re-read the same ring). The phase-reset handles a rebuild after results consumed (new push implies the old entries are stale). New tally tests added in this directory: interval_profiling_resubmit_event 1 push, 2 results interval_profiling_shared_event_resubmit 2 pushes, 4 results interval_profiling_shared_event_xphase 2 pushes, 4 results, 1 push, 3 results Existing interval_profiling_shared_event (4 pushes, 4 results) still passes. End-to-end matrix on PVC: 41/41 pass. * btx_zeinterval: tests for resubmit + shared-event-resubmit + cross-phase Three new tally-side regression tests (raw tracepoint stream input, expected output diff'd at make-check time): interval_profiling_resubmit_event single Append re-fired by a closed cl on two Executes interval_profiling_shared_event_resubmit two Appends share one event, cl re-executed interval_profiling_shared_event_xphase shared event used in two distinct build phases (the cursor's new-phase reset path) All three exercise the per-event cursor ring and would have failed under the prior single-entry-overwrite or pop_front tallies. * ze: shadow compute cl for AppendQueryKernelTimestamps Profiled Appends on a copy-only user cl previously triggered a driver abort because AppendQueryKernelTimestamps is rejected on copy engines. Move the Query op to a per-(context, device) tracer-owned immediate ASYNC compute cl (lazy-created, first compute queue group). The user cl body now just signals an injected event; the shadow cl's Query waits on it and writes the timestamps to the slab. To preserve the user's signal contract, AppendBarrier on the user cl chains user_signal off inj. To drain correctly, each slot owns its own shadow_done event the Query signals, and drain host-syncs on it before reading the slab. Execute-side bookkeeping (force-sync-prior + Append-Query + instantiate + claim in_flight_q) all moves to the Execute *epilogue* under cl_data->mtx as a single critical section. Two reasons: * Append-Query on the shadow cl before the user cl is in flight deadlocks when both share the engine (only one compute group on this hardware) — the pending shadow Query holds the engine and the user cl can never dispatch. * Concurrent Executes on the same cl from two threads need the force-sync-prior to observe a sibling's claim-in_flight_q atomically (regression test: inorder_reg_Event_11). * ze: preallocate slot array to keep slot pointers stable Slots stored raw pointers to each other in preds[] and were referenced from the global latest[ev] -> slot map. The slot array was a realloc'd buffer starting at cap 8 and doubling; on any realloc that moved the buffer, every stored slot pointer became dangling, and the dep-graph walks at drain time silently skipped any slot allocated before the last realloc. In practice this lost timing records non-uniformly: a sync that walked preds back through the chain hit a dangling pointer and the recursion ended early (or hit garbage that happened to look not-live). Standalone reproducer in tests/bugs/missing_drain_dag/repro.c: 3 waves of 12 Appends each. Before this fix, drain emits 20/36 deterministically (slots 8..11 of each wave). After: 36/36. Fix is to allocate the slot array once at first use, sized to the slab cap (64). The slab itself is already hard-capped at 64, so allowing slot growth past it gained nothing. * ze: drain preds via slot->owner, not caller cl_data _slot_drain used to recurse with the caller's cl_data, but pred slots can live in another cl when the user signal chain crosses cls (cl_A signal=e1, cl_B wait=e1 signal=e2, sync(e2) only). Recursing with caller's cl_data read cl_B->slab + slot_A->off and emitted slot_A's tracepoint with cl_B's timestamps. Each slot now carries an owner back-pointer set at _cl_slot_append. _slot_drain drops its cl_data argument and reads from s->owner->slab, so cross-cl preds resolve to the right slab. _on_sync_drain_event collapses from the cl-walk to a direct s->owner->mtx lock. Reproducer: tests/matrix/inorder_reg_Event_15 (busy_b/busy_a>=3 ratio); fails before this commit, passes after. Full matrix 43/43. * ze: document why _slot_drain needs no cycle guard Code review flagged the unguarded recursion. In practice cycles are impossible: in-order preds index strictly downward, and latest[] preds always point to slots published earlier. Forming a cycle requires the user to declare two Appends each waiting on the other's signal event, which deadlocks L0 itself before we ever drain. The live-clear-before-recurse would also stop a cycle if one appeared, so we are belt-and-suspenders by accident. Comment only, no code change. * ze: drop wrong claim about live-clear stopping cycles The previous commit said live-clear-before-recurse would stop a cycle. live is cleared AFTER the recursion (see below), not before. The construction argument alone is the reason no guard is needed; remove the misleading sentence. * ze: fold slab/slots alloc into _cl_slot_append, bump n_slots last Two single-use helpers (_cl_slab_ensure, _cl_slots_grow) are inlined into their only caller. The cap-check and the two lazy allocations read better adjacent than dispatched through helpers. Side-effect fix: previous code bumped n_slots BEFORE attempting the slab alloc, so a failing first alloc left a permanent hole at index 0 (n_slots=1 but no slab). New order: cap-check, slots-alloc, slab-alloc, slot fill-in, THEN n_slots++. An OOM at any allocation returns NULL with cl_data unchanged. Rollbacks in _universal_record_append still apply: they undo a fully successful append after a downstream Barrier/shadow Append fails, which is correct under either ordering. Net -10 lines. Full matrix 43/43. * ze: inline _cl_get_device into its two callers The helper had two callers (immediate-Append shadow lookup, Execute epilogue shadow lookup) and just lazy-cached the device handle. Inlined both with the same lazy-cache idiom; the helper and forward-declared cached_device-read disappear. The two call sites pass different command_list values to the L0 call: the Append epilogue uses its local `command_list` parameter, the Execute epilogue uses `phCommandLists[i]`. The old helper read cl_data->ptr (set at create time, same handle), so behavior matches. Full matrix 43/43. * ze: collapse _universal_record_append cleanup with goto fail Four near-identical early-return blocks shared a cleanup pattern that varied only in how much state had been claimed (event(s), cl from hash, slot). Two labels capture the two boundaries: fail_locked: a slot was allocated AND/OR the cl hash entry was claimed AND mtx is held. Rollback slot if present, unlock, ADD_ZE_CL, fall through. fail: events still owned, no cl claim. PUT them back. Net -12 lines. Full matrix 43/43. * ze: collapse THAPI_DBGLOG / _NO_ARGS via , ##__VA_ARGS__ C99 forbids an empty __VA_ARGS__ which forced the no-args fork. GCC's `, ##__VA_ARGS__` extension swallows the dangling comma — already used elsewhere in THAPI (utils/tracepoint_gen.rb). Verified clean compile with -DTHAPI_DEBUG (which actually exercises the variadic body) in addition to the default no-op branch. Full matrix 43/43. * ze: drop THAPI_REPORT_INJECTED_EVENTS debug counter Added in d5603e4 as a regression guard for a planned bats assertion that never landed. No test, internal code, or external doc references it. Easy to reintroduce on demand. Full matrix 43/43. * ze: drop dead cap_slots field cap_slots was set once in _cl_slot_append to _ZE_SLAB_SLOTS_INITIAL and never read. The old _on_sync_drain_event walked the cls array using `s >= slots && s < slots + cap_slots` to find a slot's owner, but that walk went away when slots gained an owner back-pointer (cbb928b). Field has been dead since. Full matrix 43/43. * ze: drop empty _lib_cleanup and its scaffolding After THAPI_REPORT_INJECTED_EVENTS went away (322402d), _lib_cleanup had an empty body. Its registration plumbing (_do_cleanup, THAPI_ATTRIBUTE_DESTRUCTOR, the atexit call, the THAPI_USE_DESTRUCTORS ifdef pair) was all in service of running it. The CUDA backend still has a non-empty _lib_cleanup, so its copy of the same plumbing stays. Full matrix 43/43. * ze: extract _cl_cache_{device,context} + _on_execute_one_cl The lazy cached_device / cached_context resolve dance appeared 4 times (2 callsites x 2 fields) with subtle variations. Two tiny helpers collapse all four. Walks back commit 83191b9's inline-_cl_get_device with a better factoring that handles both fields together. The Execute epilogue's body was one 30-line for-loop with 5 distinct phases per cl. Extracting _on_execute_one_cl makes the loop a one-liner and lets the per-cl prose live with the code that implements it. Matches the granularity of _on_sync_drain_cl / _on_sync_drain_queue. Full matrix 43/43. * ze: chain user_signal off inj BEFORE the slot allocation Previously _universal_record_append wired user_signal -> inj via AppendBarrier AFTER reserving a slot. If slot reservation failed (e.g. _ZE_SLAB_SLOTS_INITIAL=64 cap hit by the 65th Append on an immediate cl), the function jumped to fail_locked and skipped the barrier — meaning user_signal was never signaled by anything on the cl, and the user's Sync(user_signal) hung forever. Reorder so the barrier Append runs first. We may still lose the timing tracepoint when the slot can't be reserved, but the user's event still fires and Sync no longer hangs. Reproducer: tests/matrix/ooo_imm_Event_06. Previously: timed out after 20s, 0 tracepoints. Now: 64 tracepoints out of expected 65 (cap still loses one slot; the actual cap removal is the next commit's job). * ze: refcount slots, recycle imm-cl events back to the per-context pool Slots gain a `refs` field: the count of downstream slots whose preds[] points here AND that have not yet been drained. _slot_instantiate atomically increments each pred's refs; _slot_drain decrements them at cleanup. When a slot reaches live==0 && refs==0 it is reclaimable. Reclaim is implemented for immediate cls only. Regular cls bake inj into the cl body — recycling inj would corrupt the next Execute round, so their events are reclaimed only at cl destroy (Phase-3). For immediate slots, _slot_release PUTs inj and shadow_done back to the per-context pool and frees waits. Slot storage itself stays for now; Phase-3 will free it along with chunked slot/slab storage that lifts the 64-Append cap for imm cls. Full matrix 43/43 (Event_06 still fails on cap until Phase-3). * ze: chunked slot/slab storage + refcount + cl-destroy hook Imm cls used to deadlock the user program at 65 Appends in one phase (_cl_slot_append returned NULL above _ZE_SLAB_SLOTS_INITIAL=64 and _universal_record_append's fail path stopped chaining user_signal off inj). The previous commit unblocked the chain; this commit lifts the cap altogether for imm cls and reclaims slot/event storage as ops complete. Storage: cl_data->slots/slab pair is replaced by a utlist DL chain of _ze_slab_chunk{}, each holding 64 slots and a 64-ts slab. Imm cls grow chunk-by-chunk on demand; regular cls stop at one chunk (their inj events are baked into the closed cl body, so adding a chunk post-Close would create slots the body doesn't address). Lifetime: slots gain a `refs` atomic counter, incremented at _slot_instantiate per pred edge, decremented at _slot_drain. A slot is reclaimable iff live==0 AND refs==0. _slot_release PUTs inj + shadow_done back to the per-context pool, frees waits, and decrements its chunk's n_held. A chunk frees itself when n_held reaches 0 AND it isn't the active tail. Regular-cl slots are not released at drain — their events come back at cl destroy. Destroy: zeCommandListDestroy hook walks chunks, PUTs events back, frees chunks + cl_data. Re-added because the chunked layout makes the leak-on-destroy bound by per-cl lifetime instead of program lifetime. Memory shape: O(in-flight slots) at any time, matching the typical "one long-lived imm cl with many syncs" use case. Full matrix 45/45. * ze: tear down our context-bound state in zeContextDestroy prologue The tracer was leaving its own L0 objects (event pools/events used as the recycle pool, shadow command lists keyed by (ctx, device)) alive inside the user's context. When the user app destroyed the context before destroying a command list that had our injected events baked into it (the Intel libomptarget L0 plugin does this at _dl_fini), the following zeCommandListDestroy segfaulted inside libze_intel_gpu.so. Add a zeContextDestroy prologue that, while the context is still valid, walks _ze_cls / _ze_shadow_cls / _ze_event_pools and frees everything we own keyed to the dying context. * rubocopt * Clang-format * ze: fix UAF in _cl_drain when draining the chunk's last slot _slot_drain on the last slot of a chunk calls _slot_release, which decrements n_held to 0 and frees the chunk. The next iteration of the inner for loop then reads c->n_used and accesses &c->slots[i] on freed memory. Pin the chunk with an extra n_held bump during the inner loop. Drop the ref after the loop and free the chunk in _cl_drain itself if it was the last holder. The chunk-free condition (n_held == 0 && c != tail) is unchanged. Found by ASAN running ooo_imm_Event_07 (cross-cl chunk-mutation stress). Co-Authored-By: Claude Opus 4.7 * ze: Reset shadow cl on idle to reclaim L0 driver per-QKT storage The L0 driver records each zeCommandListAppendQueryKernelTimestamps in cl-internal storage (~10 KB per call) and only releases it on zeCommandListReset/Destroy — never lazily, even after the device has completed the op. The shadow cl is per-(context, device) and lives for the context's lifetime, so an app that profiles many Appends grows unbounded driver memory. Track live_queries (under sh->mtx): bumped in _shadow_append_query, decremented in _slot_drain after the shadow_done host-sync. When the counter hits 0 inside _slot_drain, the shadow cl is idle from our perspective and we Reset it to reclaim the driver state. Cross-cl case is handled: with cls A and B each holding a Query in flight, A.sync() decrements 2→1 (no Reset), B.sync() decrements 1→0 (Reset). The slot stores its shadow cl so drain knows where to decrement; same sh* is set in both Append paths (immediate cl and Execute prologue for regular cls). Drive-by: split two existing `if (--x == 0 && ...)` sites in _slot_release and _cl_drain into two statements for consistent style. Validated with mem_const_steady (added in tests repo): heap delta drops from 1.93 MB/iter to 32 B/iter over 100 iters of (create cl + 200 Appends + sync + destroy cl). ASAN sweep across 46 correctness tests: clean (no UAF/OOB/leaks attributable to the tracer). Full correctness matrix: 46/46 pass. Co-Authored-By: Claude Opus 4.7 * ze: inline QKT on compute user cls to skip the shadow cl When the user's command list is on a COMPUTE queue group, the AppendQueryKernelTimestamps now lives in the user cl body itself, signaling user_signal directly. This collapses two Appends per profiled op (Barrier + shadow QKT) into one, removes the per-Append shadow_done fence event allocation and its drain-time host-sync, and drops the per-Execute shadow re-Append loop for regular compute cls. Detection runs at zeCommandListCreate{,Immediate} via the desc's commandQueueGroupOrdinal against a generalized per-device queue-group flag cache. cl_data->is_compute=0 (the shadow path) is the safe fallback for copy-only cls and any case where the group flags can't be determined. Also: add _ZE_MUST() and apply it to every tracer-issued operational L0 call (Barrier, QKT Append, EventHostSync/Reset, CommandListReset, QueueSync, pool event reset). These are calls the tracer adds on the user's behalf where failure means either a user hang (Barrier chain) or a non-self-consistent trace. Defensive: print + abort so the bug surfaces under sanitizers/CI rather than ship bad data. Driver query calls (Get*Handle, GetCommandQueueGroupProperties) keep their graceful fallbacks — they can fail transiently during teardown. Co-Authored-By: Claude Opus 4.7 * ze: document QKT placement (INLINE vs SHADOW), sweep stale comments Top-of-file algorithm sketch now names the two QKT placements and shows their ASCII shape. The previous claim that "we use the shadow cl uniformly for all engines so the code path is identical" is gone — that was true before the inline path and misleading afterwards. - Algorithm section: split "place a Query" off; spell out per-path behavior at Execute (shadow re-Appends; inline already in cl body) and at drain (shadow host-syncs shadow_done; inline does not). - New "QKT placement" section with the two diagrams, anchored from cl_data->is_compute. - struct _ze_slot: sh and shadow_done documented as shadow-path only. - struct _ze_shadow_cl preamble: re-cast as the copy-only path. - _universal_record_append: redundant function-level diagram replaced with a pointer to the top-of-file section. - Slab-chunk regular-cl note: mention the QKT is also baked into the closed cl body on the inline path (additional reason chunks can't grow after Close). Co-Authored-By: Claude Opus 4.7 * ze: include ; rename inorder_reg_Event_11 -> inorder_reg_Event_multithreaded_01 Co-Authored-By: Claude Opus 4.7 * ze: skip Execute-time _slot_instantiate when slot is already live The inline QKT path (compute user cls) instantiates each slot at Append time so the QKT can be baked into the cl body. The Execute hook then re-instantiated the same slot unconditionally, re-running the in-order pred walk and picking up later-appended live siblings as predecessors. Two siblings on an in-order cl ended up mutually referencing each other in preds[], and _slot_drain's recursive walk infinite-looped, crashing the user with SIGSEGV from stack overflow. Gate the Execute-time _slot_instantiate on !slot->live. Inline slots arrive live=1 from Append and are skipped; shadow-path slots always arrive live=0 (regular cls aren't instantiated at Append at all; second+ Execute rounds see slots reset to live=0 by the preceding _cl_drain), so the guard is a no-op for them. Tests added to thapi_ze_test that fail before / pass after: inorder_reg_Event_02 (2 Appends share one user signal event) inorder_reg_Event_05 (Event_02 + HostReset between 2 Executes) inorder_reg_Event_10 (Event_02 + cl resubmit N=2) inorder_reg_Event_11 (3 cls / 3 queues, 2 Appends each) Co-Authored-By: Claude Opus 4.7 * ze: assert in _slot_instantiate, skip live slots earlier in Execute Defensive cleanup on top of the previous double-instantiate fix. Three related changes, one commit: 1. Introduce _THAPI_LOG (always-on; THAPI(func:line) prefix; flushes stderr so the line lands before abort) and _THAPI_ASSERT (logs + aborts unconditionally — not gated on NDEBUG, since silently dropping the check would let the bug ship bad data). _ZE_MUST is rewritten in terms of _THAPI_ASSERT, and the existing THAPI_DBGLOG now routes through _THAPI_LOG when THAPI_DEBUG is set. 2. _slot_instantiate asserts !s->live at entry. Re-instantiating a live slot leaks its preds[] and lets the in-order pred walk pick up later-appended siblings as predecessors, which infinite-loops _slot_drain. The prior fix added a guard at the one known caller; the assert turns the rule into an invariant of the function so the next caller can't trip the same trap silently. 3. Drop the !slot->live guard around _slot_instantiate in _on_execute_one_cl in favor of a same-condition `continue` at the top of the per-slot body. Already-live slots have nothing left to do this Execute (dep-graph entry still valid; inline-path QKT is baked into the cl body and re-fires automatically), so they shouldn't be running the shadow_append_query branch either. Cleaner and removes the if/assert contradiction at the call site. All 51 correctness + 1 bench tests pass. Co-Authored-By: Claude Opus 4.7 * ze: collapse is_compute branches via _slot_publish; drop cached_device + dead macros _slot_publish(cl_data, s, sh) routes on s->shadow_done — the slot data itself is now the source of truth for "shadow vs inline". The Execute-time is_compute branch and the per-slot is_compute branch in _on_execute_one_cl collapse to one data-driven publish. _universal_record_append's shadow- immediate tail collapses to the same call. The inline-vs-shadow fork at Append remains: chaining user_signal via the QKT itself avoids an extra zeCommandListAppendBarrier per kernel. cached_device + _cl_cache_device go away — the only call sites are on the shadow path (immediate-copy Append, regular-copy Execute), both cold; the extra ZE_COMMAND_LIST_GET_DEVICE_HANDLE_PTR call is acceptable for one fewer field and one fewer cache to reason about. cached_context stays: load-bearing for _on_destroy_context's per-cl sweep. Delete _ZE_ERROR_MSG / _ZE_ERROR_MSG_NOTERMINATE / _ERROR_MSG: zero callers, and _ERROR_MSG was syntactically broken (orphan do/while). All 52 correctness + bench tests pass. * ze: rename _ze_latest -> _ze_event_latest_signaled The map is "event -> the most recent slot that signaled it". The old "latest" name didn't say what it was the latest OF. The new name does. Mechanical rename across the global, struct, mutex, three helpers, and the algorithm/struct/comment shorthand. No semantic change. All 53 tests pass. * ze: drop FIND_AND_DEL+ADD on _ze_cls_mutex in _universal_record_append cl_data was being yanked out of the global hash at the top of every profiled Append and reinserted at every exit, taking _ze_cls_mutex three times per Append (DEL + ADD on each return path + the cleanup ADD on fail_with_cl). Replace with a single FIND_ZE_CL: cl_data stays in the hash, the global mutex is touched once instead of three times. Safe because L0 forbids racing Append against Destroy on the same cl handle (both carry the not-thread-safe-per-cl-handle restriction), so cl_data cannot be torn out by a concurrent _on_destroy_command_list. cl_data->mtx still serializes us against another Append/Execute on this same cl. This was the single biggest contention point in the N-threads-N-CLs workload (inorder_imm_Event_multithreaded_01 and friends). * ze: fetch context once per profiled Append; thread it through prologue->epilogue Previously each profiled Append issued zeCommandListGetContextHandle three times: once inside the generated prologue's _get_profiling_event, once at the top of _universal_record_append, and once at the first Execute via _cl_cache_context. Same handle every time. Fetch it once in the generated prologue, pass it to _get_profiling_event, also pass it to _universal_record_append. _universal_record_append now publishes cached_context up front so the first _on_execute_one_cl hits the cache too. Net: one driver call per Append instead of three. The cached_context write is unlocked. Safe: every writer stores the same value (the cl's true context), and the only reader (_on_destroy_context's per-cl sweep) is gated on a user contract forbidding concurrent Append + DestroyContext on the same context. All 51 correctness tests pass. * ze: inline cached_context read in _on_execute_one_cl; drop _cl_cache_context Since cached_context is now published at the top of _universal_record_append (before any slot exists), the L0-fetch fallback inside _cl_cache_context is provably dead — _on_execute_one_cl's only caller path is gated on slot->shadow_done, which can only be set by a prior shadow-path Append that already populated cached_context. Replace the call with a direct field read and delete the helper. One fewer indirection, one fewer "what if the cache is empty here?" branch to think about. All 51 correctness tests pass. * ze: PUT_ZE_EVENT macro -> _put_ze_event function; reset + alloc outside pools mutex Two changes, both targeting work that was happening with global mutexes held: 1. PUT_ZE_EVENT was a 20+ line macro spanning hash lookup, allocation, error-path L0 destroys, HostReset, and prepend. Convert to a function (debugger can step in, callers no longer pay the macro-expansion readability tax). Rename all 7 call sites (helpers + ze_model.rb generator); no shim macro left behind. 2. Move two pieces of work out from under the locks they shouldn't be under: - _put_ze_event: HostReset is thread-safe — issue it before taking _ze_event_pools_mutex instead of serializing an L0 round-trip behind a global lock. Pre-allocate the bucket entry outside the lock too, freeing the unused copy if we lose the publish race (same pattern _qgroup_cache_get already uses). - _event_latest_signaled_set: calloc outside the lock for the same reason; same lose-the-race-free-our-copy handling. All 51 correctness tests pass. * ze: convert remaining macros to functions FIND_ZE_CL / ADD_ZE_CL / FIND_AND_DEL_ZE_CL -> _cl_find / _cl_add / _cl_find_and_del GET_ZE_EVENT -> _get_ze_event GET_ZE_EVENT_WRAPPER / PUT_ZE_EVENT_WRAPPER -> _get_ze_event_wrapper / _put_ze_event_wrapper All six were thin lock-wrapped hash/list ops in macro form, with the output threaded through an `out` parameter. Convert to ordinary functions returning the value; debugger steps in, callers don't pay the macro-expansion readability tax, and the lock-held region is easier to audit when it's a function body instead of a macro you have to mentally expand at each call site. No shims left behind — every call site updated. While here: _get_ze_event_wrapper now allocates the fresh wrapper outside the wrappers mutex (matches the [pre-alloc | lock | publish | free-our-copy-on-race] pattern already established by _put_ze_event and _event_latest_signaled_set), so the freelist mutex never wraps a heap call. All 51 correctness tests pass. * ze: extract _cl_chunk_free helper The "DL_DELETE the chunk + zeMemFree the slab (sometimes) + free the chunk struct" body was open-coded in 5 places: _universal_record_append's fail_locked rollback, _slot_release, _cl_drain, _on_destroy_command_list, and _on_destroy_context step 1. They differed only in whether to issue zeMemFree on the slab (skipped in _on_destroy_context because the ctx is dying and the driver reclaims). Extract _cl_chunk_free(cl_data, c, free_slab). Cleaner site-by-site, and the `free_slab=0` call site documents the "ctx-dying skip-zeMemFree" invariant in one place (the helper's comment) instead of relying on readers to spot the missing zeMemFree at a single open-coded site. No behavior change. All 51 correctness tests pass. * ze: unify cl_data destroy paths via _cl_data_destroy(ctx_dying) _on_destroy_command_list and _on_destroy_context step 1 were ~20 lines of near-identical code each: walk all chunks, free per-slot events/waits/ preds, free the chunk, then free cl_data itself. They differed only in two ctx-scoped resource decisions that move in lockstep: - event wrappers: recycle to the per-ctx pool (ctx alive) vs recycle wrapper struct only (ctx dying, pool is about to die too) - chunk slab: zeMemFree (ctx alive) vs skip (ctx dying) Extract _cl_data_destroy(cl_data, int ctx_dying) that captures both choices. _on_destroy_command_list passes 0; _on_destroy_context step 1 passes 1. The "why dying differs from alive" reasoning now lives in one place (the helper's comment) instead of being split across two sites where future readers had to diff them to spot the invariant. Caller still owns hash removal (single-cl: _cl_find_and_del; per-ctx: HASH_DEL inside the iter) since those are genuinely different. Lock order unchanged. All 51 correctness tests pass. * ze: collapse every per-domain tracer mutex into one _ze_state_mutex The dep-graph caches cross-cl slot pointers in s->preds[], so any drain may mutate ANY cl's chunks. A per-cl mtx scheme requires cross-cl lock acquisition with ordering rules. One mutex covering all tracer state makes that go away — drain freely follows pred pointers; Append on different cls serializes through the same lock. Once we had a single mutex for cl_data state, every other per-domain lock (event freelists, event-pool registry, qgroup cache, shadow-cl registry, per-shadow-cl L0 Append serializer, latest-signaled map) was guarding its own little island while every caller already held _ze_state_mutex. Folded them all in. Removed: cl_data->mtx (per-cl) _ze_cls_mutex (cl registry) _ze_event_wrappers_mutex _ze_event_pools_mutex _ze_event_latest_signaled_mutex _ze_qgroup_cache_mutex _ze_shadow_cls_mutex sh->mtx (per shadow cl) Kept: _ze_state_mutex (the one) ze_closures_mutex (separate domain — ffi closures) Fixes two TSan-confirmed bugs in the process: R3: _on_destroy_context read cl_data->cached_context under _ze_cls_mutex while _universal_record_append wrote it without any lock. Different mutexes, observable race. Bug 2: _slot_drain recursed through s->preds[i] into another cl's slots and mutated chunk->n_held / slot->live while _cl_slot_append on that cl mutated the same bytes. Reproduced by ooo_imm_Event_multithreaded_01 / _04. All the _cl_*, _get_*, _put_*, _event_latest_signaled_*, _qgroup_cache_get, _get_shadow_cl, _shadow_append_query, _get_profiling_event helpers run under the assumption that the caller holds _ze_state_mutex. Entry points that used to call these without the lock (_on_create_command_list calling _ordinal_is_compute, _on_destroy_context's three steps, the generated profiling_prologue calling _get_profiling_event) take it now. Other simplifications that fall out: - _Atomic on cached_context reverts to plain ze_context_handle_t. - __atomic_* on slot->refs reverts to plain ++/--. - The "allocate outside lock, race-publish inside" pattern in _event_latest_signaled_set / _put_ze_event / _get_ze_event_wrapper / _get_shadow_cl all collapse to straight HASH/DL ops. - sh->live_queries no longer needs its own mtx — manipulated under the state mutex like everything else. Perf trade: Append on different cls and the various freelist accesses serialize through one mutex. The L0 calls inside the critical section (zeCommandListAppendBarrier or AppendQueryKernelTimestamps) are short — the GPU just queues, doesn't execute. Worth it for the dramatic simplification: one mutex, zero lock-ordering rules, zero atomics, zero cross-cl acquisitions. Net -81 lines on the helpers file. 10/10 multithreaded tests under TSan report 0 races. 51/51 correctness tests pass; the imm_Event_multithreaded_01 case that exposed an unlocked-prologue regression mid-development passes 5/5 on loop. * ze: comment cleanup — add Concurrency header section, trim verbose docs Add a "Concurrency" section to the algorithm header that explains why a single global mutex covers all tracer state (cross-cl pred edges in the dep graph would force any per-cl scheme into multi-mutex acquisition with ordering rules; one global mutex sidesteps that entirely; the perf trade is bounded because the held region is just short L0 queue ops). With that in place, scrub per-function "Caller holds _ze_state_mutex" boilerplate that the section comment now covers, fix the one stale "lock cl.mtx" line in the algorithm pseudocode, and tighten the longer docstrings on _cl_data_destroy / _on_destroy_context / _on_execute_one_cl / _universal_record_append / shadow-cl helpers. Pure comments diff: -44 lines, no code change. 52/52 correctness tests still pass. * ze: attribute device-profiling results from AppendSignalEvent correctly zeCommandListAppendSignalEvent's payload field is `hEvent`, not `hSignalEvent`, so it never qualifies for the hSignalEvent_* matching sets in btx_zematching_model.yaml and hSignalEvent_rest_entry_callback never fires for it. threadToLastLaunchInfo therefore retains whatever prior Append last populated it (typically MemoryFill / Kernel / Memcpy on the same thread). The next event_profiling — which IS emitted for AppendSignalEvent — pushes a ring entry tagged with that stale commandName, and the downstream device tally counts the signal's profiling result as one more MemoryFill/Kernel/etc. Symptom in the wild: a trace with 12000 host AppendMemoryFill calls reports 16002 device MemoryFill(D) — the extra 4002 are AppendSignalEvent results mis-attributed to the prior fill on each of two reused events. Add a dedicated zeCommandListAppendSignalEvent_entry_callback that refreshes threadToLastLaunchInfo with the real command name and a new btx_event_t::SIGNAL tag, then short-circuit event_profiling_result_callback on SIGNAL so no device-tally record is emitted (AppendSignalEvent is a host-side signal and does no GPU work to time). The ring entry itself is still pushed and the cursor still advances, so subsequent profiling results for the same event handle land on the correct slot. Verified against the full bats correctness suite (53/53) and a new reproducer in thapi_ze_test (inorder_imm_Event_08). * ze tests: document ring/cursor scenarios; drop unreachable _fast test The four shared_event / resubmit BTX tests are synthetic inputs fed straight into the interval filter — they don't include Create/Execute because the test_wrapper converter can't synthesize struct values, and that omission makes the trace read as physically impossible (4 results from 2 Appends with no Execute in between). Add a short header to each explaining what ring/cursor branch it exercises. interval_profiling_fast assumed event_profiling_results could arrive inside the Append_exit window. With the new tracer, results are only emitted at sync time, so that interleaving is no longer reachable — delete the test and drop it from TRACE_COMMON. To allow header comments in the .thapi_text_pretty files, teach thapi_log_to_bt_source_component.rb to skip blank/# lines. Co-Authored-By: Claude Opus 4.7 * ze tests: rewrite ring fixture headers to describe observable behavior The four shared_event / resubmit BTX fixture headers leaked consumer internals (ring, cursor, entries, a clear+reset branch name, and a hard-coded source line number that was already off-by-one). Rewrite each to describe only what is observable: shared events, resubmissions, and results attributed to their Appends in submission order. Also fix the xphase header's result count (phase 2 is 3 results, not 2). Co-Authored-By: Claude Opus 4.8 (1M context) * ze: drain profiling slots on fence-only sync A program that Executed a regular cl with a fence and waited only via zeFenceHostSynchronize lost all its profiling results: drain ran from the queue / event / cl-host sync hooks but not from fence sync, so that anchor never triggered a drain and the slots were freed un-emitted at teardown. Stamp the Execute's fence onto each cl (in_flight_fence, alongside the existing in_flight_q) and add an _on_sync_drain_fence hook on zeFenceHostSynchronize that drains every cl whose fence matches. The fence signals when all cls in its Execute complete, so it is a valid drain anchor for exactly those cls. zeFenceQueryStatus is intentionally not hooked (a non-blocking poll could race a still-building reuse). Cleared together with in_flight_q wherever a cl is drained. Measured on PVC (8 fills): before, fence-sync gave results=0; after, results=8, matching the queue-sync control. Co-Authored-By: Claude Opus 4.8 (1M context) * ze: warn once when a regular cl exceeds its profiled-Append cap Regular cls store profiling slots in a single 64-slot chunk (inj events are baked into the closed cl body, so storage can't move). Appends past the cap were dropped silently: the kernel ran and the issued tracepoint fired, but no result tracepoint ever followed. Warn once (guarded, under the state mutex) so the data loss is visible instead of silent. A full fix needs multi-chunk regular-cl storage and is left as follow-up. Confirmed on PVC: 100 Appends in one build -> 64 results; warning now fires. Co-Authored-By: Claude Opus 4.8 (1M context) * ze: reclaim slot state on cl reset/reuse; fix cross-cl UAF; cap driver QKT leak Three related fixes for command-list reuse: 1. zeCommandListReset hook (_on_reset_command_list). Regular cls never reclaim slots at drain (_slot_release is a no-op for them — their inj is baked into the cl body for cross-Execute reuse). Without a Reset hook the stale slots were re-published on the next Execute: a regular cl reused across Reset over-counted massively (80 rounds -> 3104 results). The hook drains defensively then reclaims, keeping cl_data registered and empty. 2. Cross-cl use-after-free in teardown (reset AND pre-existing destroy). A drained slot may still be a pred of a LIVE slot in another cl (refs>0); freeing its chunk dangled that preds[] pointer. Deterministically reproduced (quarantine probe, 3/3) on reset and on zeCommandListDestroy. Fix: _cl_chunk_reclaim frees a chunk only when no slot has refs>0; else it DETACHES the chunk (unlink from cl, null each slot's owner, free slab, set n_pinned) and the downstream drain frees the bare struct when the last ref drops (detached branch in _slot_release). ctx-dying destroy still frees wholesale (no slot outlives the ctx). 3. Driver per-QKT storage growth on a long-lived reused IMMEDIATE cl (~795 KB/round). The driver only reclaims QKT storage at Reset/Destroy of the cl. _imm_reset_if_drained raw-resets a fully-drained immediate cl on sync (untraced; safe — all work complete) and reclaims our bookkeeping. Verified on PVC: reg_reset_reappend 80/80; cross-cl UAF probes clean post-fix; mem_persistent_cl 159 MB -> 96 bytes (flat); correctness suite 57/57. Co-Authored-By: Claude Opus 4.8 (1M context) * ze: serve kernel timestamps back to the user's own zeEventQueryKernelTimestamp The Append prologue swaps the user's signal event for our injected event, so the user's event ends up carrying the QKT/barrier op timing instead of the kernel's. A tracer-unaware program that signals a KERNEL_TIMESTAMP event from a kernel and then calls zeEventQueryKernelTimestamp on it read the wrong (op-scale) duration — measured ratio 0.77x vs the true 2x. Stash the kernel result (already read from the slab at drain) keyed by the user's event, and add a zeEventQueryKernelTimestamp epilogue that overwrites *dstptr with it. Re-signaling overwrites the entry with the latest result. Verified on PVC: user_kts_query_ratio 0.77 -> 2.00; correctness suite 58/58 (incl. tests where the user appends their own AppendQueryKernelTimestamps). Co-Authored-By: Claude Opus 4.8 (1M context) * ze: consolidate tracer engine — unified sync, shared slot helpers Behavior-preserving cleanup of the ZE profiling engine. No change to the QKT placement paths, the single state mutex, the cross-cl pred-ref/detach UAF fix, or the imm/shadow reset logic; verified 63/63 on the correctness + bench suite. - Collapse the four sync entry points (queue/fence/event/cl) into one _on_sync(enum _ze_sync_kind, void *h); ze_model.rb hooks pass the kind. The QUEUE/FENCE in-flight match and the EVENT/CL direct lookups are now one switch instead of four near-parallel lock/find/drain/unlock funcs. - Add _ZE_FOREACH_SLOT to replace the open-coded "walk every used slot" loop (8 sites), and _cl_any_live to fold the two identical in-flight scans into one predicate. - Replace the three divergent per-slot teardown copies with one _slot_dispose_resources(s, mode) primitive (POOL vs WRAPPER disposal). - Stamp the two event-keyed maps (latest_signaled, kts) from one _ZE_EVENT_MAP_DEFINE[_NOCLEAR] macro instead of hand-written accessors. - Linearize _universal_record_append: extract _append_inline_query, _chain_user_signal, _slot_append_rollback; preserve the is_immediate gate around _slot_publish and the out-of-lock failure-path barrier. Co-Authored-By: Claude Opus 4.8 (1M context) * ze: fix event-state stale reads on handle reuse + index queue/fence sync Two correctness/perf fixes plus a refactor in the ZE tracer engine: - Per-event state (latest signaling slot + stashed kernel timestamp) was keyed by the user's event handle address but never evicted on destroy. The L0 driver recycles freed event addresses, so a fresh event could be served the prior event's stale kernel timestamp (SUCCESS instead of NOT_READY) and a wait on the reused address could resolve to a freed slot (UAF in the pred walk). Evict on a successful zeEventDestroy. - Queue/fence sync (_on_sync) scanned every live command list to find the ones stamped with the synced queue/fence — O(live cls) per sync (measured 164x/176x cost growth at 4096 live cls; plain L0 is flat). Add per-queue and per-fence in-flight indexes so a sync drains only the matching cls. - Merge the two event-keyed maps (latest_signaled + kts) into one _ze_event_state entry: same key, same lifecycle, one alloc/lookup/eviction, and they can no longer desync (the class of bug the first fix addressed). Co-Authored-By: Claude Opus 4.8 (1M context) * ze: clang-format-18 the tracer helpers; declare ForEach macros Run clang-format-18 over tracer_ze_helpers.include.c to satisfy CI. Also add the tracer's loop macros (_ZE_FOREACH_SLOT, DL_FOREACH*, HASH_ITER) to .clang-format's ForEachMacros so their bodies stay indented instead of being flattened as if they weren't loops. Formatting only — no behavior change. Co-Authored-By: Claude Opus 4.8 (1M context) --------- Co-authored-by: Claude Opus 4.7 Co-authored-by: Thomas Applencourt Co-authored-by: Thomas Applencourt --- .clang-format | 6 + backends/ze/Makefile.am | 7 +- backends/ze/btx_zeinterval_callbacks.cpp | 63 +- backends/ze/btx_zeinterval_callbacks.hpp | 17 +- backends/ze/gen_ze.rb | 1 + ...l_profiling_resubmit_event.bt_text_pretty} | 3 +- ...rofiling_resubmit_event.thapi_text_pretty} | 10 +- ...rval_profiling_shared_event.bt_text_pretty | 8 + ...l_profiling_shared_event.thapi_text_pretty | 18 + ...iling_shared_event_resubmit.bt_text_pretty | 6 + ...ng_shared_event_resubmit.thapi_text_pretty | 13 + ...ofiling_shared_event_xphase.bt_text_pretty | 10 + ...ling_shared_event_xphase.thapi_text_pretty | 21 + backends/ze/tracer_ze_helpers.include.c | 1775 +++++++++++++---- backends/ze/ze_model.rb | 169 +- utils/thapi_log_to_bt_source_component.rb | 5 +- 16 files changed, 1687 insertions(+), 445 deletions(-) rename backends/ze/tests/{interval_profiling_fast.bt_text_pretty => interval_profiling_resubmit_event.bt_text_pretty} (57%) rename backends/ze/tests/{interval_profiling_fast.thapi_text_pretty => interval_profiling_resubmit_event.thapi_text_pretty} (50%) create mode 100644 backends/ze/tests/interval_profiling_shared_event.bt_text_pretty create mode 100644 backends/ze/tests/interval_profiling_shared_event.thapi_text_pretty create mode 100644 backends/ze/tests/interval_profiling_shared_event_resubmit.bt_text_pretty create mode 100644 backends/ze/tests/interval_profiling_shared_event_resubmit.thapi_text_pretty create mode 100644 backends/ze/tests/interval_profiling_shared_event_xphase.bt_text_pretty create mode 100644 backends/ze/tests/interval_profiling_shared_event_xphase.thapi_text_pretty diff --git a/.clang-format b/.clang-format index 99557dc7..cb9c4a12 100644 --- a/.clang-format +++ b/.clang-format @@ -1,2 +1,8 @@ BinPackParameters: false ColumnLimit: 100 +ForEachMacros: + - _ZE_FOREACH_SLOT + - DL_FOREACH + - DL_FOREACH_SAFE + - DL_FOREACH_SAFE2 + - HASH_ITER diff --git a/backends/ze/Makefile.am b/backends/ze/Makefile.am index 2445e1d6..942c0947 100644 --- a/backends/ze/Makefile.am +++ b/backends/ze/Makefile.am @@ -278,9 +278,12 @@ TRACE_COMMON = \ tests/interval_profiling_normal.thapi_text_pretty \ tests/interval_profiling_multithread.thapi_text_pretty \ tests/interval_profiling_API_call.thapi_text_pretty \ - tests/interval_profiling_fast.thapi_text_pretty \ tests/interval_profiling_interleave_process.thapi_text_pretty \ - tests/interval_profiling_ignore.thapi_text_pretty + tests/interval_profiling_ignore.thapi_text_pretty \ + tests/interval_profiling_shared_event.thapi_text_pretty \ + tests/interval_profiling_resubmit_event.thapi_text_pretty \ + tests/interval_profiling_shared_event_resubmit.thapi_text_pretty \ + tests/interval_profiling_shared_event_xphase.thapi_text_pretty BTX_ZE_GENERATED_SOURCE_TEST = \ btx_source_ze_test/metababel/metababel.h \ diff --git a/backends/ze/btx_zeinterval_callbacks.cpp b/backends/ze/btx_zeinterval_callbacks.cpp index c6355fdb..2a1980e8 100644 --- a/backends/ze/btx_zeinterval_callbacks.cpp +++ b/backends/ze/btx_zeinterval_callbacks.cpp @@ -559,6 +559,20 @@ static void hSignalEvent_rest_entry_callback(void *btx_handle, hCommandList, name, ts, btx_event_t::OTHER, {}}; } +static void zeCommandListAppendSignalEvent_entry_callback(void *btx_handle, + void *usr_data, + int64_t ts, + const char *hostname, + int64_t vpid, + uint64_t vtid, + ze_command_list_handle_t hCommandList, + ze_event_handle_t hEvent) { + (void)hEvent; + auto *data = static_cast(usr_data); + data->threadToLastLaunchInfo[{hostname, vpid, vtid}] = { + hCommandList, "zeCommandListAppendSignalEvent", ts, btx_event_t::SIGNAL, {}}; +} + /* * _ _ _ * _ _ / _ ._ _ ._ _ _. ._ _| / \ _ _ |_ _ _ _|_ _ @@ -584,9 +598,11 @@ zeCommandQueueExecuteCommandLists_entry_callback(void *btx_handle, const auto commandQueueDesc = data->commandQueueToDesc[{hostname, vpid, hCommandQueue}]; for (size_t i = 0; i < _phCommandLists_vals_length; i++) { for (auto &hEvent : data->commandListToEvents[{hostname, vpid, phCommandLists_vals[i]}]) { - auto &h = data->eventToBtxDesct[{hostname, vpid, hEvent}]; - std::get(h) = commandQueueDesc; - std::get(h) = ts; + auto &ring = data->eventToBtxDesct[{hostname, vpid, hEvent}]; + for (auto &h : ring.entries) { + std::get(h) = commandQueueDesc; + std::get(h) = ts; + } } } } @@ -825,11 +841,16 @@ static void event_profiling_callback(void *btx_handle, } // If not IMM will be commandQueueDesc overwrited latter - data->eventToBtxDesct[{hostname, vpid, hEvent}] = {vtid, commandQueueDesc, - hCommandList, hCommandListIsImmediate, - hDevice, commandName, - ts_min, clockLttngDevice, - type, ptr}; + // Push onto the per-event ring. If the cursor has advanced (we've + // already consumed at least one result for this event), the prior + // ring belongs to a finished build phase — clear and start fresh. + auto &ring = data->eventToBtxDesct[{hostname, vpid, hEvent}]; + if (ring.cursor > 0) { + ring.entries.clear(); + ring.cursor = 0; + } + ring.entries.push_back({vtid, commandQueueDesc, hCommandList, hCommandListIsImmediate, hDevice, + commandName, ts_min, clockLttngDevice, type, ptr}); // Prepare job for non IMM if (!hCommandListIsImmediate) data->commandListToEvents[{hostname, vpid, hCommandList}].insert(hEvent); @@ -880,14 +901,17 @@ static void event_profiling_result_callback(void *btx_handle, auto *data = static_cast(usr_data); - // TODO: Should we always find the eventToBtxDesct? - // We didn't find the partial payload, that mean we should ignore it + // Read the current ring slot for this event; advance the cursor; + // wrap to 0 on overflow. Resubmits re-cycle through the same ring. const auto it_p = data->eventToBtxDesct.find({hostname, vpid, hEvent}); - if (it_p == data->eventToBtxDesct.cend()) + if (it_p == data->eventToBtxDesct.cend() || it_p->second.entries.empty()) return; - // We don't erase, may have one entry for multiple result + auto &ring = it_p->second; + if (ring.cursor >= ring.entries.size()) + ring.cursor = 0; const auto &[vtid_submission, commandQueueDesc, hCommandList, hCommandListIsImmediate, device, - commandName, lltngMin, clockLttngDevice, type, ptr] = it_p->second; + commandName, lltngMin, clockLttngDevice, type, ptr] = ring.entries[ring.cursor]; + ring.cursor++; std::string metadata = ""; { std::stringstream ss_metadata; @@ -901,6 +925,13 @@ static void event_profiling_result_callback(void *btx_handle, if (!hCommandListIsImmediate) data->commandListToEvents[{hostname, vpid, hCommandList}].erase(hEvent); + /* AppendSignalEvent is a host-side signal with no GPU work to time. + * We pushed a ring entry to keep state consistent (so a future + * profiling_results lookup doesn't walk a stale prior entry), but + * suppress the device-side tally emission here. */ + if (type == btx_event_t::SIGNAL) + return; + if ((type == btx_event_t::TRAFFIC) && (status == ZE_RESULT_SUCCESS)) { auto &[ts, size] = std::get(ptr); btx_push_message_lttng_traffic(btx_handle, hostname, vpid, vtid, ts, BACKEND_ZE, @@ -1400,6 +1431,12 @@ void btx_register_usr_callbacks(void *btx_handle) { REGISTER_ASSOCIATED_CALLBACK(eventMemory_without_hSignalEvent_exit); REGISTER_ASSOCIATED_CALLBACK(hSignalEvent_rest_entry); + /* zeCommandListAppendSignalEvent doesn't match the hSignalEvent_* sets + * (payload is `hEvent`, not `hSignalEvent`), so it needs its own entry + * callback to keep threadToLastLaunchInfo from going stale. */ + btx_register_callbacks_lttng_ust_ze_zeCommandListAppendSignalEvent_entry( + btx_handle, &zeCommandListAppendSignalEvent_entry_callback); + /* Remove Memory */ REGISTER_ASSOCIATED_CALLBACK(memFree_entry); REGISTER_ASSOCIATED_CALLBACK(memFree_exit); diff --git a/backends/ze/btx_zeinterval_callbacks.hpp b/backends/ze/btx_zeinterval_callbacks.hpp index 80c2dc11..a6cdb0e0 100644 --- a/backends/ze/btx_zeinterval_callbacks.hpp +++ b/backends/ze/btx_zeinterval_callbacks.hpp @@ -55,7 +55,9 @@ using btx_kernel_group_size_t = std::tuple; using btx_kernel_desct_t = std::tuple; -enum class btx_event_t { TRAFFIC, KERNEL, OTHER }; +// SIGNAL = zeCommandListAppendSignalEvent. Ring entry is created so state +// stays consistent, but filtered out of the device tally (no GPU work). +enum class btx_event_t { TRAFFIC, KERNEL, SIGNAL, OTHER }; using btx_additional_info_traffic_t = std::tuple; using btx_additional_info_kernel_t = std::string /*metadata*/; using btx_additional_info = @@ -93,7 +95,18 @@ struct data_s { std::unordered_map commandQueueToDesc; std::unordered_map threadToLastLaunchInfo; - std::unordered_map eventToBtxDesct; + + /* Per-event metadata ring. An hEvent can be the signal event of N + * Appends in one build phase, and the cl can be resubmitted M times, + * yielding M*N result events. We store the N Appends as a vector and + * advance `cursor` per result, wrapping at the end. A new push that + * arrives after the cursor advanced indicates a new build phase — + * we clear and start over so the ring tracks only the current phase. */ + struct event_ring_t { + std::vector entries; + size_t cursor = 0; + }; + std::unordered_map eventToBtxDesct; // Require for non IMM std::unordered_map> commandListToEvents; diff --git a/backends/ze/gen_ze.rb b/backends/ze/gen_ze.rb index 12dc75df..9e662b95 100644 --- a/backends/ze/gen_ze.rb +++ b/backends/ze/gen_ze.rb @@ -8,6 +8,7 @@ #include #include #include + #include #include #include #include diff --git a/backends/ze/tests/interval_profiling_fast.bt_text_pretty b/backends/ze/tests/interval_profiling_resubmit_event.bt_text_pretty similarity index 57% rename from backends/ze/tests/interval_profiling_fast.bt_text_pretty rename to backends/ze/tests/interval_profiling_resubmit_event.bt_text_pretty index 3403ebcd..68e12d80 100644 --- a/backends/ze/tests/interval_profiling_fast.bt_text_pretty +++ b/backends/ze/tests/interval_profiling_resubmit_event.bt_text_pretty @@ -1,2 +1,3 @@ +lttng:host: { hostname = "testhost", vpid = 10, vtid = 1, ts = 1704110400000000000, backend = 1 }, { name = "zeCommandListAppendBarrier", dur = 20, err = false } lttng:device: { hostname = "testhost", vpid = 10, vtid = 1, ts = 1704110400000000000, backend = 1 }, { name = "zeCommandListAppendBarrier", dur = 10, did = 0, sdid = 0, err = false, metadata = "{ordinal: 0, index: 0}" } -lttng:host: { hostname = "testhost", vpid = 10, vtid = 1, ts = 1704110400000000000, backend = 1 }, { name = "zeCommandListAppendBarrier", dur = 30, err = false } +lttng:device: { hostname = "testhost", vpid = 10, vtid = 1, ts = 1704110400000000000, backend = 1 }, { name = "zeCommandListAppendBarrier", dur = 30, did = 0, sdid = 0, err = false, metadata = "{ordinal: 0, index: 0}" } diff --git a/backends/ze/tests/interval_profiling_fast.thapi_text_pretty b/backends/ze/tests/interval_profiling_resubmit_event.thapi_text_pretty similarity index 50% rename from backends/ze/tests/interval_profiling_fast.thapi_text_pretty rename to backends/ze/tests/interval_profiling_resubmit_event.thapi_text_pretty index fb6f10a7..b4c3ca9b 100644 --- a/backends/ze/tests/interval_profiling_fast.thapi_text_pretty +++ b/backends/ze/tests/interval_profiling_resubmit_event.thapi_text_pretty @@ -1,4 +1,8 @@ +# 1 Append, but the underlying cl is Executed twice in a real run, so +# 2 results arrive for the same hEvent. Both are attributed to that one +# Append. 12:00:00.000000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze:zeCommandListAppendBarrier_entry: { hCommandList: 0x1000000000000000, hSignalEvent: 0x0000000000000000, numWaitEvents: 0, phWaitEvents: 0x0000000000000000, _phWaitEvents_vals_length: 0, phWaitEvents_vals: 0x0000000000000000 } -12:00:00.010000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling: { hEvent: 0x1000000000000000 } -12:00:00.020000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling_results: { hEvent: 0x1000000000000000, status: ZE_RESULT_SUCCESS, timestampStatus: ZE_RESULT_SUCCESS, globalStart: 0, globalEnd: 10, contextStart: 0, contextEnd: 10 } -12:00:00.030000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze:zeCommandListAppendBarrier_exit: { zeResult: ZE_RESULT_SUCCESS } +12:00:00.010000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling: { hEvent: 0x4000000000000000 } +12:00:00.020000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze:zeCommandListAppendBarrier_exit: { zeResult: ZE_RESULT_SUCCESS } +12:00:00.100000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling_results: { hEvent: 0x4000000000000000, status: ZE_RESULT_SUCCESS, timestampStatus: ZE_RESULT_SUCCESS, globalStart: 0, globalEnd: 10, contextStart: 0, contextEnd: 10 } +12:00:00.200000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling_results: { hEvent: 0x4000000000000000, status: ZE_RESULT_SUCCESS, timestampStatus: ZE_RESULT_SUCCESS, globalStart: 100, globalEnd: 130, contextStart: 100, contextEnd: 130 } diff --git a/backends/ze/tests/interval_profiling_shared_event.bt_text_pretty b/backends/ze/tests/interval_profiling_shared_event.bt_text_pretty new file mode 100644 index 00000000..2205557d --- /dev/null +++ b/backends/ze/tests/interval_profiling_shared_event.bt_text_pretty @@ -0,0 +1,8 @@ +lttng:host: { hostname = "testhost", vpid = 10, vtid = 1, ts = 1704110400000000000, backend = 1 }, { name = "zeCommandListAppendBarrier", dur = 20, err = false } +lttng:host: { hostname = "testhost", vpid = 10, vtid = 1, ts = 1704110400000000100, backend = 1 }, { name = "zeCommandListAppendBarrier", dur = 20, err = false } +lttng:host: { hostname = "testhost", vpid = 10, vtid = 1, ts = 1704110400000000200, backend = 1 }, { name = "zeCommandListAppendBarrier", dur = 20, err = false } +lttng:host: { hostname = "testhost", vpid = 10, vtid = 1, ts = 1704110400000000300, backend = 1 }, { name = "zeCommandListAppendBarrier", dur = 20, err = false } +lttng:device: { hostname = "testhost", vpid = 10, vtid = 1, ts = 1704110400000000000, backend = 1 }, { name = "zeCommandListAppendBarrier", dur = 10, did = 0, sdid = 0, err = false, metadata = "{ordinal: 0, index: 0}" } +lttng:device: { hostname = "testhost", vpid = 10, vtid = 1, ts = 1704110400000000100, backend = 1 }, { name = "zeCommandListAppendBarrier", dur = 30, did = 0, sdid = 0, err = false, metadata = "{ordinal: 0, index: 0}" } +lttng:device: { hostname = "testhost", vpid = 10, vtid = 1, ts = 1704110400000000200, backend = 1 }, { name = "zeCommandListAppendBarrier", dur = 40, did = 0, sdid = 0, err = false, metadata = "{ordinal: 0, index: 0}" } +lttng:device: { hostname = "testhost", vpid = 10, vtid = 1, ts = 1704110400000000300, backend = 1 }, { name = "zeCommandListAppendBarrier", dur = 50, did = 0, sdid = 0, err = false, metadata = "{ordinal: 0, index: 0}" } diff --git a/backends/ze/tests/interval_profiling_shared_event.thapi_text_pretty b/backends/ze/tests/interval_profiling_shared_event.thapi_text_pretty new file mode 100644 index 00000000..64199d25 --- /dev/null +++ b/backends/ze/tests/interval_profiling_shared_event.thapi_text_pretty @@ -0,0 +1,18 @@ +# 4 Appends share one hEvent. Each Append's result is attributed back to +# its own Append, in submission order. +12:00:00.000000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze:zeCommandListAppendBarrier_entry: { hCommandList: 0x1000000000000000, hSignalEvent: 0x0000000000000000, numWaitEvents: 0, phWaitEvents: 0x0000000000000000, _phWaitEvents_vals_length: 0, phWaitEvents_vals: 0x0000000000000000 } +12:00:00.010000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling: { hEvent: 0x4000000000000000 } +12:00:00.020000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze:zeCommandListAppendBarrier_exit: { zeResult: ZE_RESULT_SUCCESS } +12:00:00.100000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze:zeCommandListAppendBarrier_entry: { hCommandList: 0x1000000000000000, hSignalEvent: 0x0000000000000000, numWaitEvents: 0, phWaitEvents: 0x0000000000000000, _phWaitEvents_vals_length: 0, phWaitEvents_vals: 0x0000000000000000 } +12:00:00.110000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling: { hEvent: 0x4000000000000000 } +12:00:00.120000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze:zeCommandListAppendBarrier_exit: { zeResult: ZE_RESULT_SUCCESS } +12:00:00.200000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze:zeCommandListAppendBarrier_entry: { hCommandList: 0x1000000000000000, hSignalEvent: 0x0000000000000000, numWaitEvents: 0, phWaitEvents: 0x0000000000000000, _phWaitEvents_vals_length: 0, phWaitEvents_vals: 0x0000000000000000 } +12:00:00.210000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling: { hEvent: 0x4000000000000000 } +12:00:00.220000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze:zeCommandListAppendBarrier_exit: { zeResult: ZE_RESULT_SUCCESS } +12:00:00.300000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze:zeCommandListAppendBarrier_entry: { hCommandList: 0x1000000000000000, hSignalEvent: 0x0000000000000000, numWaitEvents: 0, phWaitEvents: 0x0000000000000000, _phWaitEvents_vals_length: 0, phWaitEvents_vals: 0x0000000000000000 } +12:00:00.310000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling: { hEvent: 0x4000000000000000 } +12:00:00.320000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze:zeCommandListAppendBarrier_exit: { zeResult: ZE_RESULT_SUCCESS } +12:00:00.400000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling_results: { hEvent: 0x4000000000000000, status: ZE_RESULT_SUCCESS, timestampStatus: ZE_RESULT_SUCCESS, globalStart: 0, globalEnd: 10, contextStart: 0, contextEnd: 10 } +12:00:00.410000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling_results: { hEvent: 0x4000000000000000, status: ZE_RESULT_SUCCESS, timestampStatus: ZE_RESULT_SUCCESS, globalStart: 100, globalEnd: 130, contextStart: 100, contextEnd: 130 } +12:00:00.420000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling_results: { hEvent: 0x4000000000000000, status: ZE_RESULT_SUCCESS, timestampStatus: ZE_RESULT_SUCCESS, globalStart: 200, globalEnd: 240, contextStart: 200, contextEnd: 240 } +12:00:00.430000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling_results: { hEvent: 0x4000000000000000, status: ZE_RESULT_SUCCESS, timestampStatus: ZE_RESULT_SUCCESS, globalStart: 300, globalEnd: 350, contextStart: 300, contextEnd: 350 } diff --git a/backends/ze/tests/interval_profiling_shared_event_resubmit.bt_text_pretty b/backends/ze/tests/interval_profiling_shared_event_resubmit.bt_text_pretty new file mode 100644 index 00000000..25a5c77b --- /dev/null +++ b/backends/ze/tests/interval_profiling_shared_event_resubmit.bt_text_pretty @@ -0,0 +1,6 @@ +lttng:host: { hostname = "testhost", vpid = 10, vtid = 1, ts = 1704110400000000000, backend = 1 }, { name = "zeCommandListAppendBarrier", dur = 20, err = false } +lttng:host: { hostname = "testhost", vpid = 10, vtid = 1, ts = 1704110400000000100, backend = 1 }, { name = "zeCommandListAppendBarrier", dur = 20, err = false } +lttng:device: { hostname = "testhost", vpid = 10, vtid = 1, ts = 1704110400000000000, backend = 1 }, { name = "zeCommandListAppendBarrier", dur = 10, did = 0, sdid = 0, err = false, metadata = "{ordinal: 0, index: 0}" } +lttng:device: { hostname = "testhost", vpid = 10, vtid = 1, ts = 1704110400000000100, backend = 1 }, { name = "zeCommandListAppendBarrier", dur = 30, did = 0, sdid = 0, err = false, metadata = "{ordinal: 0, index: 0}" } +lttng:device: { hostname = "testhost", vpid = 10, vtid = 1, ts = 1704110400000000000, backend = 1 }, { name = "zeCommandListAppendBarrier", dur = 40, did = 0, sdid = 0, err = false, metadata = "{ordinal: 0, index: 0}" } +lttng:device: { hostname = "testhost", vpid = 10, vtid = 1, ts = 1704110400000000100, backend = 1 }, { name = "zeCommandListAppendBarrier", dur = 50, did = 0, sdid = 0, err = false, metadata = "{ordinal: 0, index: 0}" } diff --git a/backends/ze/tests/interval_profiling_shared_event_resubmit.thapi_text_pretty b/backends/ze/tests/interval_profiling_shared_event_resubmit.thapi_text_pretty new file mode 100644 index 00000000..fb64b5d7 --- /dev/null +++ b/backends/ze/tests/interval_profiling_shared_event_resubmit.thapi_text_pretty @@ -0,0 +1,13 @@ +# 2 Appends share one hEvent, then the underlying cl is Executed twice, +# so 4 results arrive. Each submission's pair of results is attributed to +# the two Appends in submission order. +12:00:00.000000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze:zeCommandListAppendBarrier_entry: { hCommandList: 0x1000000000000000, hSignalEvent: 0x0000000000000000, numWaitEvents: 0, phWaitEvents: 0x0000000000000000, _phWaitEvents_vals_length: 0, phWaitEvents_vals: 0x0000000000000000 } +12:00:00.010000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling: { hEvent: 0x4000000000000000 } +12:00:00.020000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze:zeCommandListAppendBarrier_exit: { zeResult: ZE_RESULT_SUCCESS } +12:00:00.100000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze:zeCommandListAppendBarrier_entry: { hCommandList: 0x1000000000000000, hSignalEvent: 0x0000000000000000, numWaitEvents: 0, phWaitEvents: 0x0000000000000000, _phWaitEvents_vals_length: 0, phWaitEvents_vals: 0x0000000000000000 } +12:00:00.110000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling: { hEvent: 0x4000000000000000 } +12:00:00.120000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze:zeCommandListAppendBarrier_exit: { zeResult: ZE_RESULT_SUCCESS } +12:00:00.200000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling_results: { hEvent: 0x4000000000000000, status: ZE_RESULT_SUCCESS, timestampStatus: ZE_RESULT_SUCCESS, globalStart: 0, globalEnd: 10, contextStart: 0, contextEnd: 10 } +12:00:00.210000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling_results: { hEvent: 0x4000000000000000, status: ZE_RESULT_SUCCESS, timestampStatus: ZE_RESULT_SUCCESS, globalStart: 100, globalEnd: 130, contextStart: 100, contextEnd: 130 } +12:00:00.300000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling_results: { hEvent: 0x4000000000000000, status: ZE_RESULT_SUCCESS, timestampStatus: ZE_RESULT_SUCCESS, globalStart: 200, globalEnd: 240, contextStart: 200, contextEnd: 240 } +12:00:00.310000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling_results: { hEvent: 0x4000000000000000, status: ZE_RESULT_SUCCESS, timestampStatus: ZE_RESULT_SUCCESS, globalStart: 300, globalEnd: 350, contextStart: 300, contextEnd: 350 } diff --git a/backends/ze/tests/interval_profiling_shared_event_xphase.bt_text_pretty b/backends/ze/tests/interval_profiling_shared_event_xphase.bt_text_pretty new file mode 100644 index 00000000..ebbbc90a --- /dev/null +++ b/backends/ze/tests/interval_profiling_shared_event_xphase.bt_text_pretty @@ -0,0 +1,10 @@ +lttng:host: { hostname = "testhost", vpid = 10, vtid = 1, ts = 1704110400000000000, backend = 1 }, { name = "zeCommandListAppendBarrier", dur = 20, err = false } +lttng:host: { hostname = "testhost", vpid = 10, vtid = 1, ts = 1704110400000000100, backend = 1 }, { name = "zeCommandListAppendBarrier", dur = 20, err = false } +lttng:device: { hostname = "testhost", vpid = 10, vtid = 1, ts = 1704110400000000000, backend = 1 }, { name = "zeCommandListAppendBarrier", dur = 10, did = 0, sdid = 0, err = false, metadata = "{ordinal: 0, index: 0}" } +lttng:device: { hostname = "testhost", vpid = 10, vtid = 1, ts = 1704110400000000100, backend = 1 }, { name = "zeCommandListAppendBarrier", dur = 30, did = 0, sdid = 0, err = false, metadata = "{ordinal: 0, index: 0}" } +lttng:device: { hostname = "testhost", vpid = 10, vtid = 1, ts = 1704110400000000000, backend = 1 }, { name = "zeCommandListAppendBarrier", dur = 40, did = 0, sdid = 0, err = false, metadata = "{ordinal: 0, index: 0}" } +lttng:device: { hostname = "testhost", vpid = 10, vtid = 1, ts = 1704110400000000100, backend = 1 }, { name = "zeCommandListAppendBarrier", dur = 50, did = 0, sdid = 0, err = false, metadata = "{ordinal: 0, index: 0}" } +lttng:host: { hostname = "testhost", vpid = 10, vtid = 1, ts = 1704110400000000400, backend = 1 }, { name = "zeCommandListAppendBarrier", dur = 20, err = false } +lttng:device: { hostname = "testhost", vpid = 10, vtid = 1, ts = 1704110400000000400, backend = 1 }, { name = "zeCommandListAppendBarrier", dur = 10, did = 0, sdid = 0, err = false, metadata = "{ordinal: 0, index: 0}" } +lttng:device: { hostname = "testhost", vpid = 10, vtid = 1, ts = 1704110400000000400, backend = 1 }, { name = "zeCommandListAppendBarrier", dur = 20, did = 0, sdid = 0, err = false, metadata = "{ordinal: 0, index: 0}" } +lttng:device: { hostname = "testhost", vpid = 10, vtid = 1, ts = 1704110400000000400, backend = 1 }, { name = "zeCommandListAppendBarrier", dur = 40, did = 0, sdid = 0, err = false, metadata = "{ordinal: 0, index: 0}" } diff --git a/backends/ze/tests/interval_profiling_shared_event_xphase.thapi_text_pretty b/backends/ze/tests/interval_profiling_shared_event_xphase.thapi_text_pretty new file mode 100644 index 00000000..e9d336d8 --- /dev/null +++ b/backends/ze/tests/interval_profiling_shared_event_xphase.thapi_text_pretty @@ -0,0 +1,21 @@ +# Two build phases on the same cl, both reusing the same hEvent. +# Phase 1: 2 Appends, cl Executed twice -> 4 results. +# Phase 2: cl is Reset, then 1 Append, cl Executed three times -> 3 results. +# Results from a phase are attributed only to that phase's Appends; the +# phase-1 Appends do not bleed into the phase-2 results. +12:00:00.000000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze:zeCommandListAppendBarrier_entry: { hCommandList: 0x1000000000000000, hSignalEvent: 0x0000000000000000, numWaitEvents: 0, phWaitEvents: 0x0000000000000000, _phWaitEvents_vals_length: 0, phWaitEvents_vals: 0x0000000000000000 } +12:00:00.010000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling: { hEvent: 0x4000000000000000 } +12:00:00.020000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze:zeCommandListAppendBarrier_exit: { zeResult: ZE_RESULT_SUCCESS } +12:00:00.100000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze:zeCommandListAppendBarrier_entry: { hCommandList: 0x1000000000000000, hSignalEvent: 0x0000000000000000, numWaitEvents: 0, phWaitEvents: 0x0000000000000000, _phWaitEvents_vals_length: 0, phWaitEvents_vals: 0x0000000000000000 } +12:00:00.110000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling: { hEvent: 0x4000000000000000 } +12:00:00.120000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze:zeCommandListAppendBarrier_exit: { zeResult: ZE_RESULT_SUCCESS } +12:00:00.200000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling_results: { hEvent: 0x4000000000000000, status: ZE_RESULT_SUCCESS, timestampStatus: ZE_RESULT_SUCCESS, globalStart: 0, globalEnd: 10, contextStart: 0, contextEnd: 10 } +12:00:00.210000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling_results: { hEvent: 0x4000000000000000, status: ZE_RESULT_SUCCESS, timestampStatus: ZE_RESULT_SUCCESS, globalStart: 100, globalEnd: 130, contextStart: 100, contextEnd: 130 } +12:00:00.300000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling_results: { hEvent: 0x4000000000000000, status: ZE_RESULT_SUCCESS, timestampStatus: ZE_RESULT_SUCCESS, globalStart: 200, globalEnd: 240, contextStart: 200, contextEnd: 240 } +12:00:00.310000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling_results: { hEvent: 0x4000000000000000, status: ZE_RESULT_SUCCESS, timestampStatus: ZE_RESULT_SUCCESS, globalStart: 300, globalEnd: 350, contextStart: 300, contextEnd: 350 } +12:00:00.400000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze:zeCommandListAppendBarrier_entry: { hCommandList: 0x1000000000000000, hSignalEvent: 0x0000000000000000, numWaitEvents: 0, phWaitEvents: 0x0000000000000000, _phWaitEvents_vals_length: 0, phWaitEvents_vals: 0x0000000000000000 } +12:00:00.410000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling: { hEvent: 0x4000000000000000 } +12:00:00.420000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze:zeCommandListAppendBarrier_exit: { zeResult: ZE_RESULT_SUCCESS } +12:00:00.500000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling_results: { hEvent: 0x4000000000000000, status: ZE_RESULT_SUCCESS, timestampStatus: ZE_RESULT_SUCCESS, globalStart: 400, globalEnd: 410, contextStart: 400, contextEnd: 410 } +12:00:00.510000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling_results: { hEvent: 0x4000000000000000, status: ZE_RESULT_SUCCESS, timestampStatus: ZE_RESULT_SUCCESS, globalStart: 500, globalEnd: 520, contextStart: 500, contextEnd: 520 } +12:00:00.520000000 - testhost - vpid: 10, vtid: 1 - lttng_ust_ze_profiling:event_profiling_results: { hEvent: 0x4000000000000000, status: ZE_RESULT_SUCCESS, timestampStatus: ZE_RESULT_SUCCESS, globalStart: 600, globalEnd: 640, contextStart: 600, contextEnd: 640 } diff --git a/backends/ze/tracer_ze_helpers.include.c b/backends/ze/tracer_ze_helpers.include.c index 8cfe31d7..10693c1a 100644 --- a/backends/ze/tracer_ze_helpers.include.c +++ b/backends/ze/tracer_ze_helpers.include.c @@ -1,30 +1,139 @@ -#ifdef THAPI_DEBUG -#define TAHPI_LOG stderr -#define THAPI_DBGLOG(fmt, ...) \ - do { \ - fprintf(TAHPI_LOG, "THAPI(%s:%d): " fmt "\n", __func__, __LINE__, __VA_ARGS__); \ - } while (0) -#define THAPI_DBGLOG_NO_ARGS(fmt) \ +/* Algorithm + * ========= + * + * On profiled Append (cl, sig=user_sig, waits=user_waits): + * - allocate inj from per-context pool; swap user_sig -> inj + * - place a Query (see "QKT placement" below) + * - allocate a slot {inj, attr=user_sig, off, waits=copy(user_waits)} + * - immediate cl: instantiate(slot) inline + * + * instantiate(s): + * - s.preds = [event_latest_signaled[w] for w in s.waits if live] + * + previous live slot in same cl (if cl is in-order) + * - s.live = true; event_latest_signaled[s.attr] = &s + * + * On Execute(q, cl) prologue: + * - if cl.in_flight_q: Synchronize(in_flight_q); drain_cl(cl) + * - shadow-path slots: re-Append Query on shadow cl + * inline-path slots: nothing (Query is baked into cl body) + * - instantiate every slot in cl + * - cl.in_flight_q = q; index cl under q (and its fence) for sync lookup + * + * On Sync (the synced anchor tells us what to drain): + * - Sync(ev): drain(event_latest_signaled[ev]) + * - Sync(q): drain_cl(cl) for every cl in the q-index bucket for q + * (O(matching cls), not a scan of every live cl) + * - Sync(cl): drain_cl(cl) + * + * drain(s): + * - for p in s.preds: drain(p) + * - shadow-path: host-sync on shadow_done, reset, decrement live_queries + * - read slab[s.off], emit tracepoint(s.attr or inj) + * - clear event_latest_signaled[s.attr] (if it still points at s) + * - clear s.live and s.preds + * (Build-time fields inj, attr, off, waits stay so the next Execute + * can re-instantiate without re-Appending.) + * + * Concurrency + * =========== + * + * One global mutex (_ze_state_mutex) covers all tracer state: the cl + * registry, every cl's chunks/slots/preds, the event freelist + pool + * registry, the latest-signaled map, the shadow cl registry, the + * qgroup cache. Append / Execute / Drain / Destroy all take it. + * + * Per-cl mutexes don't work because drain follows cross-cl pred edges + * (event_latest_signaled[ev] can point at a slot in any cl) and + * mutates the pred's chunk via _slot_release. Any per-cl scheme has + * to acquire multiple cl mutexes with cross-cl ordering rules. One + * global mutex sidesteps that entirely. + * + * Perf: Append on different cls and freelist accesses serialize + * through one lock. The L0 calls inside the critical section + * (AppendBarrier, AppendQueryKernelTimestamps) just queue work on + * the GPU — the GPU executes asynchronously, so the held region is + * short. Drain is host-blocking (zeEventHostSynchronize on shadow + * fence events) and was effectively serial anyway. + * + * QKT placement + * ============= + * + * AppendQueryKernelTimestamps (the device-side timestamp read) lives + * in one of two places, picked at cl create from the queue group's + * COMPUTE flag and stored in cl_data->is_compute. Both paths share the + * slot/drain/dep-graph machinery; they only differ in where the QKT is + * Appended and how the drain knows it has fired. + * + * INLINE (user cl is on a COMPUTE queue group): + * + * Kernel(sig=inj) ──> QKT(wait=inj, sig=user_signal) [on user cl] + * + * One Append. user_signal IS the QKT-done edge — any user-level + * sync (event/queue/cl) that covers user_signal also covers the + * QKT. No tracer fence event, no host-sync at drain. For regular + * cls the QKT is baked into the cl body once and re-fires on every + * Execute. + * + * SHADOW (user cl is copy-only, or queue group unknown): + * + * ┌─> Barrier(wait=inj, sig=user_signal) [on user cl] + * Kernel(sig=inj) ──┤ + * └─> QKT(wait=inj, sig=shadow_done) [on shadow cl] + * + * Two Appends. The shadow cl is a per-(context, device) tracer-owned + * immediate compute cl; QKT goes there because copy queue groups + * reject AppendQueryKernelTimestamps. shadow_done is a tracer-owned + * fence event that drain host-syncs on — required because the + * shadow cl's completion isn't implied by any user-level sync. For + * regular cls the shadow QKT is (re-)Appended in the Execute + * epilogue (the user cl is in flight by then, so Appending the + * Query won't deadlock on a shared engine). + */ + +/* Always-on tracer log. Prefixes THAPI(func:line) so messages are + * grep-able across the bench/test harness which often interleaves + * tracer and user output. GCC's `, ##__VA_ARGS__` extension swallows + * the leading comma when the variadic list is empty. fflush so the + * line lands even if we abort() right after. */ +#define _THAPI_LOG(fmt, ...) \ do { \ - fprintf(TAHPI_LOG, "THAPI(%s:%d): " fmt "\n", __func__, __LINE__); \ + fprintf(stderr, "THAPI(%s:%d): " fmt "\n", __func__, __LINE__, ##__VA_ARGS__); \ + fflush(stderr); \ } while (0) + +#ifdef THAPI_DEBUG +#define THAPI_DBGLOG(fmt, ...) _THAPI_LOG(fmt, ##__VA_ARGS__) #else #define THAPI_DBGLOG(...) \ do { \ } while (0) -#define THAPI_DBGLOG_NO_ARGS(fmt) \ +#endif + +/* Tracer invariant check: print + abort. Unconditional (not gated on + * NDEBUG) — silently dropping the check would let the bug ship bad + * data instead of crashing. Use for "this can never happen" preconditions + * inside the tracer, not for user-input validation. */ +#define _THAPI_ASSERT(cond, fmt, ...) \ do { \ + if (!(cond)) { \ + _THAPI_LOG("assertion failed: %s — " fmt, #cond, ##__VA_ARGS__); \ + abort(); \ + } \ } while (0) -#endif -#ifdef THAPI_USE_DESTRUCTORS -#define THAPI_ATTRIBUTE_DESTRUCTOR __attribute__((destructor)) -#else -#define THAPI_ATTRIBUTE_DESTRUCTOR -#endif +/* Wrap a tracer-issued L0 call whose failure means we'd either hang the + * user (sync chain Barrier) or produce a non-self-consistent trace + * (QKT, event create, ...). Defensive: print + abort so the bug surfaces + * under sanitizers/CI rather than ship bad data. NOT for driver query + * calls (Get*Handle, GetCommandQueueGroupProperties) — those can fail + * transiently during teardown and have graceful fallbacks. */ +#define _ZE_MUST(call) \ + do { \ + ze_result_t _r = (call); \ + _THAPI_ASSERT(_r == ZE_RESULT_SUCCESS, "%s = 0x%x", #call, _r); \ + } while (0) static int _do_profile = 0; -static int _do_cleanup = 0; static int _do_chained_structs = 0; static int _do_paranoid_drift = 0; static int _do_paranoid_memory_location = 0; @@ -43,110 +152,425 @@ struct ze_closure { struct ze_closure *ze_closures = NULL; -typedef enum _ze_command_list_flag { _ZE_EXECUTED = ZE_BIT(0) } _ze_command_list_flag_t; -typedef _ze_command_list_flag_t _ze_command_list_flags_t; - struct _ze_event_h; +struct _ze_slot; +struct _ze_slab_chunk; + +/* Dependency-tracking slot: one per profiled Append. Slots carry the + * happens-before edges the user established (via cl in-order semantics + * and via phWaitEvents). At sync time we walk these edges from the + * synced anchor and drain everything reachable. Drain is pop semantics: + * after emit, the slot is dropped from the cl's list. */ +struct _ze_slot { + struct _ze_command_list_obj_data *owner; /* cl_data this slot lives in */ + struct _ze_slab_chunk *chunk; /* chunk this slot lives in (==> .slab to read at drain) */ + /* Shadow path only: shadow cl the Query was Appended to. Inline-path + * slots leave this NULL — their Query lives in the user cl body and + * the dep-graph walk that triggers drain already implies it has run. */ + struct _ze_shadow_cl *sh; + struct _ze_event_h *inj; /* tracer-owned event the Query waits on */ + /* Shadow path only: tracer-owned fence event the Query signals; drain + * host-syncs on it. Inline-path slots leave this NULL. */ + struct _ze_event_h *shadow_done; + ze_event_handle_t attr; /* user's original signal event (NULL => inj->event) */ + size_t off; /* byte offset within chunk->slab */ + /* User wait events copied at Append time (stable across rebuilds); + * preds[] is computed at instantiate from waits[] by looking up + * event_latest_signaled[w] for each w. */ + ze_event_handle_t *waits; + uint32_t n_waits; + struct _ze_slot **preds; /* points at slots whose drain must come first (may be in another cl) */ + uint32_t n_preds; + unsigned char live; /* in-flight (instantiated, not drained) */ + /* Incoming pred edges: count of downstream slots whose preds[] points + * here AND that have not yet been drained. Incremented at downstream + * _slot_instantiate, decremented at downstream _slot_drain. Slot is + * reclaimable iff live==0 AND refs==0. */ + uint32_t refs; +}; + +#define _ZE_SLAB_CHUNK_SLOTS 64 + +/* Slot + slab storage in fixed-size chunks; cl_data->chunks is a utlist + * DL of these. Imm cls allocate new chunks as needed (no cap); regular + * cls stop at one chunk — the inj events (and on the inline path, the + * QKT itself) are baked into the closed cl body, so adding a chunk + * after Close would create slots the body doesn't address. + * + * Within a chunk, slots[i].off is i * sizeof(timestamp) into slab. The + * chunk frees itself when n_held drops to 0 AND it is not the tail + * (new Appends still want to land on the tail). */ +struct _ze_slab_chunk { + void *slab; /* _ZE_SLAB_CHUNK_SLOTS * sizeof(ze_kernel_timestamp_result_t) */ + ze_context_handle_t slab_ctx; /* context the slab was allocated against (zeMemFree target) */ + uint32_t n_used; /* slots ever assigned in this chunk (monotonic until chunk free) */ + uint32_t n_held; /* unreleased slots (n_used minus _slot_release calls) */ + /* Nonzero only on a DETACHED chunk: one whose owning cl was torn down + * (reset/destroy) while >=1 slot was still referenced as a pred by a live + * slot in ANOTHER cl. The chunk is removed from cl_data->chunks, its slots' + * resources are already released and owner==NULL — only the struct survives + * so the referrers' preds[] pointers stay valid. n_pinned counts those + * surviving referenced slots; the downstream drain that drops the last ref + * frees the struct. 0 for normal attached chunks. */ + uint32_t n_pinned; + struct _ze_slab_chunk *next, *prev; + struct _ze_slot slots[_ZE_SLAB_CHUNK_SLOTS]; +}; + +/* Iterate every used slot in a cl, oldest-to-newest (chunk DL order, then + * slot order within a chunk) — the natural time order. Binds `s` to each + * `struct _ze_slot *`. Only for read/dispose passes that do NOT free chunks + * mid-walk; the drain path bumps n_held by hand and uses DL_FOREACH_SAFE. */ +#define _ZE_FOREACH_SLOT(cl_data, s) \ + for (struct _ze_slab_chunk *_c = (cl_data)->chunks; _c; _c = _c->next) \ + for (struct _ze_slot *s = _c->slots, *_se = _c->slots + _c->n_used; s < _se; ++s) struct _ze_command_list_obj_data { - void *ptr; /* the ze_command_list_handle_t this entry tracks */ + void *ptr; UT_hash_handle hh; - _ze_command_list_flags_t flags; - struct _ze_event_h *events; + + struct _ze_slab_chunk *chunks; /* utlist DL_ head; tail = chunks->prev (circular) */ + + /* in_flight_q is the queue this cl was last Executed on AND not yet + * drained. NULL means "not in flight" — safe to Execute without a + * force-sync. Set on Execute, cleared on drain. + * + * Held only for regular cls; immediate cls never Execute. */ + ze_command_queue_handle_t in_flight_q; + /* The fence (if any) passed to that same Execute. NULL when the user + * Executed without a fence. Lets a fence-only sync find which cls to + * drain — the fence signals when all cls in its Execute complete, so + * zeFenceHostSynchronize(f) drains every cl whose in_flight_fence == f. + * Set on Execute alongside in_flight_q, cleared together on drain. */ + ze_fence_handle_t in_flight_fence; + unsigned char is_immediate; + unsigned char is_in_order; + /* 1 if this cl's queue group exposes COMPUTE — its body can host + * AppendQueryKernelTimestamps directly, so we skip the per-(ctx,device) + * shadow cl and bake QKT into the user cl itself. See the placement + * diagram at the top of this file. 0 for copy-only cls and for any cl + * whose group flags we couldn't determine. Set at create; immutable. */ + unsigned char is_compute; + + /* Cached on first use: context handle for this cl. Immutable for the + * cl's lifetime. Load-bearing for _on_destroy_context's sweep: lets it + * associate cls back to their ctx without an L0 roundtrip per cl. */ + ze_context_handle_t cached_context; + + /* Membership in the per-queue / per-fence in-flight indexes (see + * _ze_q_index / _ze_fence_index below). A cl in flight is linked into both + * its queue's bucket (q_prev/q_next) and, if Executed with a fence, its + * fence's bucket (f_prev/f_next), so a queue/fence sync drains exactly the + * matching cls without scanning every live cl. Linked at Execute, unlinked + * at drain, both via _cl_index_clear. */ + struct _ze_command_list_obj_data *q_prev, *q_next; + struct _ze_command_list_obj_data *f_prev, *f_next; }; struct _ze_command_list_obj_data *_ze_cls = NULL; -pthread_mutex_t _ze_cls_mutex = PTHREAD_MUTEX_INITIALIZER; -#define FIND_ZE_CL(key, val) \ - do { \ - pthread_mutex_lock(&_ze_cls_mutex); \ - HASH_FIND_PTR(_ze_cls, key, val); \ - pthread_mutex_unlock(&_ze_cls_mutex); \ - } while (0) +/* The single mutex covering all tracer state — see the "Concurrency" + * section in the file header for rationale. Every static helper in this + * file that touches tracer state assumes the caller holds it. */ +pthread_mutex_t _ze_state_mutex = PTHREAD_MUTEX_INITIALIZER; -#define ADD_ZE_CL(val) \ - do { \ - pthread_mutex_lock(&_ze_cls_mutex); \ - HASH_ADD_PTR(_ze_cls, ptr, val); \ - pthread_mutex_unlock(&_ze_cls_mutex); \ - } while (0) +/* Pure HASH wrappers. */ +static struct _ze_command_list_obj_data *_cl_find(ze_command_list_handle_t command_list) { + struct _ze_command_list_obj_data *cl = NULL; + HASH_FIND_PTR(_ze_cls, &command_list, cl); + return cl; +} -#define FIND_AND_DEL_ZE_CL(key, val) \ - do { \ - pthread_mutex_lock(&_ze_cls_mutex); \ - HASH_FIND_PTR(_ze_cls, key, val); \ - if (val) { \ - HASH_DEL(_ze_cls, val); \ - } \ - pthread_mutex_unlock(&_ze_cls_mutex); \ - } while (0) +static void _cl_add(struct _ze_command_list_obj_data *cl) { HASH_ADD_PTR(_ze_cls, ptr, cl); } -static inline void _on_create_command_list(ze_command_list_handle_t command_list, int immediate) { - struct _ze_command_list_obj_data *cl_data = NULL; +static struct _ze_command_list_obj_data *_cl_find_and_del(ze_command_list_handle_t command_list) { + struct _ze_command_list_obj_data *cl = _cl_find(command_list); + if (cl) + HASH_DEL(_ze_cls, cl); + return cl; +} - FIND_ZE_CL(&command_list, cl_data); - if (cl_data) { - THAPI_DBGLOG("Command list already registered: %p", command_list); +/* In-flight indexes: queue handle -> the cls currently in flight on that queue, + * and fence handle -> the cls in flight under that fence. A queue/fence sync + * completes exactly the cls of the matching Execute, so these let _on_sync + * drain just those cls instead of scanning every live cl (which is O(live cls) + * per sync — see bench/sync_scaling). Buckets are created lazily at Execute and + * freed when they go empty at drain. */ +struct _ze_inflight_bucket { + void *key; /* ze_command_queue_handle_t or ze_fence_handle_t */ + struct _ze_command_list_obj_data *cls; /* DL via q_prev/q_next or f_prev/f_next */ + UT_hash_handle hh; +}; +static struct _ze_inflight_bucket *_ze_q_index = NULL; +static struct _ze_inflight_bucket *_ze_fence_index = NULL; + +static void _index_link(struct _ze_inflight_bucket **index, + void *key, + struct _ze_command_list_obj_data *cl, + int is_fence) { + if (!key) return; + struct _ze_inflight_bucket *b = NULL; + HASH_FIND_PTR(*index, &key, b); + if (!b) { + b = (struct _ze_inflight_bucket *)calloc(1, sizeof(*b)); + if (!b) + return; + b->key = key; + HASH_ADD_PTR(*index, key, b); } + if (is_fence) + DL_APPEND2(b->cls, cl, f_prev, f_next); + else + DL_APPEND2(b->cls, cl, q_prev, q_next); +} - cl_data = (struct _ze_command_list_obj_data *)calloc(1, sizeof(*cl_data)); - if (!cl_data) { - THAPI_DBGLOG_NO_ARGS("Failed to allocate memory"); +static void _index_unlink(struct _ze_inflight_bucket **index, + void *key, + struct _ze_command_list_obj_data *cl, + int is_fence) { + if (!key) + return; + struct _ze_inflight_bucket *b = NULL; + HASH_FIND_PTR(*index, &key, b); + if (!b) return; + if (is_fence) + DL_DELETE2(b->cls, cl, f_prev, f_next); + else + DL_DELETE2(b->cls, cl, q_prev, q_next); + if (!b->cls) { + HASH_DEL(*index, b); + free(b); } +} - cl_data->ptr = (void *)command_list; - /* Immediate cls have no Execute step; their appends run on the device the - * moment they're submitted. Treat them as already-executed so drainers - * (Reset/Destroy hooks) query their events via _ZE_EXECUTED uniformly. */ - if (immediate) - cl_data->flags = _ZE_EXECUTED; +/* Link cl into the queue (and, if non-NULL, fence) in-flight indexes. Called + * once per Execute, after in_flight_q/in_flight_fence are stamped. */ +static void _cl_index_set(struct _ze_command_list_obj_data *cl, + ze_command_queue_handle_t q, + ze_fence_handle_t f) { + _index_link(&_ze_q_index, q, cl, /*is_fence=*/0); + _index_link(&_ze_fence_index, f, cl, /*is_fence=*/1); +} + +/* Remove cl from both in-flight indexes. Uses cl's own in_flight_q/_fence as + * the keys, so it MUST run before those are cleared. Idempotent: a cl not in + * flight has NULL keys and is a no-op. */ +static void _cl_index_clear(struct _ze_command_list_obj_data *cl) { + _index_unlink(&_ze_q_index, cl->in_flight_q, cl, /*is_fence=*/0); + _index_unlink(&_ze_fence_index, cl->in_flight_fence, cl, /*is_fence=*/1); +} + +/* Per-device cache of the queue-group flag bitmap. The lookup is + * read-mostly: scan zeDeviceGetCommandQueueGroupProperties once, + * remember the per-ordinal flags. flags==NULL means "we already checked + * and the device returned no groups". Used by two readers: + * _get_compute_ordinal(dev) -> first COMPUTE ord, or -1 + * _ordinal_is_compute(dev, ord) -> 1 if ord is COMPUTE on dev */ +struct _ze_qgroup_cache_entry { + ze_device_handle_t device; + ze_command_queue_group_property_flags_t *flags; /* owned; n_groups entries */ + uint32_t n_groups; + UT_hash_handle hh; +}; +static struct _ze_qgroup_cache_entry *_ze_qgroup_cache = NULL; + +/* Populate (or return cached) flag bitmap for device. The cache lives + * for process lifetime. First-touch L0 queries happen under the state + * mutex; cost is bounded since lookups are once per device. */ +static struct _ze_qgroup_cache_entry *_qgroup_cache_get(ze_device_handle_t device) { + struct _ze_qgroup_cache_entry *e = NULL; + HASH_FIND_PTR(_ze_qgroup_cache, &device, e); + if (e) + return e; + + uint32_t n_groups = 0; + if (ZE_DEVICE_GET_COMMAND_QUEUE_GROUP_PROPERTIES_PTR(device, &n_groups, NULL) != + ZE_RESULT_SUCCESS || + n_groups == 0) + return NULL; + ze_command_queue_group_properties_t *groups = + (ze_command_queue_group_properties_t *)calloc(n_groups, sizeof(*groups)); + if (!groups) + return NULL; + for (uint32_t i = 0; i < n_groups; ++i) + groups[i].stype = ZE_STRUCTURE_TYPE_COMMAND_QUEUE_GROUP_PROPERTIES; + if (ZE_DEVICE_GET_COMMAND_QUEUE_GROUP_PROPERTIES_PTR(device, &n_groups, groups) != + ZE_RESULT_SUCCESS) { + free(groups); + return NULL; + } + ze_command_queue_group_property_flags_t *flags = + (ze_command_queue_group_property_flags_t *)calloc(n_groups, sizeof(*flags)); + if (!flags) { + free(groups); + return NULL; + } + for (uint32_t i = 0; i < n_groups; ++i) + flags[i] = groups[i].flags; + free(groups); - ADD_ZE_CL(cl_data); + e = (struct _ze_qgroup_cache_entry *)calloc(1, sizeof(*e)); + if (!e) { + free(flags); + return NULL; + } + e->device = device; + e->flags = flags; + e->n_groups = n_groups; + HASH_ADD_PTR(_ze_qgroup_cache, device, e); + return e; } -typedef enum _ze_event_flag { _ZE_IMMEDIATE_CMD = ZE_BIT(0) } _ze_event_flag_t; -typedef _ze_event_flag_t _ze_event_flags_t; +/* Returns the first COMPUTE queue group ordinal for device, or (uint32_t)-1 + * if the device exposes no compute group (fatal — caller should bail). */ +static uint32_t _get_compute_ordinal(ze_device_handle_t device) { + struct _ze_qgroup_cache_entry *e = _qgroup_cache_get(device); + if (!e) + return (uint32_t)-1; + for (uint32_t i = 0; i < e->n_groups; ++i) + if (e->flags[i] & ZE_COMMAND_QUEUE_GROUP_PROPERTY_FLAG_COMPUTE) + return i; + return (uint32_t)-1; +} +/* 1 iff `ordinal` on `device` is a COMPUTE queue group. Returns 0 on any + * uncertainty (unknown device, OOB ordinal, driver error) — callers + * should treat the cl as non-compute and use the shadow-cl QKT path. */ +static int _ordinal_is_compute(ze_device_handle_t device, uint32_t ordinal) { + if (!device) + return 0; + struct _ze_qgroup_cache_entry *e = _qgroup_cache_get(device); + return e && ordinal < e->n_groups && + (e->flags[ordinal] & ZE_COMMAND_QUEUE_GROUP_PROPERTY_FLAG_COMPUTE) + ? 1 + : 0; +} + +/* Per-(context, device) tracer-owned immediate OOO compute cl used by + * the SHADOW path to host AppendQueryKernelTimestamps. Copy queue + * groups reject QKT, so the shadow cl exists to give those user cls + * somewhere compute-capable to put their Query. Compute user cls take + * the INLINE path and never touch a shadow cl — see the QKT placement + * diagram at the top of this file. */ +struct _ze_shadow_key { + ze_context_handle_t context; + ze_device_handle_t device; +}; +struct _ze_shadow_cl { + struct _ze_shadow_key key; + ze_command_list_handle_t cl; + uint32_t live_queries; /* QKTs appended but not yet host-synced */ + UT_hash_handle hh; +}; +static struct _ze_shadow_cl *_ze_shadow_cls = NULL; + +/* Returns the shadow cl for (context, device), creating it lazily on + * first use (first-touch L0 zeCommandListCreateImmediate runs under + * the state mutex; cost bounded). Returns NULL if the device has no + * compute group (fatal: log to stderr) or if creation fails. */ +static struct _ze_shadow_cl *_get_shadow_cl(ze_context_handle_t context, + ze_device_handle_t device) { + struct _ze_shadow_key key = {context, device}; + struct _ze_shadow_cl *sh = NULL; + HASH_FIND(hh, _ze_shadow_cls, &key, sizeof(key), sh); + if (sh) + return sh; + + uint32_t ord = _get_compute_ordinal(device); + if (ord == (uint32_t)-1) { + fprintf(stderr, + "THAPI: device %p has no COMPUTE queue group; " + "cannot create shadow cl. Profiling disabled for " + "command lists on this device.\n", + (void *)device); + return NULL; + } + /* ASYNCHRONOUS mode is critical: with SYNCHRONOUS (the DEFAULT), + * each AppendQueryKernelTimestamps on this immediate cl blocks until + * the Query completes — which it can't, because Query is waiting on + * inj, and inj is signaled by the user cl's kernel that hasn't been + * submitted yet (we're called from the user's Execute prologue). + * Deadlock. ASYNCHRONOUS lets the Append return immediately and the + * Query run device-side at its own pace. */ + ze_command_queue_desc_t qd = { + ZE_STRUCTURE_TYPE_COMMAND_QUEUE_DESC, NULL, ord, 0, 0, ZE_COMMAND_QUEUE_MODE_ASYNCHRONOUS, + ZE_COMMAND_QUEUE_PRIORITY_NORMAL}; + ze_command_list_handle_t new_cl = NULL; + if (ZE_COMMAND_LIST_CREATE_IMMEDIATE_PTR(context, device, &qd, &new_cl) != ZE_RESULT_SUCCESS || + !new_cl) { + fprintf(stderr, + "THAPI: failed to create shadow cl for " + "context=%p device=%p\n", + (void *)context, (void *)device); + return NULL; + } + sh = (struct _ze_shadow_cl *)calloc(1, sizeof(*sh)); + if (!sh) { + ZE_COMMAND_LIST_DESTROY_PTR(new_cl); + return NULL; + } + sh->key = key; + sh->cl = new_cl; + HASH_ADD(hh, _ze_shadow_cls, key, sizeof(sh->key), sh); + return sh; +} + +/* Append AppendQueryKernelTimestamps on the shadow cl: wait on inj, + * signal shadow_done, write timestamps into slab[*off]. The state + * mutex also serializes the not-thread-safe-per-cl-handle L0 Append + * on the shared shadow cl. Aborts on L0 failure (defensive — a missing + * Query would silently drop this kernel's timing). */ +static void _shadow_append_query(struct _ze_shadow_cl *sh, + ze_event_handle_t inj_event, + void *slab, + size_t *off, + ze_event_handle_t shadow_done_event) { + sh->live_queries++; + _ZE_MUST(ZE_COMMAND_LIST_APPEND_QUERY_KERNEL_TIMESTAMPS_PTR(sh->cl, 1, &inj_event, slab, off, + /*hSignalEvent=*/shadow_done_event, + /*numWaitEvents=*/1, &inj_event)); +} + +static inline void _on_create_command_list(ze_command_list_handle_t command_list, + ze_device_handle_t device, + uint32_t ordinal, + int immediate, + int in_order) { + struct _ze_command_list_obj_data *cl_data = + (struct _ze_command_list_obj_data *)calloc(1, sizeof(*cl_data)); + if (!cl_data) { + THAPI_DBGLOG("Failed to allocate memory"); + return; + } + cl_data->ptr = (void *)command_list; + cl_data->is_immediate = immediate ? 1 : 0; + cl_data->is_in_order = in_order ? 1 : 0; + + pthread_mutex_lock(&_ze_state_mutex); + /* _ordinal_is_compute touches the qgroup cache (state-mutex-protected). */ + cl_data->is_compute = _ordinal_is_compute(device, ordinal) ? 1 : 0; + if (_cl_find(command_list)) { + pthread_mutex_unlock(&_ze_state_mutex); + THAPI_DBGLOG("Command list already registered: %p", command_list); + free(cl_data); + return; + } + _cl_add(cl_data); + pthread_mutex_unlock(&_ze_state_mutex); +} + +/* Wrapper around an injected event we own. Lives either in the per-context + * free pool (between uses) or anchored to one of cl_data->slots[] (in flight). */ struct _ze_event_h { ze_event_handle_t event; - UT_hash_handle hh; ze_event_pool_handle_t event_pool; ze_context_handle_t context; - _ze_event_flags_t flags; - /* to remember events in command lists */ + /* doubly-linked list pointers used by the per-context free pool */ struct _ze_event_h *next, *prev; }; -static struct _ze_event_h *_ze_events = NULL; -static pthread_mutex_t _ze_events_mutex = PTHREAD_MUTEX_INITIALIZER; - -#define FIND_ZE_EVENT(key, val) \ - do { \ - pthread_mutex_lock(&_ze_events_mutex); \ - HASH_FIND_PTR(_ze_events, key, val); \ - pthread_mutex_unlock(&_ze_events_mutex); \ - } while (0) - -#define ADD_ZE_EVENT(val) \ - do { \ - pthread_mutex_lock(&_ze_events_mutex); \ - HASH_ADD_PTR(_ze_events, event, val); \ - pthread_mutex_unlock(&_ze_events_mutex); \ - } while (0) - -#define FIND_AND_DEL_ZE_EVENT(key, val) \ - do { \ - pthread_mutex_lock(&_ze_events_mutex); \ - HASH_FIND_PTR(_ze_events, key, val); \ - if (val) { \ - HASH_DEL(_ze_events, val); \ - } \ - pthread_mutex_unlock(&_ze_events_mutex); \ - } while (0) - struct _ze_event_pool_entry { ze_context_handle_t context; UT_hash_handle hh; @@ -154,166 +578,196 @@ struct _ze_event_pool_entry { }; struct _ze_event_pool_entry *_ze_event_pools = NULL; -static pthread_mutex_t _ze_event_pools_mutex = PTHREAD_MUTEX_INITIALIZER; -#define GET_ZE_EVENT(key, val) \ - do { \ - struct _ze_event_pool_entry *pool = NULL; \ - pthread_mutex_lock(&_ze_event_pools_mutex); \ - HASH_FIND_PTR(_ze_event_pools, key, pool); \ - if (pool && pool->events) { \ - val = pool->events; \ - DL_DELETE(pool->events, val); \ - } else \ - val = NULL; \ - pthread_mutex_unlock(&_ze_event_pools_mutex); \ - } while (0) +/* Per-event tracer state, keyed by the user's event handle. Two facts live + * here, both populated around drain and both bound to the event's lifetime, so + * they share one uthash entry (one lookup, one alloc, one eviction): + * + * latest -> the most recent slot whose attr==ev. Resolves happens-before + * edges: when a new Append waits on ev, that slot becomes a pred. + * Set at instantiate; cleared at drain/dispose only if it still + * points at the draining slot (a newer Append may have overwritten + * it — don't clobber that). + * kts -> last kernel-timestamp result we drained for ev. The Append + * prologue swaps the user's signal for our inj, so the user's event + * carries QKT/barrier op timing, not the kernel's. At drain we read + * the real kernel result from the slab and stash it here so the + * user's own zeEventQueryKernelTimestamp can be served kernel + * timing; re-signaling overwrites. + * + * The whole entry is evicted by _on_destroy_event so a recycled handle address + * (the L0 driver reuses freed event addresses) never serves a dead event's + * latest slot (a dangling pred -> UAF) or stale kts. The value stays inline in + * the entry — no per-set heap box. */ +struct _ze_event_state_entry { + ze_event_handle_t ev; /* key */ + struct _ze_slot *latest; + ze_kernel_timestamp_result_t kts; + unsigned char has_kts; + UT_hash_handle hh; +}; +static struct _ze_event_state_entry *_ze_event_state = NULL; -#define PUT_ZE_EVENT(val) \ - do { \ - struct _ze_event_pool_entry *pool = NULL; \ - pthread_mutex_lock(&_ze_event_pools_mutex); \ - HASH_FIND_PTR(_ze_event_pools, &(val->context), pool); \ - if (!pool) { \ - pool = (struct _ze_event_pool_entry *)calloc(1, sizeof(struct _ze_event_pool_entry)); \ - if (!pool) { \ - THAPI_DBGLOG_NO_ARGS("Failed to allocate memory"); \ - pthread_mutex_unlock(&_ze_event_pools_mutex); \ - if (val->event_pool) { \ - if (val->event) \ - ZE_EVENT_DESTROY_PTR(val->event); \ - ZE_EVENT_POOL_DESTROY_PTR(val->event_pool); \ - } \ - free(val); \ - break; \ - } \ - pool->context = val->context; \ - HASH_ADD_PTR(_ze_event_pools, context, pool); \ - } \ - val->flags = 0; \ - ZE_EVENT_HOST_RESET_PTR(val->event); \ - DL_PREPEND(pool->events, val); \ - pthread_mutex_unlock(&_ze_event_pools_mutex); \ - } while (0) +/* Find-or-create the entry for ev. NULL only on ev==NULL or OOM. */ +static struct _ze_event_state_entry *_event_state_get_or_add(ze_event_handle_t ev) { + if (!ev) + return NULL; + struct _ze_event_state_entry *e = NULL; + HASH_FIND_PTR(_ze_event_state, &ev, e); + if (!e) { + e = (struct _ze_event_state_entry *)calloc(1, sizeof(*e)); + if (!e) + return NULL; + e->ev = ev; + HASH_ADD_PTR(_ze_event_state, ev, e); + } + return e; +} -struct _ze_event_h *_ze_event_wrappers = NULL; -static pthread_mutex_t _ze_event_wrappers_mutex = PTHREAD_MUTEX_INITIALIZER; +/* Drop the entry if it carries nothing worth keeping (no latest slot, no + * stashed kts) — keeps the map bounded as facts are cleared. */ +static inline void _event_state_gc(struct _ze_event_state_entry *e) { + if (e && !e->latest && !e->has_kts) { + HASH_DEL(_ze_event_state, e); + free(e); + } +} -#define GET_ZE_EVENT_WRAPPER(val) \ - do { \ - pthread_mutex_lock(&_ze_event_wrappers_mutex); \ - if (_ze_event_wrappers) { \ - val = _ze_event_wrappers; \ - DL_DELETE(_ze_event_wrappers, val); \ - } else { \ - val = calloc(1, sizeof(struct _ze_event_h)); \ - } \ - pthread_mutex_unlock(&_ze_event_wrappers_mutex); \ - } while (0) +static inline struct _ze_slot *_event_latest_get(ze_event_handle_t ev) { + struct _ze_event_state_entry *e = NULL; + HASH_FIND_PTR(_ze_event_state, &ev, e); + return e ? e->latest : NULL; +} -#define PUT_ZE_EVENT_WRAPPER(val) \ - do { \ - memset(val, 0, sizeof(struct _ze_event_h)); \ - pthread_mutex_lock(&_ze_event_wrappers_mutex); \ - DL_PREPEND(_ze_event_wrappers, val); \ - pthread_mutex_unlock(&_ze_event_wrappers_mutex); \ - } while (0) +static inline void _event_latest_set(ze_event_handle_t ev, struct _ze_slot *slot) { + struct _ze_event_state_entry *e = _event_state_get_or_add(ev); + if (e) + e->latest = slot; +} -/* Snapshot context + immediate-flag from cmdlist into the event wrapper. - * The immediate flag is read at register time (not at _on_reset_event - * time) because by reset time the cmdlist may already be destroyed and - * zeCommandListIsImmediate would dereference a freed handle. */ -static inline void _tag_event_from_cl(struct _ze_event_h *_ze_event, - ze_command_list_handle_t command_list) { - ze_context_handle_t context = NULL; - ze_result_t res = ZE_COMMAND_LIST_GET_CONTEXT_HANDLE_PTR(command_list, &context); - if (res == ZE_RESULT_SUCCESS && context) - _ze_event->context = context; - else - THAPI_DBGLOG("zeCommandListGetContextHandle failed with %d for command list: %p", res, - command_list); - - ze_bool_t is_immediate = 0; - if (ZE_COMMAND_LIST_IS_IMMEDIATE_PTR(command_list, &is_immediate) == ZE_RESULT_SUCCESS && - is_immediate) - _ze_event->flags |= _ZE_IMMEDIATE_CMD; -} - -/* Append an event wrapper we own to its cmdlist's events list, under the - * cl-hash lock (the FIND_AND_DEL/ADD pattern guards cl_data against a - * concurrent free in _on_destroy_command_list). */ -static inline void _attach_event_to_cl(struct _ze_event_h *_ze_event, - ze_command_list_handle_t command_list) { - struct _ze_command_list_obj_data *cl_data = NULL; - FIND_AND_DEL_ZE_CL(&command_list, cl_data); - if (!cl_data) { - THAPI_DBGLOG("Could not get command list associated to event: %p", _ze_event->event); +/* Clear latest iff it still points at `slot` (a newer Append may own it now). */ +static inline void _event_latest_clear_if(ze_event_handle_t ev, struct _ze_slot *slot) { + if (!ev) return; + struct _ze_event_state_entry *e = NULL; + HASH_FIND_PTR(_ze_event_state, &ev, e); + if (e && e->latest == slot) { + e->latest = NULL; + _event_state_gc(e); } - DL_APPEND(cl_data->events, _ze_event); - ADD_ZE_CL(cl_data); } -/* Register an injected (tracer-owned) event. Caller has already populated - * _ze_event->event and _ze_event->event_pool via _get_profiling_event. */ -static inline void _register_our_event(struct _ze_event_h *_ze_event, - ze_command_list_handle_t command_list) { - _tag_event_from_cl(_ze_event, command_list); - _attach_event_to_cl(_ze_event, command_list); - ADD_ZE_EVENT(_ze_event); +static inline void _event_kts_set(ze_event_handle_t ev, ze_kernel_timestamp_result_t val) { + struct _ze_event_state_entry *e = _event_state_get_or_add(ev); + if (e) { + e->kts = val; + e->has_kts = 1; + } } -/* Register a user event (we don't own its lifetime). Look up or create the - * wrapper; users are responsible for reset/destroy, so we don't attach it - * to the cl's events list. */ -static inline void _register_user_event(ze_event_handle_t event, - ze_command_list_handle_t command_list) { - struct _ze_event_h *_ze_event = NULL; - FIND_ZE_EVENT(&event, _ze_event); - if (_ze_event) - return; /* already tracked, nothing more to do */ +static inline int _event_kts_get(ze_event_handle_t ev, ze_kernel_timestamp_result_t *out) { + struct _ze_event_state_entry *e = NULL; + HASH_FIND_PTR(_ze_event_state, &ev, e); + if (!e || !e->has_kts) + return 0; + *out = e->kts; + return 1; +} - GET_ZE_EVENT_WRAPPER(_ze_event); - if (!_ze_event) { - THAPI_DBGLOG("Could not get event wrapper for: %p", event); +/* Evict the whole entry (both facts) — called when the event is destroyed. */ +static inline void _event_state_del(ze_event_handle_t ev) { + if (!ev) return; + struct _ze_event_state_entry *e = NULL; + HASH_FIND_PTR(_ze_event_state, &ev, e); + if (e) { + HASH_DEL(_ze_event_state, e); + free(e); } - /* GET_ZE_EVENT_WRAPPER returns a fully-zeroed wrapper (calloc on first use, - * memset by PUT_ZE_EVENT_WRAPPER on recycle), so event_pool and flags are - * already 0 — only set the fields we actually want non-zero. */ - _ze_event->event = event; - - _tag_event_from_cl(_ze_event, command_list); - ADD_ZE_EVENT(_ze_event); } -static struct _ze_event_h *_get_profiling_event(ze_command_list_handle_t command_list) { - struct _ze_event_h *e_w; - - ze_context_handle_t context = NULL; - ze_result_t res = ZE_COMMAND_LIST_GET_CONTEXT_HANDLE_PTR(command_list, &context); - if (res != ZE_RESULT_SUCCESS || !context) { - THAPI_DBGLOG("zeCommandListGetContextHandle failed with %d, for command list: %p", res, - command_list); +/* Pop one recycled event wrapper from the per-context freelist; NULL + * if none cached (caller falls back to creating a fresh L0 event). */ +static struct _ze_event_h *_get_ze_event(ze_context_handle_t context) { + struct _ze_event_pool_entry *pool = NULL; + HASH_FIND_PTR(_ze_event_pools, &context, pool); + if (!pool || !pool->events) return NULL; + struct _ze_event_h *e = pool->events; + DL_DELETE(pool->events, e); + return e; +} + +/* Return an event wrapper to its per-context freelist. On total failure + * (no bucket can be allocated), destroy the backing L0 objects and free + * the wrapper — we'd rather leak nothing than poison the freelist. */ +static void _put_ze_event(struct _ze_event_h *val) { + _ZE_MUST(ZE_EVENT_HOST_RESET_PTR(val->event)); + struct _ze_event_pool_entry *pool = NULL; + HASH_FIND_PTR(_ze_event_pools, &val->context, pool); + if (!pool) { + pool = (struct _ze_event_pool_entry *)calloc(1, sizeof(*pool)); + if (!pool) { + THAPI_DBGLOG("Failed to allocate memory"); + if (val->event_pool) { + if (val->event) + ZE_EVENT_DESTROY_PTR(val->event); + ZE_EVENT_POOL_DESTROY_PTR(val->event_pool); + } + free(val); + return; + } + pool->context = val->context; + HASH_ADD_PTR(_ze_event_pools, context, pool); } - GET_ZE_EVENT(&context, e_w); + DL_PREPEND(pool->events, val); +} + +struct _ze_event_h *_ze_event_wrappers = NULL; + +/* Get a zeroed event wrapper struct: pop from the global recycle list if + * any, else calloc a fresh one. The wrapper is context-agnostic — only + * the backing L0 event + pool inside it bind to a specific ctx. */ +static struct _ze_event_h *_get_ze_event_wrapper(void) { + struct _ze_event_h *e = _ze_event_wrappers; + if (e) + DL_DELETE(_ze_event_wrappers, e); + else + e = (struct _ze_event_h *)calloc(1, sizeof(*e)); + return e; +} + +/* Return a wrapper struct to the recycle list. Used in two situations: + * 1) wrapper construction failed, no L0 objects ever attached; + * 2) the wrapper's context is being destroyed — caller has already + * arranged for the L0 event/pool inside to be released (or left + * them to die with the context). + * We zero before publishing so a future _get_ze_event_wrapper returns + * something equivalent to a fresh calloc. */ +static void _put_ze_event_wrapper(struct _ze_event_h *val) { + memset(val, 0, sizeof(*val)); + DL_PREPEND(_ze_event_wrappers, val); +} + +/* Caller-supplied ctx avoids a redundant zeCommandListGetContextHandle + * (the prologue already fetched it). L0 event/pool create runs under + * the state mutex; cold path, bounded cost. */ +static struct _ze_event_h *_get_profiling_event(ze_context_handle_t context) { + struct _ze_event_h *e_w = _get_ze_event(context); if (e_w) return e_w; - - GET_ZE_EVENT_WRAPPER(e_w); + e_w = _get_ze_event_wrapper(); if (!e_w) { - THAPI_DBGLOG("Could not create a new event wrapper for command list: %p", command_list); + THAPI_DBGLOG("Could not create a new event wrapper for context: %p", context); return NULL; } ze_event_pool_desc_t desc = { ZE_STRUCTURE_TYPE_EVENT_POOL_DESC, NULL, ZE_EVENT_POOL_FLAG_KERNEL_TIMESTAMP | ZE_EVENT_POOL_FLAG_HOST_VISIBLE, 1}; - res = ZE_EVENT_POOL_CREATE_PTR(context, &desc, 0, NULL, &e_w->event_pool); + ze_result_t res = ZE_EVENT_POOL_CREATE_PTR(context, &desc, 0, NULL, &e_w->event_pool); if (res != ZE_RESULT_SUCCESS) { - THAPI_DBGLOG("zeEventPoolCreate failed with %d, for command list: %p, context: %p", res, - command_list, context); + THAPI_DBGLOG("zeEventPoolCreate failed with %d, for context: %p", res, context); goto cleanup_wrapper; } ze_event_desc_t e_desc = {ZE_STRUCTURE_TYPE_EVENT_DESC, NULL, 0, ZE_EVENT_SCOPE_FLAG_HOST, @@ -328,188 +782,782 @@ static struct _ze_event_h *_get_profiling_event(ze_command_list_handle_t command cleanup_ep: ZE_EVENT_POOL_DESTROY_PTR(e_w->event_pool); cleanup_wrapper: - PUT_ZE_EVENT_WRAPPER(e_w); + _put_ze_event_wrapper(e_w); return NULL; } -static void _profile_event_results(ze_event_handle_t event) { - ze_kernel_timestamp_result_t res = {0}; - ze_result_t status; - ze_result_t timestamp_status; +/* Unlink chunk c from cl_data->chunks and free its slab + struct. + * `free_slab` controls whether to issue zeMemFree on the slab — false when + * the chunk's context is being destroyed (driver reclaims; zeMemFree on a + * doomed ctx is at best racy). Slot-side cleanup (events, waits, preds) + * is the caller's responsibility — this helper only owns the chunk + * envelope and the slab. */ +static void +_cl_chunk_free(struct _ze_command_list_obj_data *cl_data, struct _ze_slab_chunk *c, int free_slab) { + DL_DELETE(cl_data->chunks, c); + if (free_slab && c->slab) + ZE_MEM_FREE_PTR(c->slab_ctx, c->slab); + free(c); +} - if (tracepoint_enabled(lttng_ust_ze_profiling, event_profiling_results)) { - status = ZE_EVENT_QUERY_STATUS_PTR(event); - timestamp_status = ZE_EVENT_QUERY_KERNEL_TIMESTAMP_PTR(event, &res); - do_tracepoint(lttng_ust_ze_profiling, event_profiling_results, event, status, timestamp_status, - res.global.kernelStart, res.global.kernelEnd, res.context.kernelStart, - res.context.kernelEnd); +/* Allocate a new chunk and append it to cl_data->chunks. */ +static struct _ze_slab_chunk *_cl_chunk_alloc(struct _ze_command_list_obj_data *cl_data, + ze_context_handle_t ctx) { + struct _ze_slab_chunk *c = (struct _ze_slab_chunk *)calloc(1, sizeof(*c)); + if (!c) + return NULL; + size_t bytes = (size_t)_ZE_SLAB_CHUNK_SLOTS * sizeof(ze_kernel_timestamp_result_t); + ze_host_mem_alloc_desc_t hd = {ZE_STRUCTURE_TYPE_HOST_MEM_ALLOC_DESC, NULL, 0}; + if (ZE_MEM_ALLOC_HOST_PTR(ctx, &hd, bytes, sizeof(uint64_t), &c->slab) != ZE_RESULT_SUCCESS || + !c->slab) { + free(c); + return NULL; } + memset(c->slab, 0, bytes); + c->slab_ctx = ctx; + DL_APPEND(cl_data->chunks, c); + return c; } -static inline void _on_destroy_event(ze_event_handle_t event) { - struct _ze_event_h *ze_event = NULL; +/* Allocate one new slot at the tail of cl_data->chunks. Grows by one + * chunk for imm cls; regular cls stay at one chunk and return NULL when + * full (their inj events are baked into the closed cl body, so storage + * must keep addressing them via the same (slab, off) pair). */ +static struct _ze_slot *_cl_slot_append(struct _ze_command_list_obj_data *cl_data, + ze_context_handle_t ctx, + struct _ze_event_h *inj, + struct _ze_event_h *shadow_done, + ze_event_handle_t attr, + ze_event_handle_t *waits, + uint32_t n_waits) { + struct _ze_slab_chunk *tail = cl_data->chunks ? cl_data->chunks->prev : NULL; + if (!tail || tail->n_used >= _ZE_SLAB_CHUNK_SLOTS) { + if (tail && !cl_data->is_immediate) { + /* Regular cl is capped at one chunk (inj events are baked into the + * closed cl body, so storage can't move). Past the cap we drop the + * Append's profiling silently — warn once so the data loss is at + * least visible. Called under _ze_state_mutex, so the guard is safe. */ + static int warned = 0; + if (!warned) { + warned = 1; + _THAPI_LOG("warning: regular command list %p exceeded %d profiled " + "Appends in one build; further Appends will not be timed", + (void *)cl_data->ptr, _ZE_SLAB_CHUNK_SLOTS); + } + return NULL; + } + tail = _cl_chunk_alloc(cl_data, ctx); + if (!tail) + return NULL; + } + uint32_t idx = tail->n_used; + struct _ze_slot *s = &tail->slots[idx]; + /* Chunk memory is calloc'd, so all other slot fields are already zero. */ + s->owner = cl_data; + s->chunk = tail; + s->inj = inj; + s->shadow_done = shadow_done; + s->attr = attr; + s->off = (size_t)idx * sizeof(ze_kernel_timestamp_result_t); + if (n_waits) { + s->waits = (ze_event_handle_t *)malloc(n_waits * sizeof(ze_event_handle_t)); + if (s->waits) { + memcpy(s->waits, waits, n_waits * sizeof(ze_event_handle_t)); + s->n_waits = n_waits; + } + } + tail->n_used++; + tail->n_held++; + return s; +} - FIND_AND_DEL_ZE_EVENT(&event, ze_event); - if (!ze_event) { - return; +/* Compute s->preds from s->waits via the global event_latest_signaled + * map, plus the previous live slot on this cl if the cl is in-order. + * Marks s live and publishes s as the new event_latest_signaled[attr]. */ +static void _slot_instantiate(struct _ze_command_list_obj_data *cl_data, struct _ze_slot *s) { + /* Slot must be inert: live=0, preds NULL. Re-instantiating a live slot + * would overwrite preds[] (leaking the prior pred refs) and let the + * in-order pred walk pick up later-appended live siblings as predecessors, + * forming cycles that infinite-loop _slot_drain. */ + _THAPI_ASSERT(!s->live, "slot %p already live (double _slot_instantiate)", (void *)s); + s->live = 1; + uint32_t cap = s->n_waits + 1; /* +1 for in-order prev */ + s->preds = (struct _ze_slot **)calloc(cap, sizeof(struct _ze_slot *)); + s->n_preds = 0; + for (uint32_t i = 0; i < s->n_waits; ++i) { + struct _ze_slot *p = _event_latest_get(s->waits[i]); + if (p && p->live) + s->preds[s->n_preds++] = p; + } + if (cl_data->is_in_order) { + /* Walk chunks newest-to-oldest, slots high-to-low, stop at the first + * live slot strictly before s. Chunks are appended in time order + * (DL_APPEND) and slots within a chunk in time order, so reverse-walk + * yields reverse time order. Skip s itself; s might still have + * live=0 here but the !=s guard is safe and clearer. */ + struct _ze_slab_chunk *c; + struct _ze_slot *prev = NULL; + for (c = cl_data->chunks ? cl_data->chunks->prev : NULL; c && !prev; + c = (c == cl_data->chunks) ? NULL : c->prev) { + for (int32_t i = (int32_t)c->n_used - 1; i >= 0; --i) { + if (&c->slots[i] == s) + continue; + if (c->slots[i].live) { + prev = &c->slots[i]; + break; + } + } + } + if (prev) + s->preds[s->n_preds++] = prev; } + /* Each new pred edge holds a ref on its target. */ + for (uint32_t i = 0; i < s->n_preds; ++i) + s->preds[i]->refs++; + if (s->attr) + _event_latest_set(s->attr, s); +} - _profile_event_results(event); - PUT_ZE_EVENT_WRAPPER(ze_event); +/* Publish a fresh slot: shadow path appends a Query on the per-(ctx,device) + * shadow cl; inline path is a no-op here (its QKT is baked into the user cl + * body at Append). Then instantiate in the dep graph. `s->shadow_done` is + * the single source of truth for "shadow vs inline" — no is_compute branch + * at the call site. */ +static void _slot_publish(struct _ze_command_list_obj_data *cl_data, + struct _ze_slot *s, + struct _ze_shadow_cl *sh) { + if (s->shadow_done) { + _THAPI_ASSERT(sh, "shadow-path slot needs a shadow cl"); + _shadow_append_query(sh, s->inj->event, s->chunk->slab, &s->off, s->shadow_done->event); + s->sh = sh; + } + _slot_instantiate(cl_data, s); } -/* Caller already holds the wrapper (e.g. iterating cl_data->events) and - * has removed it from any per-cl list. Drops it from the global events - * hash, optionally emits its timestamp tracepoint, and recycles. */ -static inline void _unregister_ze_event(struct _ze_event_h *ze_event, int get_results) { - struct _ze_event_h *evicted = NULL; - FIND_AND_DEL_ZE_EVENT(&ze_event->event, evicted); - /* evicted should be == ze_event; if not, our hash bookkeeping is corrupt. */ +/* INLINE path: bake the QKT into the user cl body (wait=inj, sig=user_signal). + * Fires when Appended for immediate cls and on every Execute for regular cls + * (it is now part of the cl body). The QKT signaling user_signal IS the + * user_signal chain — no separate barrier needed. */ +static void _append_inline_query(ze_command_list_handle_t command_list, + struct _ze_slot *s, + ze_event_handle_t inj_event, + ze_event_handle_t user_signal) { + _ZE_MUST(ZE_COMMAND_LIST_APPEND_QUERY_KERNEL_TIMESTAMPS_PTR( + command_list, 1, &inj_event, s->chunk->slab, &s->off, user_signal, 1, &inj_event)); +} - if (get_results) - _profile_event_results(ze_event->event); - if (ze_event->event_pool) - PUT_ZE_EVENT(ze_event); - else - PUT_ZE_EVENT_WRAPPER(ze_event); +/* Chain the user's signal event off our inj on the user cl: the prologue + * swapped user_signal for inj, so without this the user's Sync(user_signal) + * would hang forever. No-op (returns 0) when the user passed no signal; + * returns 1 when the barrier was appended. Mutex-agnostic — it issues an + * L0 Append on the user cl and touches no tracer state, so it is correct + * both inside the critical section (shadow path) and outside it (the + * failure-path compensation). Aborts on L0 failure (a silent hang is worse). */ +static int _chain_user_signal(ze_command_list_handle_t command_list, + ze_event_handle_t inj_event, + ze_event_handle_t user_signal) { + if (!user_signal) + return 0; + _ZE_MUST(ZE_COMMAND_LIST_APPEND_BARRIER_PTR(command_list, user_signal, 1, &inj_event)); + return 1; } -static inline void _on_reset_event(ze_event_handle_t event) { - struct _ze_event_h *ze_event = NULL; +/* Roll back the slot just handed out by _cl_slot_append. We were the last to + * touch the tail chunk and hold _ze_state_mutex, so decrementing n_used/n_held + * and zeroing the slot is safe; if the chunk was freshly allocated only for + * this Append (n_used now 0), free it back so a slot-append failure doesn't + * leak a chunk. */ +static void _slot_append_rollback(struct _ze_command_list_obj_data *cl_data, struct _ze_slot *s) { + free(s->waits); + struct _ze_slab_chunk *c = s->chunk; + c->n_used--; + c->n_held--; + memset(s, 0, sizeof(*s)); + if (c->n_used == 0) + _cl_chunk_free(cl_data, c, /*free_slab=*/1); +} - FIND_AND_DEL_ZE_EVENT(&event, ze_event); - if (!ze_event) { - THAPI_DBGLOG("Could not find event: %p", event); +/* Append-time hook from profiling_epilogue. The prologue swapped user's + * hSignalEvent for inj->event; user_signal is the original (possibly NULL), + * user_waits is the user's wait list, ctx is the cl's context (fetched + * once in the prologue, threaded in). Forks on cl_data->is_compute to + * pick the QKT placement — see "QKT placement" in the file header. */ +static void _universal_record_append(ze_command_list_handle_t command_list, + ze_context_handle_t ctx, + struct _ze_event_h *inj, + ze_event_handle_t user_signal, + ze_event_handle_t *user_waits, + uint32_t user_n_waits) { + if (!inj || !ctx) return; + struct _ze_event_h *shadow_done = NULL; + struct _ze_slot *s = NULL; + int barrier_chained = 0; + + inj->context = ctx; + + pthread_mutex_lock(&_ze_state_mutex); + struct _ze_command_list_obj_data *cl_data = _cl_find(command_list); + if (!cl_data) + goto fail_locked; + int inline_path = cl_data->is_compute; + + /* Shadow path needs a fence event (Query lives on the shadow cl; + * drain host-syncs on it). Inline path uses user_signal as the fence + * via the dep graph, no extra event needed. */ + if (!inline_path) { + shadow_done = _get_profiling_event(ctx); + if (!shadow_done) + goto fail_locked; + shadow_done->context = ctx; } - _profile_event_results(event); + /* Publish the cl->ctx mapping. _on_execute_one_cl reads it directly + * (no fallback fetch) when resolving the shadow cl, and + * _on_destroy_context's per-cl sweep matches against it. */ + cl_data->cached_context = ctx; - if (!(ze_event->flags & _ZE_IMMEDIATE_CMD)) - ADD_ZE_EVENT(ze_event); - else - PUT_ZE_EVENT_WRAPPER(ze_event); + s = _cl_slot_append(cl_data, ctx, inj, shadow_done, user_signal, user_waits, user_n_waits); + if (!s) + goto fail_locked; + + if (inline_path) { + _append_inline_query(command_list, s, inj->event, user_signal); + barrier_chained = 1; /* user_signal chained via the QKT itself */ + _slot_instantiate(cl_data, s); + pthread_mutex_unlock(&_ze_state_mutex); + return; + } + + /* Shadow path: chain user_signal off inj on the user cl, then place + * the Query on the shadow cl (immediate cls only — regular cls defer + * to the Execute epilogue, see _on_execute_one_cl). */ + barrier_chained = _chain_user_signal(command_list, inj->event, user_signal); + if (cl_data->is_immediate) { + ze_device_handle_t dev = NULL; + _ZE_MUST(ZE_COMMAND_LIST_GET_DEVICE_HANDLE_PTR(command_list, &dev)); + struct _ze_shadow_cl *sh = _get_shadow_cl(ctx, dev); + if (!sh) + goto fail_locked; + _slot_publish(cl_data, s, sh); + } + pthread_mutex_unlock(&_ze_state_mutex); + return; + +fail_locked: + if (s) + _slot_append_rollback(cl_data, s); + if (shadow_done) + _put_ze_event(shadow_done); + _put_ze_event(inj); + pthread_mutex_unlock(&_ze_state_mutex); + /* Compensate outside the state mutex: if we bailed before chaining + * user_signal off inj, do it now or the user's Sync(user_signal) hangs. */ + if (!barrier_chained) + _chain_user_signal(command_list, inj->event, user_signal); } -static inline void _dump_and_reset_our_event(ze_event_handle_t event) { - struct _ze_event_h *ze_event = NULL; +/* Dispose the per-slot resources shared by every teardown path: the inj and + * shadow_done events, the waits[] copy, the preds[] array, and the slot's + * entry in event_latest_signaled. The event-disposal target differs by caller: + * _ZE_DISPOSE_POOL -> _put_ze_event (ctx alive: events recycle to the pool) + * _ZE_DISPOSE_WRAPPER -> _put_ze_event_wrapper (ctx dying: only recycle the + * wrapper struct; the L0 event/pool die with the ctx) + * Deliberately does NOT touch chunk accounting (n_held / n_pinned), refs, + * owner, or live — those are caller-specific and stay at the call site. + * Every field is nulled so the call is idempotent (safe to re-run on a slot + * whose preds/latest-signaled were already cleared during drain). */ +enum _ze_slot_dispose_mode { _ZE_DISPOSE_POOL, _ZE_DISPOSE_WRAPPER }; +static void _slot_dispose_resources(struct _ze_slot *s, enum _ze_slot_dispose_mode mode) { + if (s->inj) { + if (mode == _ZE_DISPOSE_WRAPPER) + _put_ze_event_wrapper(s->inj); + else + _put_ze_event(s->inj); + s->inj = NULL; + } + if (s->shadow_done) { + if (mode == _ZE_DISPOSE_WRAPPER) + _put_ze_event_wrapper(s->shadow_done); + else + _put_ze_event(s->shadow_done); + s->shadow_done = NULL; + } + free(s->waits); + s->waits = NULL; + s->n_waits = 0; + free(s->preds); + s->preds = NULL; + s->n_preds = 0; + _event_latest_clear_if(s->attr, s); + s->attr = NULL; +} - FIND_AND_DEL_ZE_EVENT(&event, ze_event); - if (!ze_event) { - THAPI_DBGLOG("Could not find event: %p", event); +/* Reclaim a slot: PUT events back to the per-context pool, free waits, + * decrement chunk n_held; if the chunk hits 0 AND isn't the active + * tail, unlink and free it. Regular cls are skipped (their inj is + * baked into the cl body — reclaim happens at cl destroy instead). */ +static void _slot_release(struct _ze_slot *s) { + if (!s) + return; + /* Detached slot: its owning cl was torn down (reset/destroy) while this + * slot was still a pred of a live slot elsewhere. Its resources were freed + * at reclaim and owner was nulled; the chunk struct was kept alive only to + * keep this slot's refs addressable. We are the downstream drain dropping + * the last ref — drop the chunk's pin and free the bare struct at zero. */ + if (!s->owner && s->chunk && s->chunk->n_pinned) { + struct _ze_slab_chunk *c = s->chunk; + if (--c->n_pinned == 0) + free(c); return; } + if (!s->owner || !s->owner->is_immediate) + return; + /* Reached only from _slot_drain, which already freed s->preds and cleared + * event_latest_signaled[s->attr]; the primitive re-running those is a no-op + * (free(NULL); _clear_if on a missing/overwritten key does nothing). */ + _slot_dispose_resources(s, _ZE_DISPOSE_POOL); + + struct _ze_slab_chunk *c = s->chunk; + struct _ze_command_list_obj_data *cl = s->owner; + if (!c) + return; + c->n_held--; + if (c->n_held == 0 && c != cl->chunks->prev) + _cl_chunk_free(cl, c, /*free_slab=*/1); +} - _profile_event_results(event); - ZE_EVENT_HOST_RESET_PTR(event); - ADD_ZE_EVENT(ze_event); +/* Drain one slot. Recurses on its preds, emits the slot's tracepoint, + * drops one ref on each pred (releasing fully-drained-and-unreferenced + * preds), then releases s if its own refs hit 0. Safe to call on an + * already-drained (live=0) slot. Slab read uses s->chunk->slab — preds + * may live in another cl, so we can't use the caller's slab. + * + * No cycle guard: preds come from in-order prev (strictly earlier slot + * in the same cl, DAG) and from event_latest_signaled[wait_event] (a + * slot published BEFORE us). A cycle would need user-declared mutual + * waits, which L0 itself deadlocks on. */ +static void _slot_drain(struct _ze_slot *s) { + if (!s || !s->live) + return; + for (uint32_t i = 0; i < s->n_preds; ++i) + _slot_drain(s->preds[i]); + s->live = 0; + /* Shadow-path only: block until the Query has fired, then reset + * shadow_done so the next Execute round starts with a clean event. + * The user's own sync doesn't cover the Query because it runs on the + * shadow cl. Inline-path slots have shadow_done==NULL — their QKT + * lives in the user cl body and the dep-graph walk that brought us + * here already implies it has run. */ + if (s->shadow_done && s->shadow_done->event) { + _ZE_MUST(ZE_EVENT_HOST_SYNCHRONIZE_PTR(s->shadow_done->event, UINT64_MAX)); + _ZE_MUST(ZE_EVENT_HOST_RESET_PTR(s->shadow_done->event)); + /* QKT completed device-side. Drop the live ref; if nothing else on + * this shadow cl is in flight, Reset it: the L0 driver leaks ~10 KB + * per AppendQueryKernelTimestamps and only reclaims at Reset/Destroy. */ + if (s->sh) { + s->sh->live_queries--; + if (s->sh->live_queries == 0) + _ZE_MUST(ZE_COMMAND_LIST_RESET_PTR(s->sh->cl)); + } + } + ze_event_handle_t attr = s->attr ? s->attr : (s->inj ? s->inj->event : NULL); + if (s->chunk && s->chunk->slab && attr) { + ze_kernel_timestamp_result_t r = + *(ze_kernel_timestamp_result_t *)((char *)s->chunk->slab + s->off); + /* Stash the kernel result under the user's own event so the user's + * zeEventQueryKernelTimestamp returns kernel timing, not the QKT/barrier + * op timing their event actually carries (we swapped it for inj). Only + * when the user supplied an event (s->attr); inj is ours, not queryable. */ + if (s->attr) + _event_kts_set(s->attr, r); + if (tracepoint_enabled(lttng_ust_ze_profiling, event_profiling_results)) + do_tracepoint(lttng_ust_ze_profiling, event_profiling_results, attr, ZE_RESULT_SUCCESS, + ZE_RESULT_SUCCESS, r.global.kernelStart, r.global.kernelEnd, + r.context.kernelStart, r.context.kernelEnd); + } + _event_latest_clear_if(s->attr, s); + /* Drop refs on preds; release any that hit 0 and are already drained. */ + for (uint32_t i = 0; i < s->n_preds; ++i) { + struct _ze_slot *p = s->preds[i]; + if (--p->refs == 0 && !p->live) + _slot_release(p); + } + free(s->preds); + s->preds = NULL; + s->n_preds = 0; + if (s->refs == 0) + _slot_release(s); } -/* Tear down a wrapper: optionally emit its timestamp tracepoint, then - * destroy the injected event+pool if we own them, then recycle the - * wrapper. Caller must have already removed it from any list/hash that - * references it. */ -static inline void _dispose_event_wrapper(struct _ze_event_h *ze_event, int do_dump) { - if (do_dump && ze_event->event) - _profile_event_results(ze_event->event); - if (ze_event->event_pool) { - if (ze_event->event) - ZE_EVENT_DESTROY_PTR(ze_event->event); - ZE_EVENT_POOL_DESTROY_PTR(ze_event->event_pool); +/* Drain every live slot in a cl (walk chunks oldest-to-newest, slots + * low-to-high — natural time order for emission). */ +static void _cl_drain(struct _ze_command_list_obj_data *cl_data) { + struct _ze_slab_chunk *c, *tmp; + DL_FOREACH_SAFE (cl_data->chunks, c, tmp) { + /* Bump refcount during traversal so the last _slot_drain doesn't + * free c out from under the inner loop. Drop after, free here. */ + c->n_held++; + for (uint32_t i = 0; i < c->n_used; ++i) + _slot_drain(&c->slots[i]); + c->n_held--; + if (c->n_held == 0 && c != cl_data->chunks->prev) + _cl_chunk_free(cl_data, c, /*free_slab=*/1); } - PUT_ZE_EVENT_WRAPPER(ze_event); + _cl_index_clear(cl_data); + cl_data->in_flight_q = NULL; + cl_data->in_flight_fence = NULL; +} + +static void _cl_data_reset(struct _ze_command_list_obj_data *cl_data); /* fwd */ + +/* 1 if any slot in the cl is still in flight (instantiated, not yet drained). */ +static int _cl_any_live(struct _ze_command_list_obj_data *cl_data) { + _ZE_FOREACH_SLOT (cl_data, s) + if (s->live) + return 1; + return 0; +} + +/* Immediate cls only: once every slot in the cl is drained, raw-Reset the + * user's cl so the L0 driver reclaims its per-QKT storage (it accumulates + * otherwise on a long-lived reused immediate cl — see bench/mem_persistent_cl), + * then reclaim our own slot bookkeeping (the baked state is gone after the + * driver reset, exactly like a user zeCommandListReset on a regular cl). + * Raw *_PTR = untraced; safe only when no slot is still live (no in-flight + * work). Called at the tail of every sync-drain path that can touch an imm cl. */ +static void _imm_reset_if_drained(struct _ze_command_list_obj_data *cl_data) { + if (!cl_data || !cl_data->is_immediate || _cl_any_live(cl_data)) + return; + ZE_COMMAND_LIST_RESET_PTR((ze_command_list_handle_t)cl_data->ptr); + _cl_data_reset(cl_data); } -static void _event_cleanup() { - struct _ze_event_h *ze_event = NULL; - struct _ze_event_h *tmp = NULL; - HASH_ITER(hh, _ze_events, ze_event, tmp) { - HASH_DEL(_ze_events, ze_event); - _dispose_event_wrapper(ze_event, 1); +/* Reclaim one chunk during cl teardown (reset or single-cl destroy, ctx + * alive). Releases every slot's resources (events to pool, waits, preds, + * clears latest-signaled), then either frees the chunk or — if any slot is + * still referenced as a pred by a live slot in ANOTHER cl (refs>0) — DETACHES + * it: unlink from cl_data->chunks, null each slot's owner, and keep the bare + * struct alive with n_pinned = #referenced slots. The downstream drains that + * drop those refs free the struct (see _slot_release's detached branch). + * Without this, freeing the chunk here would dangle the referrers' preds[]. */ +static void _cl_chunk_reclaim(struct _ze_command_list_obj_data *cl_data, struct _ze_slab_chunk *c) { + uint32_t pinned = 0; + for (uint32_t i = 0; i < c->n_used; ++i) { + struct _ze_slot *s = &c->slots[i]; + _slot_dispose_resources(s, _ZE_DISPOSE_POOL); + if (s->refs) + pinned++; + } + if (pinned == 0) { + _cl_chunk_free(cl_data, c, /*free_slab=*/1); + return; + } + /* Detach: keep the struct alive for the surviving referenced slots. */ + DL_DELETE(cl_data->chunks, c); + if (c->slab) { + ZE_MEM_FREE_PTR(c->slab_ctx, c->slab); + c->slab = NULL; } + for (uint32_t i = 0; i < c->n_used; ++i) + c->slots[i].owner = NULL; + c->n_pinned = pinned; } -static void _on_destroy_context(ze_context_handle_t context) { - struct _ze_event_h *ze_event = NULL; - struct _ze_event_h *tmp = NULL; - pthread_mutex_lock(&_ze_events_mutex); - HASH_ITER(hh, _ze_events, ze_event, tmp) { - if (ze_event->context == context) { - HASH_DEL(_ze_events, ze_event); - _dispose_event_wrapper(ze_event, 1); - } +/* Reclaim all of a regular cl's slot state, keeping cl_data registered and + * empty for reuse. Used by the zeCommandListReset hook. */ +static void _cl_data_reset(struct _ze_command_list_obj_data *cl_data) { + struct _ze_slab_chunk *c, *tmp; + DL_FOREACH_SAFE (cl_data->chunks, c, tmp) + _cl_chunk_reclaim(cl_data, c); + _cl_index_clear(cl_data); + cl_data->in_flight_q = NULL; + cl_data->in_flight_fence = NULL; +} + +/* Release everything cl_data owns and free cl_data itself. Caller has + * already removed cl_data from _ze_cls (single-cl: _cl_find_and_del; + * per-ctx sweep: HASH_DEL inside the iter). When ctx is dying we just + * recycle wrapper structs (the L0 event/pool will be destroyed in + * _on_destroy_context step 3) and skip zeMemFree on the slab (the + * driver reclaims, and zeMemFree on a doomed ctx is racy); no slot can + * outlive the ctx, so no detach is needed. When the ctx is alive a slot + * may still be referenced cross-cl, so we reclaim per-chunk (detaching + * referenced chunks) just like reset. */ +static void _cl_data_destroy(struct _ze_command_list_obj_data *cl_data, int ctx_dying) { + struct _ze_slab_chunk *c, *tmp; + /* Unlink from the in-flight indexes before the struct is freed, or a later + * queue/fence sync would walk a dangling cl. (When ctx_dying the whole index + * is torn down separately, but unlinking here is still correct and cheap.) */ + _cl_index_clear(cl_data); + if (!ctx_dying) { + DL_FOREACH_SAFE (cl_data->chunks, c, tmp) + _cl_chunk_reclaim(cl_data, c); + free(cl_data); + return; } - pthread_mutex_unlock(&_ze_events_mutex); - pthread_mutex_lock(&_ze_event_pools_mutex); - struct _ze_event_pool_entry *pool = NULL; - HASH_FIND_PTR(_ze_event_pools, &context, pool); - if (pool) { - HASH_DEL(_ze_event_pools, pool); - struct _ze_event_h *elt = NULL, *tmp = NULL; - DL_FOREACH_SAFE(pool->events, elt, tmp) { - DL_DELETE(pool->events, elt); - /* Wrapper is in the free list — its event was already dumped+reset - * by whoever recycled it. Don't dump again, just tear down. */ - _dispose_event_wrapper(elt, 0); - } - free(pool); + DL_FOREACH_SAFE (cl_data->chunks, c, tmp) { + for (uint32_t i = 0; i < c->n_used; ++i) + _slot_dispose_resources(&c->slots[i], _ZE_DISPOSE_WRAPPER); + _cl_chunk_free(cl_data, c, /*free_slab=*/0); } - pthread_mutex_unlock(&_ze_event_pools_mutex); + free(cl_data); +} + +/* zeCommandListDestroy epilogue. Per L0 spec the device is no longer + * referencing the cl, so we don't drain — just release our state. + * Regular cls recycle inj here (cl body is about to die anyway); + * immediate cls' slots are typically already released at drain. */ +static void _on_destroy_command_list(ze_command_list_handle_t command_list) { + pthread_mutex_lock(&_ze_state_mutex); + struct _ze_command_list_obj_data *cl_data = _cl_find_and_del(command_list); + if (cl_data) + _cl_data_destroy(cl_data, /*ctx_dying=*/0); + pthread_mutex_unlock(&_ze_state_mutex); } +/* zeCommandListReset epilogue. The L0 spec requires the user to have + * synchronized before Reset, so our slots are drained — but for a REGULAR cl + * "drained" is not "reclaimed": _slot_release is a no-op for regular cls + * (their inj is baked into the cl body, kept for reuse across Executes), so + * the slots linger. Reset wipes that body, so we must reclaim now; otherwise + * the stale slots are re-published on the next Execute (massive over-count) + * and their chunks accumulate (leak). We drain defensively first in case the + * user under-synced, then reclaim. The cl stays registered, empty for reuse. */ static void _on_reset_command_list(ze_command_list_handle_t command_list) { - struct _ze_command_list_obj_data *cl_data = NULL; + pthread_mutex_lock(&_ze_state_mutex); + struct _ze_command_list_obj_data *cl_data = _cl_find(command_list); + if (cl_data) { + _cl_drain(cl_data); + _cl_data_reset(cl_data); + } + pthread_mutex_unlock(&_ze_state_mutex); +} - FIND_AND_DEL_ZE_CL(&command_list, cl_data); - if (!cl_data) { - THAPI_DBGLOG("Could not get command list: %p", command_list); - return; +/* zeContextDestroy prologue. Three sweeps to drop our own L0 objects + * that live inside this ctx; the user's own cls/events are their + * responsibility per the L0 contract. */ +static void _on_destroy_context(ze_context_handle_t hContext) { + /* 1) Drop cls bound to this ctx. */ + pthread_mutex_lock(&_ze_state_mutex); + struct _ze_command_list_obj_data *cl_data = NULL, *cl_tmp = NULL; + HASH_ITER (hh, _ze_cls, cl_data, cl_tmp) { + if (cl_data->cached_context != hContext) + continue; + HASH_DEL(_ze_cls, cl_data); + _cl_data_destroy(cl_data, /*ctx_dying=*/1); + } + + /* 2) Shadow cls keyed by (ctx, device). */ + struct _ze_shadow_cl *sh = NULL, *sh_tmp = NULL; + HASH_ITER (hh, _ze_shadow_cls, sh, sh_tmp) { + if (sh->key.context != hContext) + continue; + HASH_DEL(_ze_shadow_cls, sh); + if (sh->cl) + ZE_COMMAND_LIST_DESTROY_PTR(sh->cl); + free(sh); } - struct _ze_event_h *elt = NULL, *tmp = NULL; - DL_FOREACH_SAFE(cl_data->events, elt, tmp) { - DL_DELETE(cl_data->events, elt); - _unregister_ze_event(elt, cl_data->flags & _ZE_EXECUTED); + + /* 3) Per-ctx event pool freelist. */ + struct _ze_event_pool_entry *pe = NULL; + HASH_FIND_PTR(_ze_event_pools, &hContext, pe); + if (pe) { + HASH_DEL(_ze_event_pools, pe); + struct _ze_event_h *w, *w_tmp; + DL_FOREACH_SAFE (pe->events, w, w_tmp) { + if (w->event) + ZE_EVENT_DESTROY_PTR(w->event); + if (w->event_pool) + ZE_EVENT_POOL_DESTROY_PTR(w->event_pool); + DL_DELETE(pe->events, w); + _put_ze_event_wrapper(w); + } + free(pe); } - cl_data->flags &= ~_ZE_EXECUTED; - ADD_ZE_CL(cl_data); + pthread_mutex_unlock(&_ze_state_mutex); } -static void _on_execute_command_lists(uint32_t numCommandLists, - ze_command_list_handle_t *phCommandLists) { - for (uint32_t i = 0; i < numCommandLists; i++) { - struct _ze_command_list_obj_data *cl_data = NULL; - FIND_AND_DEL_ZE_CL(phCommandLists + i, cl_data); +/* The four user sync APIs all reduce to "drain the slots the synced anchor + * covers". They differ only in how the anchor selects work: + * + * _ZE_SYNC_CL zeCommandListHostSynchronize -> the one named cl + * _ZE_SYNC_QUEUE zeCommandQueueSynchronize -> every cl with in_flight_q == h + * _ZE_SYNC_FENCE zeFenceHostSynchronize -> every cl with in_flight_fence == h + * _ZE_SYNC_EVENT zeEventHostSynchronize -> the slot that last signaled h, + * walking its pred edges + * + * QUEUE/FENCE share one rule: a queue/fence wait completes exactly the cls a + * given Execute submitted, identified by the handle stamped on the cl at + * Execute. CL/EVENT name their target directly. After draining, a fully-drained + * immediate cl is raw-Reset to cap the driver's per-QKT storage leak + * (_imm_reset_if_drained); for the cl/queue/fence anchors _cl_drain already + * cleared in_flight_*, while the event anchor may leave live siblings, so it + * clears in_flight_* only once the cl has no slot left in flight. */ +enum _ze_sync_kind { _ZE_SYNC_CL, _ZE_SYNC_QUEUE, _ZE_SYNC_FENCE, _ZE_SYNC_EVENT }; +static void _on_sync(enum _ze_sync_kind kind, void *h) { + pthread_mutex_lock(&_ze_state_mutex); + if (kind == _ZE_SYNC_EVENT) { + struct _ze_slot *s = _event_latest_get((ze_event_handle_t)h); + if (s && s->owner) { + _slot_drain(s); + if (!_cl_any_live(s->owner)) { + _cl_index_clear(s->owner); + s->owner->in_flight_q = NULL; + s->owner->in_flight_fence = NULL; + _imm_reset_if_drained(s->owner); + } + } + } else if (kind == _ZE_SYNC_CL) { + struct _ze_command_list_obj_data *cl_data = _cl_find((ze_command_list_handle_t)h); if (cl_data) { - /* dump events if they were executed */ - if (cl_data->flags & _ZE_EXECUTED) { - struct _ze_event_h *elt = NULL; - DL_FOREACH(cl_data->events, elt) { _dump_and_reset_our_event(elt->event); } - } else - cl_data->flags |= _ZE_EXECUTED; - ADD_ZE_CL(cl_data); - } else - THAPI_DBGLOG("Could not get command list: %p", phCommandLists[i]); + _cl_drain(cl_data); + _imm_reset_if_drained(cl_data); + } + } else { /* _ZE_SYNC_QUEUE / _ZE_SYNC_FENCE: drain just the indexed cls */ + struct _ze_inflight_bucket *b = NULL; + if (kind == _ZE_SYNC_QUEUE) + HASH_FIND_PTR(_ze_q_index, &h, b); + else + HASH_FIND_PTR(_ze_fence_index, &h, b); + if (b) { + struct _ze_command_list_obj_data *cl_data = NULL, *tmp = NULL; + /* SAFE2 because _cl_drain -> _cl_index_clear unlinks cl_data from this + * very bucket (and may free the bucket on the last unlink). */ + if (kind == _ZE_SYNC_QUEUE) { + DL_FOREACH_SAFE2 (b->cls, cl_data, tmp, q_next) + _cl_drain(cl_data); + } else { + DL_FOREACH_SAFE2 (b->cls, cl_data, tmp, f_next) + _cl_drain(cl_data); + } + } } + pthread_mutex_unlock(&_ze_state_mutex); } -static void _on_destroy_command_list(ze_command_list_handle_t command_list) { - struct _ze_command_list_obj_data *cl_data = NULL; +/* zeEventQueryKernelTimestamp epilogue. If we drained a kernel result for + * this user event, overwrite *dstptr with it: the user's event carries the + * QKT/barrier op timing (we swapped their signal for inj at Append), but the + * caller wants the KERNEL timing, which we stashed at drain. Returns 1 if it + * served a stashed result. */ +static int _on_query_kernel_timestamp(ze_event_handle_t hEvent, + ze_kernel_timestamp_result_t *dstptr) { + if (!hEvent || !dstptr) + return 0; + pthread_mutex_lock(&_ze_state_mutex); + int found = _event_kts_get(hEvent, dstptr); + pthread_mutex_unlock(&_ze_state_mutex); + return found; +} + +/* zeEventDestroy epilogue (success only). The per-event state entry is keyed by + * the event's HANDLE ADDRESS, which the L0 driver recycles: a fresh event + * created after this one is destroyed can land on the same address. Without + * eviction the new event inherits the dead one's entry — + * .kts: a never-signaled event's zeEventQueryKernelTimestamp would be + * served the prior event's stale timing; + * .latest: a wait on the reused address would resolve to a freed slot, a + * use-after-free in the pred walk. + * Evicting the entry at destroy bounds the map to live events and closes the + * recycled-address reads. Gated on a successful destroy by the caller: a failed + * destroy leaves the event (and its address) alive, so its data stays. */ +static void _on_destroy_event(ze_event_handle_t hEvent) { + pthread_mutex_lock(&_ze_state_mutex); + _event_state_del(hEvent); + pthread_mutex_unlock(&_ze_state_mutex); +} - FIND_AND_DEL_ZE_CL(&command_list, cl_data); +/* Execute-epilogue handler for ONE cl. Runs AFTER L0 Execute returned, + * with the user cl in flight. Three phases: + * + * 1) If in_flight_q is set (prior Execute by another thread), + * force-sync that queue and drain before we overwrite it. + * Regression test: inorder_reg_Event_multithreaded_01. + * 2) Publish each not-yet-live slot (_slot_publish): shadow-path slots + * Append a fresh Query on the per-(ctx,device) shadow cl, then every + * slot is instantiated into the dep graph. The Append must run AFTER + * L0 Execute — appending earlier deadlocks if the shadow shares an + * engine with the user cl (tests/bugs/query_on_separate_cl_regular_user_cl). + * Inline-path cls bake the QKT into the cl body at Append, so their + * publish is instantiate-only. + * 3) Stamp in_flight_q = hQueue and in_flight_fence = hFence (the fence + * the user passed to this Execute, or NULL). */ +static void _on_execute_one_cl(ze_command_queue_handle_t hQueue, + ze_fence_handle_t hFence, + ze_command_list_handle_t command_list) { + pthread_mutex_lock(&_ze_state_mutex); + struct _ze_command_list_obj_data *cl_data = _cl_find(command_list); if (!cl_data) { - THAPI_DBGLOG("Could not get command list: %p", command_list); + pthread_mutex_unlock(&_ze_state_mutex); return; } - if (_do_profile) { - struct _ze_event_h *elt = NULL, *tmp = NULL; - DL_FOREACH_SAFE(cl_data->events, elt, tmp) { - DL_DELETE(cl_data->events, elt); - _unregister_ze_event(elt, cl_data->flags & _ZE_EXECUTED); + + if (cl_data->in_flight_q) { + _ZE_MUST(ZE_COMMAND_QUEUE_SYNCHRONIZE_PTR(cl_data->in_flight_q, UINT64_MAX)); + _cl_drain(cl_data); + } + /* Shadow cl is resolved lazily on first shadow-path slot. Inline-only cls + * never trigger the lookup. */ + struct _ze_shadow_cl *sh = NULL; + int sh_resolved = 0; + struct _ze_slab_chunk *c; + DL_FOREACH (cl_data->chunks, c) { + for (uint32_t j = 0; j < c->n_used; ++j) { + struct _ze_slot *slot = &c->slots[j]; + if (!slot->inj) + continue; + /* Already-live slots have nothing left to do this Execute: their + * dep-graph entry from Append-time _slot_instantiate is still valid, + * and (inline path) their QKT is baked into the cl body and re-fires + * automatically. Only fresh / drained slots need work here. */ + if (slot->live) + continue; + if (slot->shadow_done && !sh_resolved) { + /* cached_context was published by _universal_record_append before any + * shadow_done slot could exist, so it's always set here — no need + * for an L0 round-trip to recover it. */ + ze_context_handle_t ctx = cl_data->cached_context; + ze_device_handle_t dev = NULL; + _ZE_MUST(ZE_COMMAND_LIST_GET_DEVICE_HANDLE_PTR(command_list, &dev)); + sh = ctx ? _get_shadow_cl(ctx, dev) : NULL; + sh_resolved = 1; + } + if (slot->shadow_done && !sh) + continue; + _slot_publish(cl_data, slot, sh); } } - free(cl_data); + cl_data->in_flight_q = hQueue; + cl_data->in_flight_fence = hFence; + /* Index this cl under its queue (and fence) so a later queue/fence sync + * drains it without scanning every live cl. The force-sync+drain above + * already unlinked any prior in-flight membership, so no double-link. */ + _cl_index_set(cl_data, hQueue, hFence); + + pthread_mutex_unlock(&_ze_state_mutex); +} + +static void _on_execute_command_lists_epilogue(ze_command_queue_handle_t hQueue, + ze_fence_handle_t hFence, + uint32_t numCommandLists, + ze_command_list_handle_t *phCommandLists) { + for (uint32_t i = 0; i < numCommandLists; ++i) + _on_execute_one_cl(hQueue, hFence, phCommandLists[i]); } +/* ======================================================================== + * Property/info dumping + tracer init + * + * Separate concern from the slot/drain engine above: read device/driver/ + * kernel/memory properties and emit the lttng_ust_ze_properties / _build + * tracepoints, plus one-time loader/symbol init. Self-contained — the + * engine never calls into this section, and the only external callers are + * ze_model.rb hooks (_do_state, _dump_memory_info, + * _dump_command_list_device_timer, _in_loader_init) and gen_ze.rb + * (_init_tracer / _init_tracer_dump). + * ======================================================================== */ + static pthread_once_t _init = PTHREAD_ONCE_INIT; static __thread volatile int _in_init = 0; static volatile unsigned int _in_loader_init = 0; @@ -524,13 +1572,6 @@ static inline int _do_state() { tracepoint_enabled(lttng_ust_ze_properties, memory_info_range); } -static void THAPI_ATTRIBUTE_DESTRUCTOR _lib_cleanup() { - if (_do_cleanup) { - if (_do_profile) - _event_cleanup(); - } -} - static void _dump_driver_subdevice_properties(ze_driver_handle_t hDriver, ze_device_handle_t hDevice) { if (!tracepoint_enabled(lttng_ust_ze_properties, subdevice)) @@ -666,24 +1707,6 @@ static inline void _dump_memory_info(ze_command_list_handle_t hCommandList, cons _dump_memory_info_ctx(hContext, ptr); } -//////////////////////////////////////////// -#define _ZE_ERROR_MSG(NAME, RES) \ - do { \ - fprintf(stderr, "%s() failed at %d(%s): res=%x\n", (NAME), __LINE__, __FILE__, (RES)); \ - } while (0) -#define _ZE_ERROR_MSG_NOTERMINATE(NAME, RES) \ - do { \ - fprintf(stderr, "%s() error at %d(%s): res=%x\n", (NAME), __LINE__, __FILE__, (RES)); \ - } while (0) -#define _ERROR_MSG(MSG) \ - { \ - perror((MSG)) do { \ - { \ - perror((MSG)); \ - fprintf(stderr, "errno=%d at %d(%s)", errno, __LINE__, __FILE__); \ - } \ - while (0) - static void _load_tracer(void) { char *s = NULL; void *handle = NULL; @@ -736,12 +1759,6 @@ static void _load_tracer(void) { s = getenv("LTTNG_UST_ZE_PARANOID_MEMORY_LOCATION"); if (s) _do_paranoid_memory_location = 1; - - _do_cleanup = 1; - -#ifndef THAPI_USE_DESTRUCTORS - atexit(_lib_cleanup); -#endif } static void _load_tracer_dump(void) { diff --git a/backends/ze/ze_model.rb b/backends/ze/ze_model.rb index ec664445..7341d253 100644 --- a/backends/ze/ze_model.rb +++ b/backends/ze/ze_model.rb @@ -139,39 +139,107 @@ def upper_snake_case(str) register_epilogue 'zeCommandListCreate', <flags & ZE_COMMAND_LIST_FLAG_IN_ORDER) ? 1 : 0; + _on_create_command_list(*phCommandList, hDevice, desc->commandQueueGroupOrdinal, + /*immediate=*/0, _io); } } EOF register_epilogue 'zeCommandListCreateImmediate', <flags & ZE_COMMAND_QUEUE_FLAG_IN_ORDER) ? 1 : 0; + _on_create_command_list(*phCommandList, hDevice, altdesc->ordinal, + /*immediate=*/1, _io); } } EOF +# Reset hook: the L0 spec +# (https://oneapi-src.github.io/level-zero-spec/level-zero/latest/core/api.html#zecommandlistreset) +# says the user must have synchronized first, so our slots are drained — but +# for a REGULAR cl "drained" is not "reclaimed" (_slot_release is a no-op for +# regular cls; their inj is baked into the cl body for reuse across Executes). +# Reset wipes that body, so we reclaim the slots/chunks/events now. Without it +# the stale slots are re-published on the next Execute (over-count) and chunks +# leak. The cl stays registered, empty for reuse. register_epilogue 'zeCommandListReset', < 0) { - _on_execute_command_lists(numCommandLists, phCommandLists); - } - } + if (_do_profile && _retval == ZE_RESULT_SUCCESS && numCommandLists > 0 && phCommandLists) + _on_execute_command_lists_epilogue(hCommandQueue, hFence, numCommandLists, phCommandLists); +EOF + +# Sync hooks: walk dependency edges from the synced anchor and drain +# everything reachable. Each sync API has a different anchor. +register_epilogue 'zeCommandQueueSynchronize', < our injected event. +# epilogue: on success, call _universal_record_append which inserts +# a QueryKernelTimestamps(wait=inj, signal=user_sig) into +# the cmdlist and records the slot for drain. +# The event_profiling tracepoint is attributed to the +# user's original signal (or inj when user passed NULL). +# on sync (queue/event/fence/cl-host): drain the slabs. profiling_prologue = lambda { |event_name| <event; + /* Fetched once per profiled Append and threaded to both + * _get_profiling_event (prologue) and _universal_record_append (epilogue) + * so the tracer issues exactly one zeCommandListGetContextHandle per + * Append instead of three. */ + ze_context_handle_t _ctx = NULL; + if (_do_profile) { + if (ZE_COMMAND_LIST_GET_CONTEXT_HANDLE_PTR(hCommandList, &_ctx) == ZE_RESULT_SUCCESS && _ctx) { + pthread_mutex_lock(&_ze_state_mutex); + _ewrapper = _get_profiling_event(_ctx); + pthread_mutex_unlock(&_ze_state_mutex); + if (_ewrapper) + #{event_name} = _ewrapper->event; + } + /* If injection failed, fall through with the user's signal unchanged; + * we won't be able to time this Append, but it still runs. */ } EOF } -profiling_epilogue = lambda { |event_name| +profiling_epilogue = lambda { |_event_name, waits_expr = 'phWaitEvents', n_waits_expr = 'numWaitEvents'| <event; + _universal_record_append(hCommandList, _ctx, _ewrapper, _user_signal, + #{waits_expr}, #{n_waits_expr}); + tracepoint(lttng_ust_ze_profiling, event_profiling, _attr); + } else { + _put_ze_event(_ewrapper); + } } EOF } @@ -319,7 +400,7 @@ def upper_snake_case(str) ['zeCommandListAppendSignalEvent'].each do |c| register_prologue c, profiling_prologue.call('hEvent') - register_epilogue c, profiling_epilogue.call('hEvent') + register_epilogue c, profiling_epilogue.call('hEvent', 'NULL', '0') end # WARNING diff --git a/utils/thapi_log_to_bt_source_component.rb b/utils/thapi_log_to_bt_source_component.rb index 3e16753f..f27fe412 100755 --- a/utils/thapi_log_to_bt_source_component.rb +++ b/utils/thapi_log_to_bt_source_component.rb @@ -146,7 +146,10 @@ def parse_event(model, line, exclude_fields) def parse_log(model, input_path, exclude_fields) File.open(input_path, 'r') do |file| - file.each_line.map do |line| + file.each_line.filter_map do |line| + stripped = line.strip + next if stripped.empty? || stripped.start_with?('#') + parse_event(model, line, exclude_fields) end end From 6130020b3695435303ab6fc298ce498c48f2b77d Mon Sep 17 00:00:00 2001 From: Thomas Applencourt Date: Fri, 19 Jun 2026 21:23:44 +0000 Subject: [PATCH 4/9] 0.15 --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index 1634331b..3ec2098d 100644 --- a/configure.ac +++ b/configure.ac @@ -2,7 +2,7 @@ # Process this file with autoconf to produce a configure script. AC_PREREQ([2.69]) -AC_INIT([thapi],[0.0.14],[bvideau@anl.gov]) +AC_INIT([thapi],[0.0.15],[bvideau@anl.gov]) AC_CONFIG_SRCDIR([backends/opencl/tracer_opencl_helpers.include.c]) AC_CONFIG_HEADERS([utils/config.h]) From dab53cbfb3cd078d87c3aac7980d29e4102fa3af Mon Sep 17 00:00:00 2001 From: Thomas Applencourt Date: Mon, 22 Jun 2026 15:11:43 -0500 Subject: [PATCH 5/9] Always report stderror when failing (#510) --- xprof/xprof.rb.in | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/xprof/xprof.rb.in b/xprof/xprof.rb.in index 1738f3bc..5e0012d9 100755 --- a/xprof/xprof.rb.in +++ b/xprof/xprof.rb.in @@ -88,11 +88,15 @@ def exec(cmd, opts: {}, debug: true, ignore_exit_codes: []) LOGGER.debug { opts } unless opts.empty? stdout_str, stderr_str, status = Open3.capture3(opts, cmd) - LOGGER.warn { stderr_str.strip } unless stderr_str.empty? LOGGER.debug { stdout_str.strip } unless stdout_str.empty? - raise "#{cmd} failed" unless status.success? || ignore_exit_codes.include?(status.exitstatus) + unless status.success? || ignore_exit_codes.include?(status.exitstatus) + msg = +"#{cmd} failed with status #{status.exitstatus}" + msg << "\nstderr:\n#{stderr_str.strip}" unless stderr_str.strip.empty? + raise msg + end + LOGGER.warn { stderr_str.strip } unless stderr_str.empty? stdout_str end From a3f198237a01912305baed402354079376b830f2 Mon Sep 17 00:00:00 2001 From: Thomas Applencourt Date: Tue, 23 Jun 2026 14:29:59 -0500 Subject: [PATCH 6/9] Better support of pre-2.7 ruby (#511) iprof (xprof.rb) uses modern ruby syntax (endless ranges), so an old ruby fails to *parse* the whole file before the version check can run, giving a raw SyntaxError instead of a friendly message. Split iprof into a tiny old-ruby-safe wrapper (xprof_wrapper.rb.in -> bin/iprof) that checks the ruby version, then requires the implementation (xprof.rb, now in DATADIR). The wrapper uses only old, plain syntax, so the version gate runs and prints a friendly message on old ruby. Also add a configure-time ruby >= 2.7 check (single-sourced via RUBY_MIN_VERSION) and a bats regression test. Co-authored-by: Claude Opus 4.8 (1M context) --- configure.ac | 12 +++++++++ integration_tests/old_ruby_frontend.bats | 34 ++++++++++++++++++++++++ xprof/Makefile.am | 11 +++++--- xprof/sync_daemon_fs | 10 +++---- xprof/xprof.rb.in | 12 +++------ xprof/xprof_wrapper.rb.in | 18 +++++++++++++ 6 files changed, 80 insertions(+), 17 deletions(-) create mode 100644 integration_tests/old_ruby_frontend.bats create mode 100755 xprof/xprof_wrapper.rb.in diff --git a/configure.ac b/configure.ac index 3ec2098d..b6268582 100644 --- a/configure.ac +++ b/configure.ac @@ -27,6 +27,17 @@ if test -z "$RUBY"; then AC_CHECK_PROG(RUBY,[ruby],[ruby],[no]) test "$RUBY" == "no" && AC_MSG_ERROR([Required program 'ruby' not found.]) fi +# iprof needs Ruby >= 2.7. It's a runtime dependency, but we assume people use +# the same ruby version at compile and runtime. Will also be used to potentially +# update other parts of the project. +RUBY_MIN_VERSION=2.7.0 +AC_SUBST([RUBY_MIN_VERSION]) +AC_MSG_CHECKING([for ruby >= $RUBY_MIN_VERSION]) +RUBY_VERSION_FOUND=`$RUBY -e 'print RUBY_VERSION'` +AX_COMPARE_VERSION([$RUBY_VERSION_FOUND], [ge], [$RUBY_MIN_VERSION], + [AC_MSG_RESULT([yes])], + [AC_MSG_RESULT([no]) + AC_MSG_ERROR([ruby version $RUBY_VERSION_FOUND is too old, need >= $RUBY_MIN_VERSION])]) if test -z "$ERB"; then AC_CHECK_PROG(ERB,[erb],[erb],[no]) test "$ERB" == "no" && AC_MSG_ERROR([Required program 'erb' not found.]) @@ -155,6 +166,7 @@ AC_CONFIG_FILES([backends/mpi/tracer_mpi.sh], [chmod +x backends/mpi/tracer_mpi. AC_CONFIG_FILES([backends/itt/tracer_itt.sh], [chmod +x backends/itt/tracer_itt.sh]) AC_CONFIG_FILES([utils/babeltrace_thapi], [chmod +x utils/babeltrace_thapi]) AC_CONFIG_FILES([xprof/xprof.rb], [chmod +x xprof/xprof.rb]) +AC_CONFIG_FILES([xprof/iprof:xprof/xprof_wrapper.rb.in], [chmod +x xprof/iprof]) AC_CONFIG_FILES([thapi.pc]) AC_OUTPUT diff --git a/integration_tests/old_ruby_frontend.bats b/integration_tests/old_ruby_frontend.bats new file mode 100644 index 00000000..39f0f23d --- /dev/null +++ b/integration_tests/old_ruby_frontend.bats @@ -0,0 +1,34 @@ +bats_require_minimum_version 1.5.0 + +# The `iprof` launcher must stay parseable by an old Ruby (the system default on +# some machines) so that, on a too-old interpreter, it prints a friendly version +# message instead of a raw SyntaxError. All modern-syntax logic lives in the +# required `xprof.rb`, which is only parsed after the version gate passes. + +setup() { + # ruby CLI need a path + iprof_path=$(command -v iprof) + + # Single source of truth: read the minimal ruby version baked into iprof itself + # (configure.ac may not be present, e.g. CI only ships the install tree). The + # value keeps its quotes, so it drops straight into the ruby below. + min_version=$(awk '/^THAPI_RUBY_MINIMAL_VERSION/ {print $3}' "${iprof_path}") + + # The ruby to test against. Defaults to `ruby`; point THAPI_OLD_RUBY at an old + # interpreter to exercise this on a machine whose default ruby is recent. + old_ruby="${THAPI_OLD_RUBY:-ruby}" + if "${old_ruby}" -e "exit(Gem::Version.new(RUBY_VERSION) >= Gem::Version.new(${min_version}))"; then + skip "${old_ruby} is >= ${min_version}; set THAPI_OLD_RUBY to an older ruby to run this test" + fi +} + +@test "frontend_parses_under_old_ruby" { + run -0 "${old_ruby}" -c "${iprof_path}" +} + +@test "frontend_prints_friendly_message_under_old_ruby" { + run ! "${old_ruby}" "${iprof_path}" --help + [[ "${output}" == *"too old"* ]] + [[ "${output}" != *"SyntaxError"* ]] + [[ "${output}" != *"unexpected"* ]] +} diff --git a/xprof/Makefile.am b/xprof/Makefile.am index affe3316..9777678b 100644 --- a/xprof/Makefile.am +++ b/xprof/Makefile.am @@ -13,6 +13,13 @@ bin_SCRIPTS = \ iprof \ sync_daemon_fs +# `iprof` is the old-Ruby-safe wrapper, generated from xprof_wrapper.rb.in by +# configure (see configure.ac). It requires xprof.rb (the implementation) from +# DATADIR at runtime, the same place as optparse_thapi.rb and version (see +# utils/Makefile.am). +data_DATA = \ + xprof.rb + if FOUND_MPI bin_SCRIPTS += sync_daemon_mpi endif @@ -20,9 +27,6 @@ endif sync_daemon_mpi: $(srcdir)/sync_daemon_mpi.cpp $(MPICXX) $(CXXFLAGS) $(AM_CXXFLAGS) -I$(top_srcdir)/utils/include $< -o $@ -iprof: xprof.rb - cp xprof.rb $@ - xprof_utils.hpp: $(top_srcdir)/utils/xprof_utils.hpp cp $< $@ @@ -227,7 +231,6 @@ EXTRA_DIST = \ sync_daemon_fs CLEANFILES = \ - iprof \ xprof_utils.hpp \ perfetto_pruned.pb.h \ perfetto_pruned.pb.cc \ diff --git a/xprof/sync_daemon_fs b/xprof/sync_daemon_fs index 1166dbfd..d188143f 100755 --- a/xprof/sync_daemon_fs +++ b/xprof/sync_daemon_fs @@ -1,8 +1,9 @@ #!/usr/bin/env ruby -# Cannot use require_relative as iprof is not a rb file -# Load MPITopo and related helpers -load(File.join(__dir__, 'iprof')) +# Load MPITopo and related helpers from the implementation (installed in +# ../share), not the iprof wrapper, which would set $thapi_launch and run the +# whole program instead of just defining helpers. +require_relative File.join('..', 'share', 'xprof') require 'open3' require 'fileutils' @@ -66,12 +67,11 @@ until msg == SyncDaemon::MSG_FINISH raise 'sync_daemon_fs: parent closed socket without sending FINISH' if msg.empty? case msg - when SyncDaemon::MSG_INIT then nil + when SyncDaemon::MSG_INIT, SyncDaemon::MSG_FINISH then nil when SyncDaemon::MSG_LOCAL_BARRIER local_barrier(local_barrier_count.to_s) local_barrier_count += 1 when SyncDaemon::MSG_GLOBAL_BARRIER then global_barrier(global_handle) - when SyncDaemon::MSG_FINISH then nil else warn("sync_daemon_fs: unknown message '#{msg}'") exit(1) diff --git a/xprof/xprof.rb.in b/xprof/xprof.rb.in index 5e0012d9..24907c63 100755 --- a/xprof/xprof.rb.in +++ b/xprof/xprof.rb.in @@ -1,11 +1,6 @@ #!/usr/bin/env ruby -# 2.7 for Lazy in Enumerable. 2.7 was released 25 Dec 2019 -THAPI_RUBY_MINIMAL_VERSION = '2.7.0' -if Gem::Version.new(RUBY_VERSION) < Gem::Version.new(THAPI_RUBY_MINIMAL_VERSION) - warn("Your ruby version #{RUBY_VERSION} is too old. #{THAPI_RUBY_MINIMAL_VERSION} or newer required") - exit(1) -end +# This file doesn't do any ruby version check. All the check are done in `iprof`. # We Cannot use "@ .. @" for libdir, bindir, and datarootdir # as they will appear as bash "${exec_prefix}/lib" @@ -1007,8 +1002,9 @@ def last_trace_saved Dir[File.join(thapi_trace_home, 'thapi*')].max_by { |f| File.mtime(f) } end -# Avoid load problem -if __FILE__ == $PROGRAM_NAME +# Run when launched through the `iprof` wrapper (which sets $thapi_launch before +# requiring this file) or when executed directly. +if $thapi_launch || __FILE__ == $PROGRAM_NAME parser = OptionParserWithDefaultAndValidation.new parser.banner = "Usage: #{File.basename($PROGRAM_NAME)} [options] [--] [command]" diff --git a/xprof/xprof_wrapper.rb.in b/xprof/xprof_wrapper.rb.in new file mode 100755 index 00000000..71f52a76 --- /dev/null +++ b/xprof/xprof_wrapper.rb.in @@ -0,0 +1,18 @@ +#!/usr/bin/env ruby + +# User-facing `iprof` launcher. It MUST stay parseable by old Ruby: Ruby parses +# the whole file before running any of it, so modern syntax here (endless ranges, +# pattern matching, ...) would raise a raw SyntaxError before the version check +# below could run. All real logic lives in xprof.rb, required only after the check +# passes, so users on an old Ruby get the friendly message below instead. + +# Named constant (not inlined) so bats can grep the minimal version out of iprof. +THAPI_RUBY_MINIMAL_VERSION = '@RUBY_MIN_VERSION@' +if Gem::Version.new(RUBY_VERSION) < Gem::Version.new(THAPI_RUBY_MINIMAL_VERSION) + warn("Your ruby version #{RUBY_VERSION} is too old. #{THAPI_RUBY_MINIMAL_VERSION} or newer required") + exit(1) +end + +$LOAD_PATH.unshift(File.join('@prefix@', 'share')) +$thapi_launch = true +require 'xprof' From 69d29c404e418f5ea05545bd0af550953e291fa0 Mon Sep 17 00:00:00 2001 From: Thomas Applencourt Date: Tue, 23 Jun 2026 15:55:12 -0500 Subject: [PATCH 7/9] Fix zes init (#512) --- backends/ze/gen_ze.rb | 5 ++++- backends/ze/ze_model.rb | 6 +++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/backends/ze/gen_ze.rb b/backends/ze/gen_ze.rb index 9e662b95..fc86a307 100644 --- a/backends/ze/gen_ze.rb +++ b/backends/ze/gen_ze.rb @@ -229,7 +229,10 @@ def gen_struct_printer(namespace, types) puts < 'ZE_STRUCTURE_TYPE_IMAGE_MEMORY_EXP_PROPERTIES', From dc4c8137f16259d0bec7b25028e5527160f91ad8 Mon Sep 17 00:00:00 2001 From: Thomas Applencourt Date: Tue, 23 Jun 2026 17:42:12 -0500 Subject: [PATCH 8/9] Make iprof a transparent stdio wrapper (#513) iprof ran the user binary via IO.popen, which connects the child's stdout to a pipe. The child's libc then sees a non-tty and switches to full block-buffering, so its output only appeared once it exited (e.g. all of `iprof -m full ./a.out` arrived in one block instead of streaming). Let the child inherit iprof's own stdout/stderr directly instead, so it sees exactly the descriptors iprof was given and picks the same buffering it would without iprof: line-buffered (streaming) on a terminal, block-buffered on a pipe/file. This also keeps stdout and stderr on their own streams. Add an integration test driving iprof under a PTY (via `script`) to check streaming with a real libc-buffered binary, and one checking stdout/stderr stay separate. The previous test used `bash echo`, which is not libc-buffered and so never exercised the bug. Co-authored-by: Claude Opus 4.8 --- integration_tests/buffering_helper.c | 20 ++++++++++++++++ integration_tests/parallel_execution.bats | 28 +++++++++++++++++------ xprof/xprof.rb.in | 12 +++++----- 3 files changed, 47 insertions(+), 13 deletions(-) create mode 100644 integration_tests/buffering_helper.c diff --git a/integration_tests/buffering_helper.c b/integration_tests/buffering_helper.c new file mode 100644 index 00000000..3327c66f --- /dev/null +++ b/integration_tests/buffering_helper.c @@ -0,0 +1,20 @@ +/* + * Helper for the iprof stdout/stderr streaming tests. + * + * It uses plain stdio (printf/fprintf) with NO explicit fflush, so the C + * library decides the buffering policy from whether the fd is a tty: + * - tty -> line-buffered -> each line is emitted as it is printed + * - pipe/file -> block-buffered -> everything is flushed only at exit + */ +#include +#include + +#define SLEEP_SECONDS 4 + +int main(void) { + printf("THAPI_STREAM_FIRST\n"); + fprintf(stderr, "THAPI_STREAM_STDERR\n"); + sleep(SLEEP_SECONDS); + printf("THAPI_STREAM_SECOND\n"); + return 0; +} diff --git a/integration_tests/parallel_execution.bats b/integration_tests/parallel_execution.bats index 35d413f6..24ae395c 100644 --- a/integration_tests/parallel_execution.bats +++ b/integration_tests/parallel_execution.bats @@ -59,24 +59,38 @@ launch_mpi() { } @test "launch_usr_bin_streams_child_stdout" { - # Verify iprof streams the user binary's stdout line-by-line rather than - # buffering it. The child sleeps 4s between two prints, so under proper - # streaming the two lines reach us ~4s apart; under buffering they arrive - # together at child exit. We assert the gap is at least 2s. + # If iprof's input is not buffered, iprof should not buffer it. We use `script` + # to give iprof a tty (an unbuffered pipe); the helper sleeps 4s between two + # prints, so under streaming the two lines reach us ~4s apart. `script` runs the + # child under a PTY, whose line discipline appends a CR to each '\n', hence the + # trailing `*` in the patterns below. + helper="${BATS_TEST_TMPDIR}/buffering_helper" + gcc "${BATS_TEST_DIRNAME}/buffering_helper.c" -o "${helper}" + ts_first="" ts_second="" while IFS= read -r line; do case "$line" in - THAPI_STREAM_FIRST) ts_first=$(date +%s%N) ;; - THAPI_STREAM_SECOND) + THAPI_STREAM_FIRST*) ts_first=$(date +%s%N) ;; + THAPI_STREAM_SECOND*) ts_second=$(date +%s%N) break ;; esac - done < <(iprof -- bash -c 'echo THAPI_STREAM_FIRST; sleep 4; echo THAPI_STREAM_SECOND' 2>/dev/null) + done < <(script -qec "iprof -- '${helper}'" /dev/null) [ -n "$ts_first" ] [ -n "$ts_second" ] delta_ms=$(((ts_second - ts_first) / 1000000)) echo "delta_ms=$delta_ms" [ "$delta_ms" -gt 2000 ] } + +@test "launch_usr_bin_keeps_stderr_separate" { + # The user binary's stdout and stderr must stay on their respective streams. + helper="${BATS_TEST_TMPDIR}/buffering_helper" + gcc "${BATS_TEST_DIRNAME}/buffering_helper.c" -o "${helper}" + + run --separate-stderr iprof --no-analysis -- "${helper}" + [[ "$output" =~ $'THAPI_STREAM_FIRST\nTHAPI_STREAM_SECOND' ]] + [[ "$stderr" =~ THAPI_STREAM_STDERR ]] +} diff --git a/xprof/xprof.rb.in b/xprof/xprof.rb.in index 24907c63..95af63b0 100755 --- a/xprof/xprof.rb.in +++ b/xprof/xprof.rb.in @@ -26,7 +26,6 @@ require 'open3' require 'fileutils' require 'etc' require 'optparse_thapi' -require 'pty' require 'digest/md5' require 'socket' require 'io/nonblock' @@ -98,17 +97,18 @@ end def launch_usr_bin(env, cmd) LOGGER.info { "Launch_usr_bin #{cmd}" } - # https://docs.ruby-lang.org/en/master/IO.html#method-i-sync - $stdout.sync = true - IO.popen(env, cmd) do |io| - io.each { |line| print line } + # Let the user binary inherit our own stdout/stderr directly. Iprof is a + # transparent wrapper. + begin + pid = spawn(env, *cmd, out: $stdout, err: $stderr) + _, status = Process.wait2(pid) rescue Interrupt # Just continue as usual, do not crash xprof rescue Errno::ENOENT warn("#{__FILE__}: Can't find executable #{cmd.first}") raise Errno::ENOENT end - XprofExitCode.update($?, cmd.join(' ')) + XprofExitCode.update(status, cmd.join(' ')) if status end def exec_store_in_file(cmd, output_file: nil) From cb7d9c41a730ee570629b277604e04b979ac2fe1 Mon Sep 17 00:00:00 2001 From: Thomas Applencourt Date: Wed, 24 Jun 2026 15:21:22 -0500 Subject: [PATCH 9/9] sync_daemon_mpi: give daemon a distinct PMIX_NAMESPACE (#514) The MPI sync daemon shares the traced application's inherited PMIx identity (PMIX_NAMESPACE). Two MPI instances claiming the same (namespace, rank) collide in the node-local SHM bootstrap and deadlock (n=128 --ppn 64) or abort with a truncation error (n=4). Append a fixed suffix to PMIX_NAMESPACE before any MPI call so the daemon's session gets its own WORLD. Verified on 2 nodes with THAPI_SYNC_DAEMON=mpi: fix -> exit 0, full tally; control (fix disabled) -> hang. Default remains fs pending MPICH feedback. Co-authored-by: Thomas Applencourt Co-authored-by: Claude Opus 4.8 --- xprof/sync_daemon_mpi.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/xprof/sync_daemon_mpi.cpp b/xprof/sync_daemon_mpi.cpp index 05a33060..18a06c17 100644 --- a/xprof/sync_daemon_mpi.cpp +++ b/xprof/sync_daemon_mpi.cpp @@ -2,6 +2,7 @@ #include "mpi.h" #include #include +#include using namespace daemon_proto; @@ -125,6 +126,10 @@ int main(int argc, char **argv) { } const int fd = atoi(argv[1]); + // WA for MPI_Session bug when running concurrently with MPI apps. + if (const char *ns = getenv("PMIX_NAMESPACE")) + setenv("PMIX_NAMESPACE", (std::string(ns) + "_thapi_sync_daemon").c_str(), 1); + CHECK_MPI(MPIX_Init_Session(&lib_shandle, &MPI_COMM_WORLD_THAPI)); CHECK_MPI(MPI_Comm_split_type(MPI_COMM_WORLD_THAPI, MPI_COMM_TYPE_SHARED, 0, MPI_INFO_NULL, &MPI_COMM_NODE));