From 07ac8a7f4e663dc6ee9142876b2d5d902e1d687f Mon Sep 17 00:00:00 2001 From: Dongdong Tao Date: Tue, 30 Jun 2026 21:33:16 +0900 Subject: [PATCH 1/8] osdtrace: recover by-value parameter location from the SysV-AMD64 ABI When a traced parameter has no DW_AT_location the parser bailed (translate_param_location returns false, and the caller asserts). GCC drops the location of a forwarded by-value aggregate under -march=x86-64-v3 -- e.g. reqid in ReplicatedBackend::submit_transaction on Ubuntu amd64v3 and CentOS Stream el10 (RHEL 10's x86-64-v3 baseline) -- which aborted osdtrace there. The value is still at its ABI incoming-argument slot, so reconstruct the location from the SysV-AMD64 calling convention: classify the formal parameters (INTEGER vs stack MEMORY), assign INTEGER eightbytes to RDI/RSI/RDX/RCX/R8/R9 and the rest (plus MEMORY-class aggregates) to the stack, and resolve the target's slot via the call-frame CFA (the same machinery DW_OP_fbreg uses). Parameter classes we cannot classify with confidence make the fallback bail, so we never emit a guessed-wrong location. Validated on ceph-r2 (amd64v3, Ceph 20.2.0): the recovered reqid location is identical to the amd64 build's ({reg RSP, offset 0x30, stack}), and a live read at the uprobe yields a valid client reqid. --- src/dwarf_parser.cc | 163 ++++++++++++++++++++++++++++++++++++++++++++ src/dwarf_parser.h | 10 +++ 2 files changed, 173 insertions(+) diff --git a/src/dwarf_parser.cc b/src/dwarf_parser.cc index 4ebc9ae..3f38707 100644 --- a/src/dwarf_parser.cc +++ b/src/dwarf_parser.cc @@ -217,6 +217,160 @@ Dwarf_Attribute *DwarfParser::find_func_frame_base( return fb_attr; } +// --- SysV-AMD64 incoming-argument classification --------------------------- +// +// Used only as a fallback when a parameter's DW_AT_location is missing. We +// reconstruct where the argument lives at function entry from the calling +// convention. Only the INTEGER and (stack) MEMORY classes are handled — the +// classes osdtrace's traced parameters actually use (pointers, references, +// integers/enums, and large by-value aggregates such as osd_reqid_t). Any +// parameter we cannot classify with confidence makes the whole fallback bail, +// so we never emit a guessed-wrong location. +namespace { +enum AbiClass { ABI_INTEGER, ABI_MEMORY, ABI_UNHANDLED }; + +// Peel typedef/const/volatile/restrict down to the underlying type DIE. +bool peel_type(Dwarf_Die *t) { + for (;;) { + switch (dwarf_tag(t)) { + case DW_TAG_typedef: + case DW_TAG_const_type: + case DW_TAG_volatile_type: + case DW_TAG_restrict_type: { + Dwarf_Attribute a; + if (dwarf_attr_integrate(t, DW_AT_type, &a) == NULL) return false; + if (dwarf_formref_die(&a, t) == NULL) return false; + break; + } + default: + return true; + } + } +} + +// Classify a parameter by its type DIE. Returns false if the type cannot be +// resolved (caller treats that as ABI_UNHANDLED / bail). +AbiClass classify_param(Dwarf_Die *param, Dwarf_Word &size) { + Dwarf_Attribute ta; + Dwarf_Die t; + if (dwarf_attr_integrate(param, DW_AT_type, &ta) == NULL) return ABI_UNHANDLED; + if (dwarf_formref_die(&ta, &t) == NULL) return ABI_UNHANDLED; + if (!peel_type(&t)) return ABI_UNHANDLED; + + switch (dwarf_tag(&t)) { + case DW_TAG_pointer_type: + case DW_TAG_reference_type: + case DW_TAG_rvalue_reference_type: + case DW_TAG_enumeration_type: + size = 8; + return ABI_INTEGER; + + case DW_TAG_base_type: { + Dwarf_Attribute ea; + Dwarf_Word enc = 0; + if (dwarf_attr_integrate(&t, DW_AT_encoding, &ea) != NULL) + dwarf_formudata(&ea, &enc); + // Floats go in SSE registers — not handled here. + if (enc == DW_ATE_float || enc == DW_ATE_complex_float) return ABI_UNHANDLED; + Dwarf_Word sz = 0; + if (dwarf_aggregate_size(&t, &sz) != 0 || sz > 8) return ABI_UNHANDLED; + size = sz; + return ABI_INTEGER; + } + + case DW_TAG_structure_type: + case DW_TAG_class_type: + case DW_TAG_union_type: + case DW_TAG_array_type: { + Dwarf_Word sz = 0; + if (dwarf_aggregate_size(&t, &sz) != 0) return ABI_UNHANDLED; + // Aggregates > 16 bytes are MEMORY class (passed on the stack). Smaller + // ones need eightbyte field classification (INTEGER/SSE) — bail. + if (sz > 16) { + size = sz; + return ABI_MEMORY; + } + return ABI_UNHANDLED; + } + + default: + return ABI_UNHANDLED; + } +} +} // namespace + +bool DwarfParser::abi_stack_location(Dwarf_Die *func, Dwarf_Addr pc, + long cfa_off, VarLocation &varloc) { + // Resolve the call-frame CFA at pc into a (reg, offset) pair, then add the + // argument's CFA-relative offset. Identical machinery to DW_OP_fbreg. + Dwarf_Attribute fb_mem; + Dwarf_Attribute *fb = find_func_frame_base(func, &fb_mem); + if (fb == NULL) { + cerr << "abi_stack_location: function has no frame base" << endl; + return false; + } + Dwarf_Op *fb_expr; + size_t fb_len; + if (dwarf_getlocation_addr(fb, pc, &fb_expr, &fb_len, 1) != 1 || fb_len == 0) { + cerr << "abi_stack_location: frame base expr failed" << endl; + return false; + } + if (!translate_expr(fb, fb_expr, pc, varloc)) return false; + varloc.offset += cfa_off; + varloc.stack = true; + return true; +} + +bool DwarfParser::abi_param_location(Dwarf_Die *func, Dwarf_Die &target, + Dwarf_Addr pc, VarLocation &varloc) { + // SysV-AMD64 INTEGER argument registers, in order, as DWARF register + // numbers: RDI=5, RSI=4, RDX=1, RCX=2, R8=8, R9=9. + static const int kIntArgRegs[6] = {5, 4, 1, 2, 8, 9}; + int int_regs_used = 0; + long stack_off = 0; // CFA-relative offset of the next stack argument + Dwarf_Off target_off = dwarf_dieoffset(&target); + + Dwarf_Die child; + if (dwarf_child(func, &child) != 0) { + cerr << "abi_param_location: function has no children" << endl; + return false; + } + do { + if (dwarf_tag(&child) != DW_TAG_formal_parameter) continue; + + Dwarf_Word size = 0; + AbiClass cls = classify_param(&child, size); + if (cls == ABI_UNHANDLED) { + cerr << "abi_param_location: unhandled parameter class; cannot recover " + "location" << endl; + return false; // safety net: never guess + } + + bool is_target = (dwarf_dieoffset(&child) == target_off); + + if (cls == ABI_INTEGER) { + if (int_regs_used < 6) { + if (is_target) { + varloc.reg = kIntArgRegs[int_regs_used]; + varloc.offset = 0; + varloc.stack = false; + return true; + } + ++int_regs_used; + } else { // overflowed to the stack, one eightbyte + if (is_target) return abi_stack_location(func, pc, stack_off, varloc); + stack_off += 8; + } + } else { // ABI_MEMORY: always on the stack, size rounded up to 8 bytes + if (is_target) return abi_stack_location(func, pc, stack_off, varloc); + stack_off += (long)((size + 7) & ~(Dwarf_Word)7); + } + } while (dwarf_siblingof(&child, &child) == 0); + + cerr << "abi_param_location: target parameter not found among formals" << endl; + return false; +} + bool DwarfParser::translate_param_location(Dwarf_Die *func, string symbol, Dwarf_Addr pc, Dwarf_Die &vardie, VarLocation &varloc) { @@ -227,6 +381,15 @@ bool DwarfParser::translate_param_location(Dwarf_Die *func, string symbol, Dwarf_Attribute loc_attr; if (dwarf_attr_integrate(&vardie, DW_AT_location, &loc_attr) == NULL) { + // The compiler emitted no location for this parameter. This happens for + // forwarded by-value aggregates under -march=x86-64-v3 (e.g. reqid in + // ReplicatedBackend::submit_transaction on Ubuntu amd64v3 / CentOS el10): + // the value is still at its ABI incoming-argument slot, so recover it from + // the SysV-AMD64 calling convention instead of giving up. + if (abi_param_location(func, vardie, pc, varloc)) { + cerr << "Recovered ABI incoming location for parameter " << symbol << endl; + return true; + } cerr << "Parameter " << symbol << " has no DW_AT_location" << endl; return false; } diff --git a/src/dwarf_parser.h b/src/dwarf_parser.h index ca9f69b..dd60e4f 100644 --- a/src/dwarf_parser.h +++ b/src/dwarf_parser.h @@ -74,6 +74,16 @@ class DwarfParser { Dwarf_Attribute *find_func_frame_base(Dwarf_Die *, Dwarf_Attribute *); bool translate_param_location(Dwarf_Die *, std::string, Dwarf_Addr, Dwarf_Die &, VarLocation &); + // Recover a parameter's incoming-argument location from the SysV-AMD64 + // calling convention when the DWARF carries no DW_AT_location for it (the + // compiler can drop the location of a forwarded by-value aggregate under + // -march=x86-64-v3, e.g. reqid in ReplicatedBackend::submit_transaction). + bool abi_param_location(Dwarf_Die *func, Dwarf_Die &target, Dwarf_Addr pc, + VarLocation &); + // Resolve the call-frame CFA at pc and add an offset, yielding a stack + // VarLocation (the same machinery DW_OP_fbreg uses). + bool abi_stack_location(Dwarf_Die *func, Dwarf_Addr pc, long cfa_off, + VarLocation &); bool func_entrypc(Dwarf_Die *, Dwarf_Addr *); bool find_prologue(Dwarf_Die *func, Dwarf_Addr &pc); void dwarf_die_type(Dwarf_Die *, Dwarf_Die *); From 391e2ecbecc5ea8867ed06fed3663a31a61dafea Mon Sep 17 00:00:00 2001 From: Dongdong Tao Date: Wed, 1 Jul 2026 16:12:04 +0900 Subject: [PATCH 2/8] files/ubuntu: add Resolute (26.04) Tentacle amd64v3 DWARF JSONs Ubuntu 26.04 ships an opt-in amd64v3 package variant (APT::Architecture-Variants "amd64v3") compiled for -march=x86-64-v3. It keeps the same version string and dpkg architecture as the baseline amd64 build but has different function addresses and a different ELF build-id, so it needs its own DWARF JSON. Add osd + rados DWARF JSON for Ceph 20.2.0-0ubuntu2 amd64v3, generated on an Ubuntu 26.04 amd64v3 host. These depend on the ABI incoming-argument location fallback in the preceding commit: without it, reqid in ReplicatedBackend::submit_transaction (whose location the v3 compiler drops) would abort generation. reqid is recovered to the same slot the amd64 build emits natively ({reg RSP, offset 0x30, stack}). Build-ids: ceph-osd 20bbaa35..., libceph-common.so.2 23210fa6... -- distinct from the amd64 files (0e6c06a0..., 0f51328e...), so embedded-DWARF build-id matching selects the correct variant automatically. --- doc/dwarf-json-files.md | 16 + .../osd-20.2.0-0ubuntu2_amd64v3_dwarf.json | 1338 +++++++++++++++++ .../20.2.0-0ubuntu2_amd64v3_dwarf.json | 388 +++++ 3 files changed, 1742 insertions(+) create mode 100644 files/ubuntu/osdtrace/osd-20.2.0-0ubuntu2_amd64v3_dwarf.json create mode 100644 files/ubuntu/radostrace/20.2.0-0ubuntu2_amd64v3_dwarf.json diff --git a/doc/dwarf-json-files.md b/doc/dwarf-json-files.md index 01dbbf2..07cdd05 100644 --- a/doc/dwarf-json-files.md +++ b/doc/dwarf-json-files.md @@ -75,10 +75,26 @@ Available versions include: - Ubuntu 20.04: Ceph 15.2.17, 17.2.x series - Ubuntu 22.04: Ceph 17.2.x, 19.2.x series - Ubuntu 24.04: Ceph 19.2.x series +- Ubuntu 26.04: Ceph 20.2.x series (Tentacle), including `amd64v3` + architecture-variant builds File naming format: `_dwarf.json` - Example: `17.2.6-0ubuntu0.22.04.2_dwarf.json` +When more than one build of the same package version is checked in — a +different architecture or microarchitecture variant — the filename carries an +architecture suffix to keep the builds distinct. Ubuntu 26.04's `amd64v3` +package variant (`APT::Architecture-Variants "amd64v3"`) keeps the same +version string and `dpkg` architecture as the baseline `amd64` build but is +compiled for `-march=x86-64-v3`, so it has different function addresses and a +different ELF build-id and needs its own DWARF JSON: +- Example: `osd-20.2.0-0ubuntu2_amd64v3_dwarf.json` +- Example: `20.2.0-0ubuntu2_amd64v3_dwarf.json` + +The embedded-DWARF path matches the target by ELF build-id, so it selects the +right variant automatically; the suffix only disambiguates the checked-in +files and the `-i` import path. + ### CentOS Stream Location: `files/centos-stream/{radostrace,osdtrace}/` diff --git a/files/ubuntu/osdtrace/osd-20.2.0-0ubuntu2_amd64v3_dwarf.json b/files/ubuntu/osdtrace/osd-20.2.0-0ubuntu2_amd64v3_dwarf.json new file mode 100644 index 0000000..6801542 --- /dev/null +++ b/files/ubuntu/osdtrace/osd-20.2.0-0ubuntu2_amd64v3_dwarf.json @@ -0,0 +1,1338 @@ +{ + "version": "20.2.0-0ubuntu2", + "arch": "amd64", + "ceph-osd": { + "build_id": "20bbaa35a85fc9fad7534f286a7087f81cc35074", + "func2pc": { + "BlueStore::_do_write": 15689168, + "BlueStore::_txc_apply_kv": 15260720, + "BlueStore::_txc_calc_cost": 15783056, + "BlueStore::_txc_state_proc": 15473408, + "BlueStore::_wctx_finish": 15461328, + "BlueStore::log_latency": 15991728, + "BlueStore::log_latency_fn": 15992512, + "BlueStore::queue_transactions": 15783664, + "ECBackend::submit_transaction": 13522672, + "OSD::dequeue_op": 8814848, + "OSD::enqueue_op": 8886928, + "OpRequest::mark_flag_point": 21325232, + "OpRequest::mark_flag_point_string": 21325648, + "PrimaryLogPG::execute_ctx": 10560928, + "PrimaryLogPG::log_op_stats": 10142560, + "ReplicatedBackend::do_repop_reply": 11072736, + "ReplicatedBackend::generate_subop": 11047760, + "ReplicatedBackend::repop_commit": 11084096, + "ReplicatedBackend::submit_transaction": 11068640 + }, + "func2vf": { + "BlueStore::_do_write": { + "var_fields": [] + }, + "BlueStore::_txc_apply_kv": { + "var_fields": [ + { + "location": { + "reg": 4, + "offset": 0, + "stack": false + }, + "fields": [ + { + "offset": 0, + "pointer": false + }, + { + "offset": 764, + "pointer": true + } + ] + } + ] + }, + "BlueStore::_txc_calc_cost": { + "var_fields": [ + { + "location": { + "reg": 4, + "offset": 0, + "stack": false + }, + "fields": [ + { + "offset": 0, + "pointer": false + }, + { + "offset": 16, + "pointer": true + }, + { + "offset": 0, + "pointer": false + }, + { + "offset": 320, + "pointer": true + } + ] + }, + { + "location": { + "reg": 4, + "offset": 0, + "stack": false + }, + "fields": [ + { + "offset": 0, + "pointer": false + }, + { + "offset": 640, + "pointer": true + }, + { + "offset": 0, + "pointer": false + }, + { + "offset": 0, + "pointer": false + } + ] + } + ] + }, + "BlueStore::_txc_state_proc": { + "var_fields": [ + { + "location": { + "reg": 4, + "offset": 0, + "stack": false + }, + "fields": [ + { + "offset": 0, + "pointer": false + }, + { + "offset": 16, + "pointer": true + }, + { + "offset": 0, + "pointer": false + }, + { + "offset": 320, + "pointer": true + } + ] + }, + { + "location": { + "reg": 4, + "offset": 0, + "stack": false + }, + "fields": [ + { + "offset": 0, + "pointer": false + }, + { + "offset": 640, + "pointer": true + }, + { + "offset": 0, + "pointer": false + }, + { + "offset": 0, + "pointer": false + } + ] + }, + { + "location": { + "reg": 4, + "offset": 0, + "stack": false + }, + "fields": [ + { + "offset": 0, + "pointer": false + }, + { + "offset": 764, + "pointer": true + } + ] + }, + { + "location": { + "reg": 4, + "offset": 0, + "stack": false + }, + "fields": [ + { + "offset": 0, + "pointer": false + }, + { + "offset": 456, + "pointer": true + }, + { + "offset": 160, + "pointer": false + } + ] + } + ] + }, + "BlueStore::_wctx_finish": { + "var_fields": [ + { + "location": { + "reg": 4, + "offset": 0, + "stack": false + }, + "fields": [ + { + "offset": 0, + "pointer": false + }, + { + "offset": 16, + "pointer": true + }, + { + "offset": 0, + "pointer": false + }, + { + "offset": 320, + "pointer": true + } + ] + }, + { + "location": { + "reg": 4, + "offset": 0, + "stack": false + }, + "fields": [ + { + "offset": 0, + "pointer": false + }, + { + "offset": 640, + "pointer": true + }, + { + "offset": 0, + "pointer": false + }, + { + "offset": 0, + "pointer": false + } + ] + } + ] + }, + "BlueStore::log_latency": { + "var_fields": [ + { + "location": { + "reg": 1, + "offset": 0, + "stack": false + }, + "fields": [ + { + "offset": 0, + "pointer": false + } + ] + }, + { + "location": { + "reg": 2, + "offset": 0, + "stack": false + }, + "fields": [ + { + "offset": 0, + "pointer": false + }, + { + "offset": 0, + "pointer": true + } + ] + } + ] + }, + "BlueStore::log_latency_fn": { + "var_fields": [ + { + "location": { + "reg": 1, + "offset": 0, + "stack": false + }, + "fields": [ + { + "offset": 0, + "pointer": false + } + ] + }, + { + "location": { + "reg": 2, + "offset": 0, + "stack": false + }, + "fields": [ + { + "offset": 0, + "pointer": false + }, + { + "offset": 0, + "pointer": true + } + ] + } + ] + }, + "BlueStore::queue_transactions": { + "var_fields": [] + }, + "ECBackend::submit_transaction": { + "var_fields": [ + { + "location": { + "reg": 7, + "offset": 48, + "stack": true + }, + "fields": [ + { + "offset": 0, + "pointer": false + }, + { + "offset": 0, + "pointer": false + }, + { + "offset": 8, + "pointer": false + } + ] + }, + { + "location": { + "reg": 7, + "offset": 48, + "stack": true + }, + "fields": [ + { + "offset": 0, + "pointer": false + }, + { + "offset": 16, + "pointer": false + } + ] + } + ] + }, + "OSD::dequeue_op": { + "var_fields": [ + { + "location": { + "reg": 1, + "offset": 0, + "stack": false + }, + "fields": [ + { + "offset": 0, + "pointer": false + }, + { + "offset": 0, + "pointer": false + }, + { + "offset": 272, + "pointer": true + }, + { + "offset": 24, + "pointer": true + }, + { + "offset": 16, + "pointer": false + } + ] + }, + { + "location": { + "reg": 1, + "offset": 0, + "stack": false + }, + "fields": [ + { + "offset": 0, + "pointer": false + }, + { + "offset": 0, + "pointer": false + }, + { + "offset": 280, + "pointer": true + }, + { + "offset": 0, + "pointer": false + }, + { + "offset": 8, + "pointer": false + } + ] + }, + { + "location": { + "reg": 1, + "offset": 0, + "stack": false + }, + "fields": [ + { + "offset": 0, + "pointer": false + }, + { + "offset": 0, + "pointer": false + }, + { + "offset": 280, + "pointer": true + }, + { + "offset": 16, + "pointer": false + } + ] + }, + { + "location": { + "reg": 4, + "offset": 0, + "stack": false + }, + "fields": [ + { + "offset": 0, + "pointer": false + }, + { + "offset": 0, + "pointer": false + }, + { + "offset": 32, + "pointer": true + }, + { + "offset": 0, + "pointer": false + }, + { + "offset": 0, + "pointer": false + } + ] + }, + { + "location": { + "reg": 4, + "offset": 0, + "stack": false + }, + "fields": [ + { + "offset": 0, + "pointer": false + }, + { + "offset": 0, + "pointer": false + }, + { + "offset": 32, + "pointer": true + }, + { + "offset": 0, + "pointer": false + }, + { + "offset": 8, + "pointer": false + } + ] + } + ] + }, + "OSD::enqueue_op": { + "var_fields": [ + { + "location": { + "reg": 4, + "offset": 0, + "stack": false + }, + "fields": [ + { + "offset": 0, + "pointer": false + }, + { + "offset": 0, + "pointer": true + }, + { + "offset": 272, + "pointer": true + }, + { + "offset": 24, + "pointer": true + }, + { + "offset": 16, + "pointer": false + } + ] + }, + { + "location": { + "reg": 4, + "offset": 0, + "stack": false + }, + "fields": [ + { + "offset": 0, + "pointer": false + }, + { + "offset": 0, + "pointer": true + }, + { + "offset": 280, + "pointer": true + }, + { + "offset": 0, + "pointer": false + }, + { + "offset": 8, + "pointer": false + } + ] + }, + { + "location": { + "reg": 4, + "offset": 0, + "stack": false + }, + "fields": [ + { + "offset": 0, + "pointer": false + }, + { + "offset": 0, + "pointer": true + }, + { + "offset": 280, + "pointer": true + }, + { + "offset": 16, + "pointer": false + } + ] + }, + { + "location": { + "reg": 4, + "offset": 0, + "stack": false + }, + "fields": [ + { + "offset": 0, + "pointer": false + }, + { + "offset": 0, + "pointer": true + }, + { + "offset": 272, + "pointer": true + }, + { + "offset": 200, + "pointer": true + } + ] + }, + { + "location": { + "reg": 4, + "offset": 0, + "stack": false + }, + "fields": [ + { + "offset": 0, + "pointer": false + }, + { + "offset": 0, + "pointer": true + }, + { + "offset": 272, + "pointer": true + }, + { + "offset": 216, + "pointer": true + } + ] + }, + { + "location": { + "reg": 4, + "offset": 0, + "stack": false + }, + "fields": [ + { + "offset": 0, + "pointer": false + }, + { + "offset": 0, + "pointer": true + }, + { + "offset": 272, + "pointer": true + }, + { + "offset": 224, + "pointer": true + } + ] + }, + { + "location": { + "reg": 4, + "offset": 0, + "stack": false + }, + "fields": [ + { + "offset": 0, + "pointer": false + }, + { + "offset": 0, + "pointer": true + }, + { + "offset": 272, + "pointer": true + }, + { + "offset": 208, + "pointer": true + } + ] + } + ] + }, + "OpRequest::mark_flag_point": { + "var_fields": [ + { + "location": { + "reg": 4, + "offset": 0, + "stack": false + }, + "fields": [ + { + "offset": 0, + "pointer": false + } + ] + }, + { + "location": { + "reg": 5, + "offset": 0, + "stack": false + }, + "fields": [ + { + "offset": 0, + "pointer": false + }, + { + "offset": 280, + "pointer": true + }, + { + "offset": 0, + "pointer": false + }, + { + "offset": 8, + "pointer": false + } + ] + }, + { + "location": { + "reg": 5, + "offset": 0, + "stack": false + }, + "fields": [ + { + "offset": 0, + "pointer": false + }, + { + "offset": 280, + "pointer": true + }, + { + "offset": 16, + "pointer": false + } + ] + } + ] + }, + "OpRequest::mark_flag_point_string": { + "var_fields": [ + { + "location": { + "reg": 4, + "offset": 0, + "stack": false + }, + "fields": [ + { + "offset": 0, + "pointer": false + } + ] + }, + { + "location": { + "reg": 5, + "offset": 0, + "stack": false + }, + "fields": [ + { + "offset": 0, + "pointer": false + }, + { + "offset": 280, + "pointer": true + }, + { + "offset": 0, + "pointer": false + }, + { + "offset": 8, + "pointer": false + } + ] + }, + { + "location": { + "reg": 5, + "offset": 0, + "stack": false + }, + "fields": [ + { + "offset": 0, + "pointer": false + }, + { + "offset": 280, + "pointer": true + }, + { + "offset": 16, + "pointer": false + } + ] + }, + { + "location": { + "reg": 1, + "offset": 0, + "stack": false + }, + "fields": [ + { + "offset": 0, + "pointer": false + }, + { + "offset": 8, + "pointer": true + } + ] + }, + { + "location": { + "reg": 1, + "offset": 0, + "stack": false + }, + "fields": [ + { + "offset": 0, + "pointer": false + }, + { + "offset": 0, + "pointer": true + }, + { + "offset": 0, + "pointer": false + } + ] + } + ] + }, + "PrimaryLogPG::execute_ctx": { + "var_fields": [ + { + "location": { + "reg": 4, + "offset": 0, + "stack": false + }, + "fields": [ + { + "offset": 0, + "pointer": false + }, + { + "offset": 8, + "pointer": true + }, + { + "offset": 0, + "pointer": false + }, + { + "offset": 8, + "pointer": false + } + ] + }, + { + "location": { + "reg": 4, + "offset": 0, + "stack": false + }, + "fields": [ + { + "offset": 0, + "pointer": false + }, + { + "offset": 8, + "pointer": true + }, + { + "offset": 16, + "pointer": false + } + ] + } + ] + }, + "PrimaryLogPG::log_op_stats": { + "var_fields": [ + { + "location": { + "reg": 4, + "offset": 0, + "stack": false + }, + "fields": [ + { + "offset": 0, + "pointer": false + }, + { + "offset": 280, + "pointer": true + }, + { + "offset": 0, + "pointer": false + }, + { + "offset": 8, + "pointer": false + } + ] + }, + { + "location": { + "reg": 4, + "offset": 0, + "stack": false + }, + "fields": [ + { + "offset": 0, + "pointer": false + }, + { + "offset": 280, + "pointer": true + }, + { + "offset": 16, + "pointer": false + } + ] + }, + { + "location": { + "reg": 1, + "offset": 0, + "stack": false + }, + "fields": [ + { + "offset": 0, + "pointer": false + } + ] + }, + { + "location": { + "reg": 2, + "offset": 0, + "stack": false + }, + "fields": [ + { + "offset": 0, + "pointer": false + } + ] + }, + { + "location": { + "reg": 4, + "offset": 0, + "stack": false + }, + "fields": [ + { + "offset": 0, + "pointer": false + }, + { + "offset": 272, + "pointer": true + }, + { + "offset": 200, + "pointer": true + } + ] + }, + { + "location": { + "reg": 4, + "offset": 0, + "stack": false + }, + "fields": [ + { + "offset": 0, + "pointer": false + }, + { + "offset": 272, + "pointer": true + }, + { + "offset": 24, + "pointer": true + }, + { + "offset": 16, + "pointer": false + } + ] + } + ] + }, + "ReplicatedBackend::do_repop_reply": { + "var_fields": [ + { + "location": { + "reg": 4, + "offset": 0, + "stack": false + }, + "fields": [ + { + "offset": 0, + "pointer": false + }, + { + "offset": 0, + "pointer": false + }, + { + "offset": 280, + "pointer": true + }, + { + "offset": 0, + "pointer": false + }, + { + "offset": 8, + "pointer": false + } + ] + }, + { + "location": { + "reg": 4, + "offset": 0, + "stack": false + }, + "fields": [ + { + "offset": 0, + "pointer": false + }, + { + "offset": 0, + "pointer": false + }, + { + "offset": 280, + "pointer": true + }, + { + "offset": 16, + "pointer": false + } + ] + }, + { + "location": { + "reg": 4, + "offset": 0, + "stack": false + }, + "fields": [ + { + "offset": 0, + "pointer": false + }, + { + "offset": 0, + "pointer": false + }, + { + "offset": 272, + "pointer": true + }, + { + "offset": 24, + "pointer": true + }, + { + "offset": 36, + "pointer": false + }, + { + "offset": 1, + "pointer": false + } + ] + } + ] + }, + "ReplicatedBackend::generate_subop": { + "var_fields": [ + { + "location": { + "reg": 7, + "offset": 8, + "stack": true + }, + "fields": [ + { + "offset": 0, + "pointer": false + }, + { + "offset": 0, + "pointer": false + }, + { + "offset": 8, + "pointer": false + } + ] + }, + { + "location": { + "reg": 7, + "offset": 8, + "stack": true + }, + "fields": [ + { + "offset": 0, + "pointer": false + }, + { + "offset": 16, + "pointer": false + } + ] + }, + { + "location": { + "reg": 7, + "offset": 96, + "stack": true + }, + "fields": [ + { + "offset": 0, + "pointer": false + }, + { + "offset": 0, + "pointer": false + } + ] + } + ] + }, + "ReplicatedBackend::repop_commit": { + "var_fields": [ + { + "location": { + "reg": 4, + "offset": 0, + "stack": false + }, + "fields": [ + { + "offset": 0, + "pointer": false + }, + { + "offset": 0, + "pointer": false + }, + { + "offset": 0, + "pointer": true + }, + { + "offset": 0, + "pointer": false + }, + { + "offset": 280, + "pointer": true + }, + { + "offset": 0, + "pointer": false + }, + { + "offset": 8, + "pointer": false + } + ] + }, + { + "location": { + "reg": 4, + "offset": 0, + "stack": false + }, + "fields": [ + { + "offset": 0, + "pointer": false + }, + { + "offset": 0, + "pointer": false + }, + { + "offset": 0, + "pointer": true + }, + { + "offset": 0, + "pointer": false + }, + { + "offset": 280, + "pointer": true + }, + { + "offset": 16, + "pointer": false + } + ] + }, + { + "location": { + "reg": 4, + "offset": 0, + "stack": false + }, + "fields": [ + { + "offset": 0, + "pointer": false + }, + { + "offset": 0, + "pointer": false + }, + { + "offset": 0, + "pointer": true + }, + { + "offset": 0, + "pointer": false + }, + { + "offset": 272, + "pointer": true + }, + { + "offset": 168, + "pointer": true + }, + { + "offset": 24, + "pointer": false + } + ] + } + ] + }, + "ReplicatedBackend::submit_transaction": { + "var_fields": [ + { + "location": { + "reg": 7, + "offset": 48, + "stack": true + }, + "fields": [ + { + "offset": 0, + "pointer": false + }, + { + "offset": 0, + "pointer": false + }, + { + "offset": 8, + "pointer": false + } + ] + }, + { + "location": { + "reg": 7, + "offset": 48, + "stack": true + }, + "fields": [ + { + "offset": 0, + "pointer": false + }, + { + "offset": 16, + "pointer": false + } + ] + } + ] + } + } + } +} \ No newline at end of file diff --git a/files/ubuntu/radostrace/20.2.0-0ubuntu2_amd64v3_dwarf.json b/files/ubuntu/radostrace/20.2.0-0ubuntu2_amd64v3_dwarf.json new file mode 100644 index 0000000..3f57ea9 --- /dev/null +++ b/files/ubuntu/radostrace/20.2.0-0ubuntu2_amd64v3_dwarf.json @@ -0,0 +1,388 @@ +{ + "version": "20.2.0-0ubuntu2", + "arch": "amd64", + "libceph-common.so.2": { + "build_id": "23210fa68b1390b171725207eae4eb202e0ecf4e", + "func2pc": { + "Objecter::_finish_op": 9960176, + "Objecter::_send_op": 9897728 + }, + "func2vf": { + "Objecter::_finish_op": { + "var_fields": [ + { + "location": { + "reg": 4, + "offset": 0, + "stack": false + }, + "fields": [ + { + "offset": 0, + "pointer": false + }, + { + "offset": 1064, + "pointer": true + } + ] + }, + { + "location": { + "reg": 5, + "offset": 0, + "stack": false + }, + "fields": [ + { + "offset": 0, + "pointer": false + }, + { + "offset": 32, + "pointer": true + }, + { + "offset": 1368, + "pointer": true + } + ] + }, + { + "location": { + "reg": 4, + "offset": 0, + "stack": false + }, + "fields": [ + { + "offset": 0, + "pointer": false + }, + { + "offset": 40, + "pointer": true + }, + { + "offset": 400, + "pointer": false + } + ] + } + ] + }, + "Objecter::_send_op": { + "var_fields": [ + { + "location": { + "reg": 4, + "offset": 0, + "stack": false + }, + "fields": [ + { + "offset": 0, + "pointer": false + }, + { + "offset": 1064, + "pointer": true + } + ] + }, + { + "location": { + "reg": 5, + "offset": 0, + "stack": false + }, + "fields": [ + { + "offset": 0, + "pointer": false + }, + { + "offset": 32, + "pointer": true + }, + { + "offset": 1368, + "pointer": true + } + ] + }, + { + "location": { + "reg": 4, + "offset": 0, + "stack": false + }, + "fields": [ + { + "offset": 0, + "pointer": false + }, + { + "offset": 40, + "pointer": true + }, + { + "offset": 400, + "pointer": false + } + ] + }, + { + "location": { + "reg": 4, + "offset": 0, + "stack": false + }, + "fields": [ + { + "offset": 0, + "pointer": false + }, + { + "offset": 40, + "pointer": true + }, + { + "offset": 8, + "pointer": false + }, + { + "offset": 0, + "pointer": false + }, + { + "offset": 8, + "pointer": false + } + ] + }, + { + "location": { + "reg": 4, + "offset": 0, + "stack": false + }, + "fields": [ + { + "offset": 0, + "pointer": false + }, + { + "offset": 40, + "pointer": true + }, + { + "offset": 8, + "pointer": false + }, + { + "offset": 0, + "pointer": false + }, + { + "offset": 0, + "pointer": false + }, + { + "offset": 0, + "pointer": false + } + ] + }, + { + "location": { + "reg": 4, + "offset": 0, + "stack": false + }, + "fields": [ + { + "offset": 0, + "pointer": false + }, + { + "offset": 40, + "pointer": true + }, + { + "offset": 0, + "pointer": false + } + ] + }, + { + "location": { + "reg": 4, + "offset": 0, + "stack": false + }, + "fields": [ + { + "offset": 0, + "pointer": false + }, + { + "offset": 40, + "pointer": true + }, + { + "offset": 272, + "pointer": false + }, + { + "offset": 0, + "pointer": false + }, + { + "offset": 0, + "pointer": false + } + ] + }, + { + "location": { + "reg": 4, + "offset": 0, + "stack": false + }, + "fields": [ + { + "offset": 0, + "pointer": false + }, + { + "offset": 40, + "pointer": true + }, + { + "offset": 272, + "pointer": false + }, + { + "offset": 0, + "pointer": false + }, + { + "offset": 8, + "pointer": false + } + ] + }, + { + "location": { + "reg": 4, + "offset": 0, + "stack": false + }, + "fields": [ + { + "offset": 0, + "pointer": false + }, + { + "offset": 40, + "pointer": true + }, + { + "offset": 336, + "pointer": false + }, + { + "offset": 0, + "pointer": false + }, + { + "offset": 0, + "pointer": false + } + ] + }, + { + "location": { + "reg": 4, + "offset": 0, + "stack": false + }, + "fields": [ + { + "offset": 0, + "pointer": false + }, + { + "offset": 40, + "pointer": true + }, + { + "offset": 336, + "pointer": false + }, + { + "offset": 0, + "pointer": false + }, + { + "offset": 8, + "pointer": false + } + ] + }, + { + "location": { + "reg": 4, + "offset": 0, + "stack": false + }, + "fields": [ + { + "offset": 0, + "pointer": false + }, + { + "offset": 464, + "pointer": true + }, + { + "offset": 0, + "pointer": false + }, + { + "offset": 0, + "pointer": false + } + ] + }, + { + "location": { + "reg": 4, + "offset": 0, + "stack": false + }, + "fields": [ + { + "offset": 0, + "pointer": false + }, + { + "offset": 464, + "pointer": true + }, + { + "offset": 0, + "pointer": false + }, + { + "offset": 8, + "pointer": false + } + ] + } + ] + } + } + } +} \ No newline at end of file From ac1838db8358826b16ef686888260fe3d4d8fa0a Mon Sep 17 00:00:00 2001 From: Dongdong Tao Date: Wed, 1 Jul 2026 16:35:55 +0900 Subject: [PATCH 3/8] ci: run build-ubuntu(+cephadm) on 26.04; key dwarf-compare by build-id Add ubuntu-26.04 to the build-ubuntu and build-ubuntu-cephadm runner matrices so the build, DWARF-compare, MicroCeph, embedded-DWARF and cephadm functional tests run on Resolute. Ubuntu 26.04 cloud images default to the amd64v3 package variant, which shares its version string and dpkg architecture with the baseline amd64 build but has different function addresses and a different ELF build-id. dwarf-compare.sh keyed the reference DWARF JSON by package version, which is ambiguous between the two variants. Key by the ELF build-id embedded in the freshly generated JSON instead: it selects the correct reference (amd64 vs amd64v3) automatically, whichever variant the runner happens to install, and the script now prints exactly which reference file was matched. If no reference build-id matches, it fails with a clear "missing reference" error. All existing ubuntu reference files already carry per-module build-ids, so build-id keying works on the 22.04 and 24.04 runners unchanged. --- .github/workflows/pr-build.yaml | 4 +-- tests/dwarf-compare.sh | 53 ++++++++++++++++++++++++--------- 2 files changed, 41 insertions(+), 16 deletions(-) diff --git a/.github/workflows/pr-build.yaml b/.github/workflows/pr-build.yaml index dc6e1d8..b24ce8c 100644 --- a/.github/workflows/pr-build.yaml +++ b/.github/workflows/pr-build.yaml @@ -11,7 +11,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-22.04, ubuntu-24.04, ubuntu-latest] + os: [ubuntu-22.04, ubuntu-24.04, ubuntu-26.04, ubuntu-latest] steps: - name: Checkout code and submodules uses: actions/checkout@v4 @@ -51,7 +51,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-22.04, ubuntu-24.04] + os: [ubuntu-22.04, ubuntu-24.04, ubuntu-26.04] release: [quincy, reef, squid, tentacle] steps: - name: Checkout code and submodules diff --git a/tests/dwarf-compare.sh b/tests/dwarf-compare.sh index c73c94b..824acc6 100755 --- a/tests/dwarf-compare.sh +++ b/tests/dwarf-compare.sh @@ -99,32 +99,57 @@ else install_dbgsyms_from_launchpad $DBGSYM_PKGS fi -# Get the ceph version for reference file lookup -matching_ref_version=$(dpkg -l | awk '$2=="ceph-common" {print $3}') - -# Reference filenames may carry an optional architecture suffix -# (e.g. osd-19.2.3-0ubuntu0.24.04.3_arm64_dwarf.json) when the same -# package version is checked in for more than one arch. Glob and pick -# the first match instead of hard-coding the filename. -find_ref() { - local dir="$1" prefix="$2" - ls "${dir}/${prefix}${matching_ref_version}"*_dwarf.json 2>/dev/null | head -1 +# Select the reference DWARF JSON by the ELF build-id embedded in the freshly +# generated JSON, NOT the package version. The amd64 and amd64v3 builds of the +# same Ceph version (Ubuntu 26.04 publishes an amd64v3 package variant) share a +# version string but have distinct build-ids and distinct function addresses, +# so version-keying could pick the wrong-arch reference. Build-id keying picks +# the correct file automatically, whichever variant the runner installed, and +# the caller prints exactly which file was matched. +find_ref_by_buildid() { + local dir="$1" module="$2" generated="$3" + python3 - "$dir" "$module" "$generated" <<'PY' +import glob, json, os, sys +ref_dir, module, generated = sys.argv[1:4] +want = json.load(open(generated)).get(module, {}).get("build_id") +if not want: + sys.stderr.write( + f"ERROR: generated JSON {generated} has no build_id for module {module}\n") + sys.exit(1) +for path in sorted(glob.glob(os.path.join(ref_dir, "*_dwarf.json"))): + try: + d = json.load(open(path)) + except (OSError, ValueError): + continue + if d.get(module, {}).get("build_id") == want: + print(path) + sys.exit(0) +sys.stderr.write( + f"ERROR: no reference DWARF JSON in {ref_dir} matches the installed " + f"{module} build-id {want}; a reference file for this exact build is " + f"probably missing and should be generated and checked in\n") +sys.exit(1) +PY } # Test osdtrace dwarf json generation echo "Testing osdtrace dwarf json generation..." osd_new_dwarf="generated-osd-dwarf.json" ./osdtrace -j $osd_new_dwarf -osd_ref_file=$(find_ref ./files/ubuntu/osdtrace osd-) -./tests/compare_dwarf_json.py $osd_ref_file $osd_new_dwarf +osd_ref_file=$(find_ref_by_buildid ./files/ubuntu/osdtrace ceph-osd "$osd_new_dwarf") \ + || { echo "osdtrace reference lookup failed"; exit 1; } +echo "Using reference DWARF JSON (matched by ceph-osd build-id): $osd_ref_file" +./tests/compare_dwarf_json.py "$osd_ref_file" "$osd_new_dwarf" echo "osdtrace dwarf json comparison passed!" # Test radostrace dwarf json generation echo "Testing radostrace dwarf json generation..." rados_new_dwarf="generated-rados-dwarf.json" ./radostrace -j $rados_new_dwarf -rados_ref_file=$(find_ref ./files/ubuntu/radostrace "") -./tests/compare_dwarf_json.py $rados_ref_file $rados_new_dwarf +rados_ref_file=$(find_ref_by_buildid ./files/ubuntu/radostrace libceph-common.so.2 "$rados_new_dwarf") \ + || { echo "radostrace reference lookup failed"; exit 1; } +echo "Using reference DWARF JSON (matched by libceph-common.so.2 build-id): $rados_ref_file" +./tests/compare_dwarf_json.py "$rados_ref_file" "$rados_new_dwarf" echo "radostrace dwarf json comparison passed!" echo "All dwarf json comparisons passed successfully!" From 376537b4f1461f86e90b663ae838327aac93a900 Mon Sep 17 00:00:00 2001 From: Dongdong Tao Date: Wed, 1 Jul 2026 17:32:09 +0900 Subject: [PATCH 4/8] tests/cephadm: fall back to container cephadm when distro package is broken MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit install_cephadm accepted the apt-installed cephadm whenever `apt-get install` returned 0, without checking it runs. On Ubuntu 26.04 (Resolute) the cephadm package is broken: it imports python modules (`ceph`, `yaml`) that aren't pulled in as dependencies, so every invocation — including `bootstrap` — dies with `ModuleNotFoundError`. The build-ubuntu-cephadm job on 26.04 therefore failed at "bootstrap failed to produce FSID" for all releases. Verify the distro cephadm actually executes (`cephadm --help`, which forces the module imports but needs no root/podman/network) before committing to it; if it doesn't run, fall through to the existing container-extraction path. The in-image /usr/sbin/cephadm is a self-contained python zipapp that bundles its own deps, so it runs regardless of host packaging. Also validate the container-extracted cephadm and fail clearly if it too cannot run. 22.04/24.04 keep using the apt cephadm (its --help succeeds); only the broken Resolute case is redirected to the container binary. --- tests/lib/cephadm-setup.sh | 49 ++++++++++++++++++++++++++------------ 1 file changed, 34 insertions(+), 15 deletions(-) diff --git a/tests/lib/cephadm-setup.sh b/tests/lib/cephadm-setup.sh index 0da5b68..92518c7 100755 --- a/tests/lib/cephadm-setup.sh +++ b/tests/lib/cephadm-setup.sh @@ -34,15 +34,26 @@ cephadm_image_for_release() { } -# install_cephadm +# Does the cephadm at $1 actually load and run? `--help` forces cephadm's +# module imports (where the Resolute package fails) yet is pure argparse: it +# needs no root, podman, or network and exits 0 on a working binary. Used to +# reject a broken distro package before we commit to it. +_cephadm_runs() { "$1" --help >/dev/null 2>&1; } + +# install_cephadm # # Make a working `cephadm` available at $dest. Strategy: # 1. Try the distro-packaged cephadm (apt install on Ubuntu). Ubuntu 24.04 # ships squid-era cephadm, which can bootstrap any v17–v20 image via -# `--image`. This is the simplest and most reliable path. -# 2. Fall back to extracting cephadm from the matching quay.io image. -# That guarantees the cephadm version matches the cluster image, which -# matters for older releases (quincy, reef) where flag names differ. +# `--image`. This is the simplest and most reliable path -- BUT only if +# it actually runs. The cephadm apt package on Ubuntu 26.04 (Resolute) +# is broken: it imports a `ceph` python module that isn't packaged, so +# every invocation dies with `ModuleNotFoundError: No module named +# 'ceph'`. We accept the distro package only after verifying it executes. +# 2. Fall back to extracting cephadm from the matching quay.io image. The +# in-image /usr/sbin/cephadm is a self-contained python zipapp (it bundles +# cephadmlib and all its deps), so it runs regardless of host packaging, +# and its bootstrap flag-set matches the image version exactly. # # Why not "curl the script from github.com/ceph/ceph/"? # - Squid (and later) repackaged cephadm as a multi-file Python package @@ -53,30 +64,38 @@ cephadm_image_for_release() { install_cephadm() { local release="$1"; local dest="${2:-/tmp/cephadm}"; local image="$3" - # Path 1: distro package. + # Path 1: distro package (already present, or installable via apt), but + # only if it actually runs -- see _cephadm_runs and the Resolute note above. + local sys_cephadm="" if command -v cephadm >/dev/null; then - cp "$(command -v cephadm)" "$dest" - chmod +x "$dest" - info "using distro cephadm: $(cephadm --version 2>&1 | head -1 | tr -d '\n')" - return 0 + sys_cephadm="$(command -v cephadm)" + elif apt-get install -y -q cephadm >/dev/null 2>&1 && command -v cephadm >/dev/null; then + sys_cephadm="$(command -v cephadm)" fi - if apt-get install -y -q cephadm >/dev/null 2>&1; then - cp "$(command -v cephadm)" "$dest" + if [ -n "$sys_cephadm" ]; then + cp "$sys_cephadm" "$dest" chmod +x "$dest" - info "installed cephadm via apt: $(cephadm --version 2>&1 | head -1 | tr -d '\n')" - return 0 + if _cephadm_runs "$dest"; then + info "using distro cephadm from $sys_cephadm" + return 0 + fi + info "distro cephadm ($sys_cephadm) does not run (broken package, e.g. Resolute); falling back to container image" fi # Path 2: extract from the matching container image. Every quay.io/ceph # image bundles /usr/sbin/cephadm; pulling that out yields a cephadm # whose bootstrap flag-set matches the image version exactly. - [ -n "$image" ] || { err "no fallback image supplied to install_cephadm"; return 1; } + [ -n "$image" ] || { err "no working distro cephadm and no fallback image supplied to install_cephadm"; return 1; } info "extracting cephadm from $image (cold start ~1 min for image pull)" podman pull "$image" >/dev/null local cid; cid=$(podman create --rm "$image" /bin/true) podman cp "${cid}:/usr/sbin/cephadm" "$dest" podman rm -f "$cid" >/dev/null 2>&1 || true chmod +x "$dest" + if ! _cephadm_runs "$dest"; then + err "cephadm extracted from $image does not run" + return 1 + fi info "extracted cephadm from $image" return 0 } From f3a527359346300fae5036666c39188f6a5d03d0 Mon Sep 17 00:00:00 2001 From: Dongdong Tao Date: Wed, 1 Jul 2026 17:43:47 +0900 Subject: [PATCH 5/8] tests/cephadm: create ceph uid/gid 167 for 26.04 uutils install compat MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit cephadm bootstrap creates /var/run/ceph with `install -d -o 167 -g 167` (167 is the ceph uid/gid baked into the ceph container images). Ubuntu 26.04 replaced GNU coreutils with uutils, whose `install` rejects a numeric owner that has no /etc/passwd entry — GNU install accepted it — so bootstrap failed on the bare 26.04 runner with `install: invalid user: '167'`. Create a matching ceph user/group (uid/gid 167) before bootstrap when they don't already resolve, falling back to the name "ceph167" if "ceph" is taken by a differently-numbered user. No-op on 22.04/24.04. Verified on a bare ubuntu:26.04 container: the previously failing `install -d -o 167 -g 167 /var/run/ceph/...` now succeeds. --- tests/lib/cephadm-setup.sh | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tests/lib/cephadm-setup.sh b/tests/lib/cephadm-setup.sh index 92518c7..dd3fffa 100755 --- a/tests/lib/cephadm-setup.sh +++ b/tests/lib/cephadm-setup.sh @@ -175,6 +175,28 @@ _orch_backend_ready() { } +# _ensure_ceph_user +# +# cephadm bootstrap creates /var/run/ceph via `install -d -o 167 -g 167`, +# where 167 is the ceph uid/gid baked into the ceph container images. Ubuntu +# 26.04 ships uutils coreutils, whose `install` rejects a numeric owner absent +# from /etc/passwd (GNU install accepted it), so bootstrap fails there with +# `install: invalid user: '167'`. Create a matching ceph user/group when uid +# 167 does not already resolve. No-op on 22.04/24.04 and anywhere ceph is +# already installed. +_ensure_ceph_user() { + getent passwd 167 >/dev/null 2>&1 && return 0 + # Only uid/gid 167 need a passwd/group entry (any name); prefer "ceph" but + # fall back to "ceph167" if that name is already taken by a differently- + # numbered ceph user (e.g. a host that already has apt ceph installed). + local g=ceph u=ceph + getent group ceph >/dev/null 2>&1 && g=ceph167 + getent passwd ceph >/dev/null 2>&1 && u=ceph167 + getent group 167 >/dev/null 2>&1 || groupadd -g 167 "$g" 2>/dev/null || true + useradd -u 167 -g 167 -M -r -s /usr/sbin/nologin "$u" 2>/dev/null || true +} + + # cephadm_bootstrap_single_host [cephadm_bin] # # Bootstrap the cluster and echo the new FSID. --single-host-defaults @@ -222,6 +244,9 @@ cephadm_bootstrap_single_host() { # on Ubuntu 22.04 rejects it. CI doesn't need the inspection anyway — # we purge partial clusters ourselves between attempts (see below). local attempt rc fsid + # cephadm's bootstrap chowns /var/run/ceph to uid/gid 167; ensure that user + # exists so 26.04's uutils `install` accepts it (see _ensure_ceph_user). + _ensure_ceph_user for (( attempt=1; attempt<=max_attempts; attempt++ )); do # Always start from a clean slate: clears any debris left by a # previous failed attempt (or a stale cluster from an earlier run on From 59e497f36f4f8b8ae5692099b777120660cdb5c9 Mon Sep 17 00:00:00 2001 From: Dongdong Tao Date: Wed, 1 Jul 2026 17:58:50 +0900 Subject: [PATCH 6/8] tests/cephadm: install container cephadm on PATH for bare `cephadm` callers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After bootstrap the test invokes bare `cephadm` (shell / orch status / ceph -s / osd apply / pool ops) in ~15 places, assuming a working cephadm on PATH. On 26.04 only /tmp/cephadm (the container-extracted binary) works; the on-PATH distro cephadm is broken, so `_orch_backend_ready`'s `cephadm shell -- ceph orch status` failed for every bootstrap attempt even though bootstrap itself completed (rc=0) — the test then rejected a healthy cluster and failed after 3 attempts. When install_cephadm falls back to the container binary, also install it on PATH (overwrite the broken on-PATH copy, else drop it in /usr/local/bin) so the bare `cephadm` calls use the working binary. 22.04/24.04 are unaffected (their distro cephadm works and this path isn't taken). Verified on a bare ubuntu:26.04 container: bare `cephadm --help` goes from broken to working after the replacement. --- tests/lib/cephadm-setup.sh | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/lib/cephadm-setup.sh b/tests/lib/cephadm-setup.sh index dd3fffa..021d2db 100755 --- a/tests/lib/cephadm-setup.sh +++ b/tests/lib/cephadm-setup.sh @@ -96,7 +96,19 @@ install_cephadm() { err "cephadm extracted from $image does not run" return 1 fi - info "extracted cephadm from $image" + # The test invokes bare `cephadm` (shell / orch / ceph -s / ...) directly in + # many places, which assumes a working cephadm on PATH. We only reach this + # path because the distro cephadm on PATH is broken (e.g. Resolute), so + # point PATH at this working binary too: overwrite the broken on-PATH copy, + # or drop one into /usr/local/bin (which precedes /usr/sbin on PATH). + local onpath; onpath="$(command -v cephadm 2>/dev/null || true)" + if [ -n "$onpath" ]; then + cp "$dest" "$onpath" + else + cp "$dest" /usr/local/bin/cephadm && chmod +x /usr/local/bin/cephadm + fi + hash -r 2>/dev/null || true + info "extracted cephadm from $image (also installed on PATH as cephadm)" return 0 } From 23c224f5bb3a762797ad7aed304a529cfc0395cf Mon Sep 17 00:00:00 2001 From: Dongdong Tao Date: Thu, 2 Jul 2026 12:07:01 +0900 Subject: [PATCH 7/8] dwarf_parser: gate the ABI location fallback to x86-64 builds abi_param_location encodes the SysV-AMD64 calling convention -- INTEGER argument register numbers (RDI=5, RSI=4, ...) and MEMORY-class stack layout -- but ran whenever DW_AT_location was missing, on any architecture. On an arm64 build the same DWARF register numbers name different registers (regnum 5 is x5, arguments live in x0-x7), so the fallback could emit a wrong-but-plausible location instead of failing. osdtrace/radostrace always run natively on the host they trace, so the target architecture equals the build architecture: compile the recovery out on non-x86-64 (return false), letting the caller take the normal missing-location failure path. x86-64 behavior is unchanged -- verified by regenerating the amd64v3 DWARF JSON on Ubuntu 26.04 and deep-comparing against the checked-in reference (identical, reqid still recovered). Addresses the Copilot review comment on PR #154. --- src/dwarf_parser.cc | 11 +++++++++++ src/dwarf_parser.h | 2 ++ 2 files changed, 13 insertions(+) diff --git a/src/dwarf_parser.cc b/src/dwarf_parser.cc index 3f38707..41b2dcc 100644 --- a/src/dwarf_parser.cc +++ b/src/dwarf_parser.cc @@ -323,6 +323,16 @@ bool DwarfParser::abi_stack_location(Dwarf_Die *func, Dwarf_Addr pc, bool DwarfParser::abi_param_location(Dwarf_Die *func, Dwarf_Die &target, Dwarf_Addr pc, VarLocation &varloc) { +#if !defined(__x86_64__) + // Everything below encodes the SysV-AMD64 calling convention (INTEGER + // register order, MEMORY-class stack layout); on any other architecture + // those register numbers and offsets would be wrong (e.g. DWARF regnum 5 + // is RDI on x86-64 but x5 on arm64). osdtrace/radostrace always run + // natively on the host they trace, so a non-x86-64 build can never face an + // x86-64 target: decline the recovery and let the caller fail gracefully. + (void)func; (void)target; (void)pc; (void)varloc; + return false; +#else // SysV-AMD64 INTEGER argument registers, in order, as DWARF register // numbers: RDI=5, RSI=4, RDX=1, RCX=2, R8=8, R9=9. static const int kIntArgRegs[6] = {5, 4, 1, 2, 8, 9}; @@ -369,6 +379,7 @@ bool DwarfParser::abi_param_location(Dwarf_Die *func, Dwarf_Die &target, cerr << "abi_param_location: target parameter not found among formals" << endl; return false; +#endif // __x86_64__ } bool DwarfParser::translate_param_location(Dwarf_Die *func, string symbol, diff --git a/src/dwarf_parser.h b/src/dwarf_parser.h index dd60e4f..9429c30 100644 --- a/src/dwarf_parser.h +++ b/src/dwarf_parser.h @@ -78,6 +78,8 @@ class DwarfParser { // calling convention when the DWARF carries no DW_AT_location for it (the // compiler can drop the location of a forwarded by-value aggregate under // -march=x86-64-v3, e.g. reqid in ReplicatedBackend::submit_transaction). + // x86-64 builds only: on other architectures it always returns false, so + // the caller falls back to the normal missing-location failure path. bool abi_param_location(Dwarf_Die *func, Dwarf_Die &target, Dwarf_Addr pc, VarLocation &); // Resolve the call-frame CFA at pc and add an offset, yielding a stack From 5680dc6681cd97e3b950d59ddd7f0be58811b81e Mon Sep 17 00:00:00 2001 From: Dongdong Tao Date: Thu, 2 Jul 2026 19:30:09 +0900 Subject: [PATCH 8/8] ci: add a baseline-amd64 dwarf-compare cell on ubuntu-26.04 The plain ubuntu-26.04 runner exercises the amd64v3 ceph build only: GitHub's 26.04 cloud images ship with APT::Architecture-Variants "amd64v3" enabled, so dwarf-compare installs the v3 packages and validates the _amd64v3 reference JSONs. Nothing covered the Resolute baseline-amd64 references. Add a (ubuntu-26.04, amd64) matrix cell that drops an apt.conf.d snippet with `#clear APT::Architecture-Variants;` before installing anything, so apt resolves the baseline amd64 ceph + dbgsyms and dwarf-compare's build-id keying selects the plain osd-20.2.0-0ubuntu2_dwarf.json references. The #clear mechanism was verified on a variant-enabled 26.04 host: the ceph-common candidate flips from resolute/main amd64v3 to amd64. The MicroCeph/embedded/qemu functional tests deploy a snap-bundled ceph that is independent of the host apt variant, so the new cell skips them and runs only build + dwarf-compare; the default cells are unchanged (they gain a ", default" suffix in the job name from the new matrix dimension). --- .github/workflows/pr-build.yaml | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/.github/workflows/pr-build.yaml b/.github/workflows/pr-build.yaml index b24ce8c..e46da3a 100644 --- a/.github/workflows/pr-build.yaml +++ b/.github/workflows/pr-build.yaml @@ -12,6 +12,18 @@ jobs: strategy: matrix: os: [ubuntu-22.04, ubuntu-24.04, ubuntu-26.04, ubuntu-latest] + # Which apt architecture variant the cell's ceph packages come from. + # "default" means whatever the runner image defaults to: baseline + # amd64 on 22.04/24.04, but amd64v3 on 26.04 (cloud images enable + # APT::Architecture-Variants "amd64v3" out of the box). The extra + # (ubuntu-26.04, amd64) cell disables the variant so the Resolute + # baseline-amd64 reference DWARF JSONs get CI coverage too; + # dwarf-compare's build-id keying selects the matching reference + # automatically in both cells. + ceph-variant: [default] + include: + - os: ubuntu-26.04 + ceph-variant: amd64 steps: - name: Checkout code and submodules uses: actions/checkout@v4 @@ -19,6 +31,15 @@ jobs: submodules: recursive fetch-depth: 0 + - name: Force baseline amd64 packages (disable amd64v3 variant) + if: matrix.ceph-variant == 'amd64' + run: | + # apt.conf.d files parse in lexical order and #clear erases the + # option wherever an earlier file set it, so zz- wins over the + # cloud image's enablement snippet. + echo '#clear APT::Architecture-Variants;' | sudo tee /etc/apt/apt.conf.d/zz-disable-arch-variants + sudo apt-get update + - name: Ensure submodules are updated run: | sudo apt-get update @@ -30,19 +51,27 @@ jobs: - name: Unit test - verify dwarf output is consistent run: ./tests/dwarf-compare.sh + # The remaining functional tests deploy MicroCeph (a snap) and trace its + # bundled ceph -- independent of the host apt variant -- so they only + # run in the default cell; the amd64 variant cell exists purely to give + # dwarf-compare coverage of the baseline-amd64 reference JSONs. - name: Uninstall pre-installed Ceph packages + if: matrix.ceph-variant != 'amd64' run: | # Remove any pre-installed Ceph packages that might conflict with MicroCeph sudo apt-get remove --purge -y ceph ceph-common ceph-osd python3-ceph-argparse python3-ceph-common 'ceph-libboost*' || true sudo apt-get autoremove -y || true - name: Functional test - verify osdtrace and radostrace with MicroCeph + if: matrix.ceph-variant != 'amd64' run: sudo ./tests/functional-test-microceph.sh - name: Embedded DWARF data test - verify integrity and E2E + if: matrix.ceph-variant != 'amd64' run: sudo ./tests/functional-test-embedded-dwarf.sh - name: Functional test - radostrace tracing a qemu VM on an RBD disk + if: matrix.ceph-variant != 'amd64' run: sudo ./tests/functional-test-qemu-rbd.sh build-ubuntu-cephadm: