objtool: Fix libopcodes linking with static libraries

Commit 436326b ("objtool: fix build failure due to missing libopcodes
check") tests for libopcodes using an empty main(), which passes even when
static libraries lack their dependencies. This causes undefined reference
errors (xmalloc, bfd_get_bits, etc.) when linking against static libopcodes
without its required libbfd and libiberty.

Fix by testing with an actual libopcodes symbol and trying increasingly
complete library combinations until one succeeds.

Fixes: 436326b ("objtool: fix build failure due to missing libopcodes check")
Reported-by: Rafael J. Wysocki <[email protected]>
Signed-off-by: Sasha Levin <[email protected]>
Signed-off-by: Peter Zijlstra (Intel) <[email protected]>
Tested-by: Rafael J. Wysocki (Intel) <[email protected]>
Link: https://patch.msgid.link/[email protected]

Author: Sasha Levin

tools/objtool/Makefile

@@ -77,19 +77,32 @@ HOST_OVERRIDES := CC="$(HOSTCC)" LD="$(HOSTLD)" AR="$(HOSTAR)"
 # We check using HOSTCC directly rather than the shared feature framework
 # because objtool is a host tool that links against host libraries.
 #
-HAVE_LIBOPCODES := $(shell echo 'int main(void) { return 0; }' | \
-			$(HOSTCC) -xc - -o /dev/null -lopcodes 2>/dev/null && echo y)
+# When using shared libraries, -lopcodes is sufficient as dependencies are
+# resolved automatically. With static libraries, we must explicitly link
+# against libopcodes' dependencies: libbfd, libiberty, and sometimes libz.
+# Try each combination and use the first one that succeeds.
+#
+LIBOPCODES_LIBS := $(shell \
+	for libs in "-lopcodes" \
+		    "-lopcodes -lbfd" \
+		    "-lopcodes -lbfd -liberty" \
+		    "-lopcodes -lbfd -liberty -lz"; do \
+		echo 'extern void disassemble_init_for_target(void *);' \
+		     'int main(void) { disassemble_init_for_target(0); return 0; }' | \
+			$(HOSTCC) -xc - -o /dev/null $$libs 2>/dev/null && \
+			echo "$$libs" && break; \
+	done)
 
 # Styled disassembler support requires binutils >= 2.39
 HAVE_DISASM_STYLED := $(shell echo '$(pound)include <dis-asm.h>' | \
 			$(HOSTCC) -E -xc - 2>/dev/null | grep -q disassembler_style && echo y)
 
 BUILD_DISAS := n
 
-ifeq ($(HAVE_LIBOPCODES),y)
+ifneq ($(LIBOPCODES_LIBS),)
 	BUILD_DISAS := y
 	OBJTOOL_CFLAGS += -DDISAS -DPACKAGE='"objtool"'
-	OBJTOOL_LDFLAGS += -lopcodes
+	OBJTOOL_LDFLAGS += $(LIBOPCODES_LIBS)
 ifeq ($(HAVE_DISASM_STYLED),y)
 	OBJTOOL_CFLAGS += -DDISASM_INIT_STYLED
 endif