8376421: C2: Missing branch on SubTypeCheck node

Reviewed-by: kvn, dfenacci

Author: Vladimir Ivanov

src/hotspot/share/opto/parse2.cpp

@@ -1833,17 +1833,21 @@ void Parse::sharpen_type_after_if(BoolTest::mask btest,
                        &obj, &cast_type)) {
     assert(obj != nullptr && cast_type != nullptr, "missing type check info");
     const Type* obj_type = _gvn.type(obj);
-    const TypeOopPtr* tboth = obj_type->join_speculative(cast_type)->isa_oopptr();
-    if (tboth != nullptr && tboth != obj_type && tboth->higher_equal(obj_type)) {
+    const Type* tboth = obj_type->filter_speculative(cast_type);
+    assert(tboth->higher_equal(obj_type) && tboth->higher_equal(cast_type), "sanity");
+    if (tboth == Type::TOP && KillPathsReachableByDeadTypeNode) {
+      // Let dead type node cleaning logic prune effectively dead path for us.
+      // CheckCastPP::Value() == TOP and it will trigger the cleanup during GVN.
+      // Don't materialize the cast when cleanup is disabled, because
+      // it kills data and control leaving IR in broken state.
+      tboth = cast_type;
+    }
+    if (tboth != Type::TOP && tboth != obj_type) {
       int obj_in_map = map()->find_edge(obj);
-      JVMState* jvms = this->jvms();
       if (obj_in_map >= 0 &&
-          (jvms->is_loc(obj_in_map) || jvms->is_stk(obj_in_map))) {
+          (jvms()->is_loc(obj_in_map) || jvms()->is_stk(obj_in_map))) {
         TypeNode* ccast = new CheckCastPPNode(control(), obj, tboth);
-        const Type* tcc = ccast->as_Type()->type();
-        assert(tcc != obj_type && tcc->higher_equal(obj_type), "must improve");
-        // Delay transform() call to allow recovery of pre-cast value
-        // at the control merge.
+        // Delay transform() call to allow recovery of pre-cast value at the control merge.
         _gvn.set_type_bottom(ccast);
         record_for_igvn(ccast);
         // Here's the payoff.

test/hotspot/jtreg/compiler/types/TestSubTypeCheckInterfaceNotImplemented.java

@@ -0,0 +1,76 @@
+/*
+ * Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ *
+ */
+
+/*
+ * @test
+ * @bug 8376421
+ * @summary "C2: Missing branch on SubTypeCheck node"
+ *
+ * @run main/othervm -Xbatch -XX:+IgnoreUnrecognizedVMOptions
+ *                   -XX:+KillPathsReachableByDeadTypeNode
+ *                   compiler.types.TestSubTypeCheckInterfaceNotImplemented
+ *
+ * @run main/othervm -Xbatch -XX:+IgnoreUnrecognizedVMOptions
+ *                   -XX:-KillPathsReachableByDeadTypeNode
+ *                   compiler.types.TestSubTypeCheckInterfaceNotImplemented
+ */
+
+package compiler.types;
+
+public class TestSubTypeCheckInterfaceNotImplemented {
+    static abstract class A           {}
+    static abstract class B extends A {}
+    static final    class C extends B {}
+
+    interface I {}
+    static final class BJ1 extends A implements I {}
+    static final class BJ2 extends A implements I {}
+
+    static boolean testHelper2(B o) {
+        return true;
+    }
+    static boolean testHelper1(Object o) {
+        if (o instanceof B) {
+            return testHelper2((B)o); // a call to place "o" on JVMS, so the map is updated after the check
+        } else {
+            return false;
+        }
+    }
+
+    static boolean test(A a) {
+        if (a instanceof I) {
+            return testHelper1((I)a); // "a" always fails instanceof check against B
+        } else {
+            return false;
+        }
+    }
+
+    public static void main(String[] args) {
+        for (int i = 0; i < 20_000; i++) {
+            testHelper1(new C()); // pollute profile
+
+            test(new BJ1()); test(new BJ2()); test(new C());
+        }
+    }
+}