8386116: vm/jvmti/StopThread/stop001/stop00103/stop00103a.html trips assert(!java_lang_Thread::is_in_vthread_transition(vthread)) failed

src/hotspot/share/prims/jvmtiEnv.cpp

@@ -1224,6 +1224,13 @@ JvmtiEnv::StopThread(jthread thread, jobject exception) {
       return JVMTI_ERROR_OPAQUE_FRAME;
     }
   }
+
+  if (is_virtual && java_thread->on_monitor_waited_event()) {
+    // The exception might end up been thrown in the carrier
+    // so skip this case.
+    return JVMTI_ERROR_OPAQUE_FRAME;
+  }
+
   if (err != JVMTI_ERROR_NONE) {
     return err;
   }

src/java.base/share/classes/java/lang/VirtualThread.java

@@ -544,8 +544,8 @@ private void unmount() {
      */
     @Hidden
     private boolean yieldContinuation() {
-        startTransition(/*mount*/false);
         try {
+            startTransition(/*mount*/false);
             return Continuation.yield(VTHREAD_SCOPE);
         } finally {
             endTransition(/*mount*/true);

test/hotspot/jtreg/serviceability/jvmti/vthread/StopThreadTest2/StopThreadTest2.java

@@ -0,0 +1,74 @@
+/*
+ * Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test
+ * @bug 8386116
+ * @summary Test suspend and send async exception to a yielding virtual thread
+ * @requires vm.continuations
+ * @requires test.thread.factory == null
+ * @library /test/lib /test/hotspot/jtreg
+ * @run main/othervm/native -agentlib:StopThreadTest2 StopThreadTest2
+ */
+
+import java.util.concurrent.CountDownLatch;
+import jdk.test.lib.Asserts;
+
+public class StopThreadTest2 {
+    static final int MAX_VTHREAD_COUNT = Runtime.getRuntime().availableProcessors();
+    static volatile boolean done;
+
+    private static native void suspendAllVirtualThreads();
+    private static native void resumeAllVirtualThreads();
+    private static native void stopThread(Thread thread, Throwable th);
+
+    public static void foo(CountDownLatch start) {
+        try {
+            start.countDown();
+            while (!done) {
+                Thread.yield();
+            }
+        } catch (Throwable t) {
+        }
+    }
+
+    public static void main(String[] args) throws Exception {
+        Thread[] vthreads = new Thread[MAX_VTHREAD_COUNT];
+        for (int i = 0; i < MAX_VTHREAD_COUNT; i++) {
+            var started = new CountDownLatch(1);
+            vthreads[i] = Thread.ofVirtual().name("VThread#" + i).start(() -> foo(started));
+            started.await();
+        }
+
+        suspendAllVirtualThreads();
+        for (Thread vthread : vthreads) {
+            stopThread(vthread, new ThreadDeath());
+        }
+        resumeAllVirtualThreads();
+        done = true;
+
+        for (Thread vthread : vthreads) {
+            vthread.join();
+        }
+    }
+}

test/hotspot/jtreg/serviceability/jvmti/vthread/StopThreadTest2/libStopThreadTest2.cpp

@@ -0,0 +1,78 @@
+/*
+ * Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+#include <jni.h>
+#include <jvmti.h>
+#include <stdio.h>
+#include <string.h>
+#include "jvmti_common.hpp"
+
+// set by Agent_OnLoad
+static jvmtiEnv* jvmti = nullptr;
+
+extern "C" {
+
+JNIEXPORT void JNICALL
+Java_StopThreadTest2_suspendAllVirtualThreads(JNIEnv* jni, jclass cls) {
+  check_jvmti_status(jni, jvmti->SuspendAllVirtualThreads(0, nullptr), "Error in SuspendAllVirtualThreads");
+}
+
+JNIEXPORT void JNICALL
+Java_StopThreadTest2_resumeAllVirtualThreads(JNIEnv* jni, jclass cls) {
+  check_jvmti_status(jni, jvmti->ResumeAllVirtualThreads(0, nullptr), "Error in ResumeAllVirtualThreads");
+}
+
+JNIEXPORT void JNICALL
+Java_StopThreadTest2_stopThread(JNIEnv *jni, jclass cls, jthread thread, jobject exception) {
+  jvmtiError err =  jvmti->StopThread(thread, exception);
+  if (err != JVMTI_ERROR_OPAQUE_FRAME) {
+    check_jvmti_status(jni, err, "Error during StopThread()");
+  }
+}
+
+JNIEXPORT jint JNICALL
+Agent_OnLoad(JavaVM* jvm, char* options, void* reserved) {
+  jvmtiCapabilities caps;
+  jvmtiError err;
+
+  printf("Agent_OnLoad: started\n");
+  if (jvm->GetEnv((void **) (&jvmti), JVMTI_VERSION) != JNI_OK) {
+    LOG("Agent_OnLoad: error in GetEnv");
+    return JNI_ERR;
+  }
+
+  memset(&caps, 0, sizeof(caps));
+  caps.can_suspend = 1;
+  caps.can_signal_thread = 1;
+  caps.can_support_virtual_threads = 1;
+  err = jvmti->AddCapabilities(&caps);
+  if (err != JVMTI_ERROR_NONE) {
+    LOG("Agent_OnLoad: error in JVMTI AddCapabilities: %d\n", err);
+  }
+
+  printf("Agent_OnLoad: finished\n");
+
+  return 0;
+}
+
+} // extern "C"