diff --git a/src/hotspot/share/memory/metaspace.cpp b/src/hotspot/share/memory/metaspace.cpp index b97ae9ab5401d..8b8b80cd893ab 100644 --- a/src/hotspot/share/memory/metaspace.cpp +++ b/src/hotspot/share/memory/metaspace.cpp @@ -867,8 +867,10 @@ size_t Metaspace::max_allocation_word_size() { // Callers are responsible for checking null. MetaWord* Metaspace::allocate(ClassLoaderData* loader_data, size_t word_size, MetaspaceObj::Type type) { - assert(word_size <= Metaspace::max_allocation_word_size(), - "allocation size too large (%zu)", word_size); + if (word_size > Metaspace::max_allocation_word_size()) { + log_warning(gc, metaspace)("allocation size too large (%zu words)", word_size); + return nullptr; + } assert(loader_data != nullptr, "Should never pass around a null loader_data. " "ClassLoaderData::the_null_class_loader_data() should have been used."); @@ -913,7 +915,7 @@ MetaWord* Metaspace::allocate(ClassLoaderData* loader_data, size_t word_size, tracer()->report_metaspace_allocation_failure(loader_data, word_size, type, mdtype); // Allocation failed. - if (is_init_completed()) { + if (is_init_completed() && word_size <= Metaspace::max_allocation_word_size()) { // Only start a GC if the bootstrapping has completed. // Try to clean out some heap memory and retry. This can prevent premature // expansion of the metaspace. diff --git a/test/hotspot/jtreg/runtime/classFileParserBug/StackMapTooLong.java b/test/hotspot/jtreg/runtime/classFileParserBug/StackMapTooLong.java new file mode 100644 index 0000000000000..5a4e2a2eee4ab --- /dev/null +++ b/test/hotspot/jtreg/runtime/classFileParserBug/StackMapTooLong.java @@ -0,0 +1,128 @@ +/* + * Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +/* + * @test Very large StackMapTable should cause OutOfMemoryError and not VM crash. + * @bug 8386562 + * @library /test/lib + * @run main StackMapTooLong + */ + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.net.URL; +import java.net.URLClassLoader; +import jdk.test.lib.helpers.ClassFileAssembler; + +public class StackMapTooLong { + public static void main(String args[]) throws Throwable { + String jcodFile = "BadClass.jcod"; + try (BufferedWriter writer = new BufferedWriter(new FileWriter(jcodFile))) { + writer.write(head); + int max = 2100000; + for (int i = 0; i < max; i++) { + writer.write("0x0000000000000000;\n"); + } + writer.write(tail); + } + + // Generate a StackMapTable that's larger than to Metaspace::max_allocation_word_size() + ClassFileAssembler.compileJcod(jcodFile); + + URL url = new File(System.getProperty("user.dir")).toURI().toURL(); + URL[] urls = new URL[] {url}; + URLClassLoader loader = new URLClassLoader(urls); + try { + loader.loadClass("BadClass"); + throw new RuntimeException("OutOfMemoryError expected but not thrown!"); + } catch (OutOfMemoryError expected) {} + } + + static final String head = """ +// This file is auto-generated +class BadClass { + 0xCAFEBABE; + 0; // minor version + 52; // version + [] { // Constant Pool + ; // first element is empty + Utf8 "BadClass"; // #1 + class #1; // #2 + Utf8 "java/lang/Object"; // #3 + class #3; // #4 + Utf8 ""; // #5 + Utf8 "()V"; // #6 + NameAndType #5 #6; // #7 + Method #4 #7; // #8 + Utf8 "java/lang/Throwable"; // #9 + class #9; // #10 + Utf8 "Code"; // #11 + Utf8 "StackMapTable"; // #12 + } // Constant Pool + + 0x0001; // access + #2;// this_cpx + #4;// super_cpx + + [] { // Interfaces + } // Interfaces + + [] { // Fields + } // Fields + + [] { // Methods + { // method + 0x0001; // access + #5; // name_index + #6; // descriptor_index + [] { // Attributes + Attr(#11) { // Code + 1; // max_stack + 1; // max_locals + Bytes[]{ + 0xB70008B100000000; + } + [] { // Traps + } // end Traps + [] { // Attributes + Attr(#12) { // StackMapTable + [] { // -- to be followed by *a lot* of zeros +"""; + + static final String tail = """ + } + } // end StackMapTable + } // Attributes + } // end Code + } // Attributes + } + } // Methods + + [] { // Attributes + } // Attributes +} // end class BadClass +"""; + +} diff --git a/test/lib/jdk/test/lib/helpers/ClassFileAssembler.java b/test/lib/jdk/test/lib/helpers/ClassFileAssembler.java new file mode 100644 index 0000000000000..8cd25b9f85055 --- /dev/null +++ b/test/lib/jdk/test/lib/helpers/ClassFileAssembler.java @@ -0,0 +1,142 @@ +/* + * Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package jdk.test.lib.helpers; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.concurrent.TimeUnit; +import java.util.List; +import jdk.test.lib.JDKToolFinder; + +/** + * .jasm and .jcod files are usually compiled with the "@compile" tag in jtreg. However, + * in some cases, a test may need to generate the .jasm and .jcod files dynamically. + * ClassFileAssembler is used to compile these files into classes. + * + * The classes are written to the program's current directory. + */ + +public class ClassFileAssembler { + private static final String JAVA_PATH = JDKToolFinder.getJDKTool("java"); + private static final int COMPILE_TIMEOUT = 60; + private static final float timeoutFactor = Float.parseFloat(System.getProperty("test.timeout.factor", "1.0")); + + public static void main(String... args) throws Exception { + for (String file : args) { + if (file.endsWith(".jasm")) { + compileJasm(file); + } else if (file.endsWith(".jcod")) { + compileJcod(file); + } else { + throw new RuntimeException("Usage: jdk.test.lib.helpers.ClassFileAssembler ...\n" + + "where each file must be a .jasm or .jcod source file\n" + + "The class files are written in " + System.getProperty("user.dir")); + } + } + } + + /** + * Compile the given .jasm source file to a class file in the current directory. + */ + public static void compileJasm(String file) throws Exception { + runTool("jasm", file); + } + + /** + * Compile the given .jcod source file to a class file in the current directory. + */ + public static void compileJcod(String file) throws Exception { + runTool("jcoder", file); + } + + private static void runTool(String tool, String file) throws Exception { + List command = new ArrayList<>(); + command.add(JAVA_PATH); + command.add("-classpath"); + command.add(getAsmToolsPath()); + command.add("org.openjdk.asmtools." + tool + ".Main"); + command.add("-d"); + command.add(System.getProperty("user.dir")); + command.add(file); + executeCompileCommand(command); + } + + /** + * Get the path of asmtools, which is shipped with JTREG. + */ + private static String getAsmToolsPath() { + for (String path : getClassPaths()) { + if (path.endsWith("jtreg.jar")) { + File jtreg = new File(path); + File dir = jtreg.getAbsoluteFile().getParentFile(); + File asmtools = new File(dir, "asmtools.jar"); + if (!asmtools.exists()) { + throw new RuntimeException("Found jtreg.jar in classpath, but could not find asmtools.jar"); + } + return asmtools.getAbsolutePath(); + } + } + throw new RuntimeException("Could not find asmtools because could not find jtreg.jar in classpath"); + } + + private static String[] getClassPaths() { + String separator = File.pathSeparator; + return System.getProperty("java.class.path").split(separator); + } + + private static void executeCompileCommand(List command) { + ProcessBuilder builder = new ProcessBuilder(command); + builder.redirectErrorStream(true); + + String output; + int exitCode; + try { + Process process = builder.start(); + long timeout = COMPILE_TIMEOUT * (long)timeoutFactor; + boolean exited = process.waitFor(timeout, TimeUnit.SECONDS); + if (!exited) { + process.destroyForcibly(); + System.out.println("Timeout: compile command: " + String.join(" ", command)); + throw new RuntimeException("Process timeout: compilation took too long."); + } + output = new String(process.getInputStream().readAllBytes(), StandardCharsets.UTF_8); + exitCode = process.exitValue(); + } catch (IOException e) { + throw new RuntimeException("IOException during compilation", e); + } catch (InterruptedException e) { + throw new RuntimeException("InterruptedException during compilation", e); + } + + // Note: the output can be non-empty even if the compilation succeeds, e.g. for warnings. + if (exitCode != 0) { + System.err.println("Compilation failed."); + System.err.println("Command: " + command); + System.err.println("Exit code: " + exitCode); + System.err.println("Output: '" + output + "'"); + throw new RuntimeException("Compilation failed."); + } + } +}