From 7d1342bb159b4a1aeaa709d1dd8276dad8f3d5a1 Mon Sep 17 00:00:00 2001 From: Foivos Zakkak Date: Wed, 8 Jan 2025 16:52:26 +0200 Subject: [PATCH 01/10] Rerun `sun.nio.ch.NioSocketImpl` class initializer Follow up to https://github.com/oracle/graal/pull/7440 --- .../src/com/oracle/svm/hosted/jdk/JNIRegistrationJavaNio.java | 1 + 1 file changed, 1 insertion(+) diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/jdk/JNIRegistrationJavaNio.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/jdk/JNIRegistrationJavaNio.java index ce89065ec271..ed27eead6ff7 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/jdk/JNIRegistrationJavaNio.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/jdk/JNIRegistrationJavaNio.java @@ -72,6 +72,7 @@ public void duringSetup(DuringSetupAccess a) { initializeAtRunTime(a, "sun.nio.ch.Net", "sun.nio.ch.SocketOptionRegistry$LazyInitialization"); initializeAtRunTime(a, "sun.nio.ch.AsynchronousSocketChannelImpl$DefaultOptionsHolder", "sun.nio.ch.AsynchronousServerSocketChannelImpl$DefaultOptionsHolder", "sun.nio.ch.DatagramChannelImpl$DefaultOptionsHolder", "sun.nio.ch.ServerSocketChannelImpl$DefaultOptionsHolder", "sun.nio.ch.SocketChannelImpl$DefaultOptionsHolder"); + initializeAtRunTime(a, "sun.nio.ch.NioSocketImpl"); /* Ensure that the interrupt signal handler is initialized at runtime. */ initializeAtRunTime(a, "sun.nio.ch.NativeThread"); initializeAtRunTime(a, "sun.nio.ch.FileDispatcherImpl", "sun.nio.ch.FileChannelImpl$Unmapper"); From e494a9b23628221a520f3868d810ed4521ffccd8 Mon Sep 17 00:00:00 2001 From: Fabio Niephaus Date: Fri, 10 Jan 2025 10:32:11 +0100 Subject: [PATCH 02/10] Rename properties in ModuleSupport. Using `svm.` as a prefix makes it clearer this is internal API. --- .../src/com/oracle/svm/util/ModuleSupport.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/substratevm/src/com.oracle.svm.util/src/com/oracle/svm/util/ModuleSupport.java b/substratevm/src/com.oracle.svm.util/src/com/oracle/svm/util/ModuleSupport.java index 0445979610fb..01e04f6715c9 100644 --- a/substratevm/src/com.oracle.svm.util/src/com/oracle/svm/util/ModuleSupport.java +++ b/substratevm/src/com.oracle.svm.util/src/com/oracle/svm/util/ModuleSupport.java @@ -44,8 +44,8 @@ public final class ModuleSupport { public static final List nonExplicitModules = List.of(MODULE_SET_ALL_DEFAULT, MODULE_SET_ALL_SYSTEM, MODULE_SET_ALL_MODULE_PATH); public static final String ENV_VAR_USE_MODULE_SYSTEM = "USE_NATIVE_IMAGE_JAVA_PLATFORM_MODULE_SYSTEM"; - public static final String PROPERTY_IMAGE_EXPLICITLY_ADDED_MODULES = "org.graalvm.nativeimage.module.addmods"; - public static final String PROPERTY_IMAGE_EXPLICITLY_LIMITED_MODULES = "org.graalvm.nativeimage.module.limitmods"; + public static final String PROPERTY_IMAGE_EXPLICITLY_ADDED_MODULES = "svm.modulesupport.addedModules"; + public static final String PROPERTY_IMAGE_EXPLICITLY_LIMITED_MODULES = "svm.modulesupport.limitedModules"; public static final boolean modulePathBuild = isModulePathBuild(); public static final Set SYSTEM_MODULES = Set.of("org.graalvm.nativeimage.builder", "org.graalvm.nativeimage", "org.graalvm.nativeimage.base", "com.oracle.svm.svm_enterprise", From 6e5cb30855eba1a18e7b9026ecf86f97fbc8516d Mon Sep 17 00:00:00 2001 From: Fabio Niephaus Date: Thu, 9 Jan 2025 12:28:14 +0100 Subject: [PATCH 03/10] Attempt to improve memory usage of builder. --- .../com/oracle/svm/core/SubstrateOptions.java | 2 + .../src/com/oracle/svm/driver/MemoryUtil.java | 202 ++++++++++++++++-- .../oracle/svm/hosted/NativeImageOptions.java | 4 +- .../oracle/svm/hosted/ProgressReporter.java | 15 +- 4 files changed, 196 insertions(+), 27 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java index 5ca5744b6619..32fbf319e3ee 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java @@ -738,6 +738,8 @@ public static boolean hasColorsEnabled(OptionValues values) { @Option(help = "Internal option to forward the value of " + NATIVE_IMAGE_OPTIONS_ENV_VAR)// public static final HostedOptionKey BuildOutputNativeImageOptionsEnvVarValue = new HostedOptionKey<>(null); + public static final String BUILD_MEMORY_USAGE_REASON_TEXT_PROPERTY = "svm.build.memoryUsageReasonText"; + /* * Object and array allocation options. */ diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/MemoryUtil.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/MemoryUtil.java index 7cae7ce04a99..a9fc9c34afd4 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/MemoryUtil.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/MemoryUtil.java @@ -24,10 +24,20 @@ */ package com.oracle.svm.driver; +import java.io.BufferedReader; +import java.io.InputStreamReader; import java.lang.management.ManagementFactory; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; +import java.util.function.Function; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import com.oracle.svm.core.OS; +import com.oracle.svm.core.SubstrateOptions; import com.oracle.svm.core.util.ExitStatus; import com.oracle.svm.driver.NativeImage.NativeImageError; @@ -39,10 +49,12 @@ class MemoryUtil { /* Builder needs at least 512MiB for building a helloworld in a reasonable amount of time. */ private static final long MIN_HEAP_BYTES = 512L * MiB_TO_BYTES; - /* If free memory is below 8GiB, use 85% of total system memory (e.g., 7GiB * 85% ~ 6GiB). */ - private static final long DEDICATED_MODE_THRESHOLD = 8L * GiB_TO_BYTES; + /* Use 85% of total system memory (e.g., 7GiB * 85% ~ 6GiB) in dedicated mode. */ private static final double DEDICATED_MODE_TOTAL_MEMORY_RATIO = 0.85D; + /* If available memory is below 8GiB, fall back to dedicated mode. */ + private static final long MIN_AVAILABLE_MEMORY_THRESHOLD = 8L * GiB_TO_BYTES; + /* * Builder uses at most 32GB to avoid disabling compressed oops (UseCompressedOops). * Deliberately use GB (not GiB) to stay well below 32GiB when relative maximum is calculated. @@ -61,9 +73,9 @@ public static List determineMemoryFlags(NativeImage.HostFlags hostFlags) * -XX:InitialRAMPercentage or -Xms. */ if (hostFlags.hasMaxRAMPercentage()) { - flags.add("-XX:MaxRAMPercentage=" + determineReasonableMaxRAMPercentage()); + flags.addAll(determineReasonableMaxRAMPercentage(value -> "-XX:MaxRAMPercentage=" + value)); } else if (hostFlags.hasMaximumHeapSizePercent()) { - flags.add("-XX:MaximumHeapSizePercent=" + (int) determineReasonableMaxRAMPercentage()); + flags.addAll(determineReasonableMaxRAMPercentage(value -> "-XX:MaximumHeapSizePercent=" + value.intValue())); } if (hostFlags.hasGCTimeRatio()) { /* @@ -82,23 +94,40 @@ public static List determineMemoryFlags(NativeImage.HostFlags hostFlags) } /** - * Returns a percentage (0.0-100.0) to be used as a value for the -XX:MaxRAMPercentage flag of - * the builder process. Prefer free memory over total memory to reduce memory pressure on the - * host machine. Note that this method uses OperatingSystemMXBean, which is container-aware. + * Returns a percentage (0.0-100.0) to be used as a value for the -XX:MaxRAMPercentage or + * -XX:MaximumHeapSizePercent flags of the builder process. Dedicated mode uses a fixed + * percentage of total memory and is the default in containers. Shared mode tries to use + * available memory to reduce memory pressure on the host machine. Note that this method uses + * OperatingSystemMXBean, which is container-aware. */ - private static double determineReasonableMaxRAMPercentage() { + private static List determineReasonableMaxRAMPercentage(Function toMemoryFlag) { var osBean = (com.sun.management.OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean(); final double totalMemorySize = osBean.getTotalMemorySize(); - double reasonableMaxMemorySize = osBean.getFreeMemorySize(); + final double dedicatedMemorySize = totalMemorySize * DEDICATED_MODE_TOTAL_MEMORY_RATIO; - if (reasonableMaxMemorySize < DEDICATED_MODE_THRESHOLD) { - /* - * When free memory is low, for example in memory-constrained environments or when a - * good amount of memory is used for caching, use a fixed percentage of total memory - * rather than free memory. In containerized environments, builds are expected to run - * more or less exclusively (builder + driver + optional Gradle/Maven process). - */ - reasonableMaxMemorySize = totalMemorySize * DEDICATED_MODE_TOTAL_MEMORY_RATIO; + String memoryUsageReason = "unknown"; + final boolean isDedicatedMemoryUsage; + if (System.getenv("CI") != null) { + isDedicatedMemoryUsage = true; + memoryUsageReason = "$CI set"; + } else if (isContainerized()) { + isDedicatedMemoryUsage = true; + memoryUsageReason = "in container"; + } else { + isDedicatedMemoryUsage = false; + } + + double reasonableMaxMemorySize; + if (isDedicatedMemoryUsage) { + reasonableMaxMemorySize = dedicatedMemorySize; + } else { + reasonableMaxMemorySize = getAvailableMemorySize(); + if (reasonableMaxMemorySize >= MIN_AVAILABLE_MEMORY_THRESHOLD) { + memoryUsageReason = "enough available"; + } else { // fall back to dedicated mode + memoryUsageReason = "not enough available"; + reasonableMaxMemorySize = dedicatedMemorySize; + } } if (reasonableMaxMemorySize < MIN_HEAP_BYTES) { @@ -111,6 +140,143 @@ private static double determineReasonableMaxRAMPercentage() { /* Ensure max memory size does not exceed upper limit. */ reasonableMaxMemorySize = Math.min(reasonableMaxMemorySize, MAX_HEAP_BYTES); - return reasonableMaxMemorySize / totalMemorySize * 100; + double reasonableMaxRamPercentage = reasonableMaxMemorySize / totalMemorySize * 100; + return List.of(toMemoryFlag.apply(reasonableMaxRamPercentage), + "-D" + SubstrateOptions.BUILD_MEMORY_USAGE_REASON_TEXT_PROPERTY + "=" + memoryUsageReason); + } + + private static boolean isContainerized() { + if (!OS.LINUX.isCurrent()) { + return false; + } + Path cgroupPath = Paths.get("/proc/self/cgroup"); + if (!Files.exists(cgroupPath)) { + return false; + } + try { + return Files.readAllLines(cgroupPath).stream().anyMatch( + s -> s.contains("docker") || s.contains("kubepods") || s.contains("containerd")); + } catch (Exception e) { + } + return false; + } + + private static double getAvailableMemorySize() { + return switch (OS.getCurrent()) { + case LINUX -> getAvailableMemorySizeLinux(); + case DARWIN -> getAvailableMemorySizeDarwin(); + case WINDOWS -> getAvailableMemorySizeWindows(); + }; + } + + /** + * Returns the total amount of available memory in bytes on Linux based on + * /proc/meminfo, otherwise -1. Note that this metric is not + * container-aware (does not take cgroups into account) and may report available memory of the + * host. + * + * @see page_alloc.c#L5137 + */ + private static long getAvailableMemorySizeLinux() { + try { + String memAvailableLine = Files.readAllLines(Paths.get("/proc/meminfo")).stream().filter(l -> l.startsWith("MemAvailable")).findFirst().orElse(""); + Matcher m = Pattern.compile("^MemAvailable:\\s+(\\d+) kB").matcher(memAvailableLine); + if (m.matches()) { + return Long.parseLong(m.group(1)) * KiB_TO_BYTES; + } + } catch (Exception e) { + } + return -1; + } + + /** + * Returns the total amount of available memory in bytes on Darwin based on + * vm_stat, otherwise -1. + * + * @see vm_stat.c + */ + private static long getAvailableMemorySizeDarwin() { + try { + Process p = Runtime.getRuntime().exec(new String[]{"vm_stat"}); + try (BufferedReader reader = new BufferedReader(new InputStreamReader(p.getInputStream()))) { + String line1 = reader.readLine(); + if (line1 == null) { + return -1; + } + Matcher m1 = Pattern.compile("^Mach Virtual Memory Statistics: \\(page size of (\\d+) bytes\\)").matcher(line1); + long pageSize = -1; + if (m1.matches()) { + pageSize = Long.parseLong(m1.group(1)); + } + if (pageSize <= 0) { + return -1; + } + String line2 = reader.readLine(); + Matcher m2 = Pattern.compile("^Pages free:\\s+(\\d+).").matcher(line2); + long freePages = -1; + if (m2.matches()) { + freePages = Long.parseLong(m2.group(1)); + } + if (freePages <= 0) { + return -1; + } + String line3 = reader.readLine(); + if (!line3.startsWith("Pages active")) { + return -1; + } + String line4 = reader.readLine(); + Matcher m4 = Pattern.compile("^Pages inactive:\\s+(\\d+).").matcher(line4); + long inactivePages = -1; + if (m4.matches()) { + inactivePages = Long.parseLong(m4.group(1)); + } + if (inactivePages <= 0) { + return -1; + } + assert freePages > 0 && inactivePages > 0 && pageSize > 0; + return (freePages + inactivePages) * pageSize; + } finally { + p.waitFor(); + } + } catch (Exception e) { + } + return -1; + } + + /** + * Returns the total amount of available memory in bytes on Windows based on wmic, + * otherwise -1. + * + * @see Win32_OperatingSystem + * class + */ + private static long getAvailableMemorySizeWindows() { + try { + Process p = Runtime.getRuntime().exec(new String[]{"cmd.exe", "/c", "wmic", "OS", "get", "FreePhysicalMemory"}); + try (BufferedReader reader = new BufferedReader(new InputStreamReader(p.getInputStream()))) { + String line1 = reader.readLine(); + if (line1 == null || !line1.startsWith("FreePhysicalMemory")) { + return -1; + } + String line2 = reader.readLine(); + if (line2 == null) { + return -1; + } + String line3 = reader.readLine(); + if (line3 == null) { + return -1; + } + Matcher m = Pattern.compile("^(\\d+)\\s+").matcher(line3); + if (m.matches()) { + return Long.parseLong(m.group(1)) * KiB_TO_BYTES; + } + } + p.waitFor(); + } catch (Exception e) { + } + return -1; } } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageOptions.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageOptions.java index 56b1c07019c1..6cff39f645bf 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageOptions.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageOptions.java @@ -186,11 +186,11 @@ public static CStandards getCStandard() { } /** - * Configures the number of threads of the common pool (see driver). + * Configures the number of threads of the common pool. */ private static final String PARALLELISM_OPTION_NAME = "parallelism"; @APIOption(name = PARALLELISM_OPTION_NAME)// - @Option(help = "The maximum number of threads to use concurrently during native image generation.")// + @Option(help = "The maximum number of threads to use concurrently by the build process.")// public static final HostedOptionKey NumberOfThreads = new HostedOptionKey<>(Math.max(1, Math.min(Runtime.getRuntime().availableProcessors(), 32)), key -> { int numberOfThreads = key.getValue(); if (numberOfThreads < 1) { diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporter.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporter.java index cf8debbbbd27..efb2b2893003 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporter.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporter.java @@ -424,14 +424,16 @@ private void printResourceInfo() { recordJsonMetric(ResourceUsageKey.MEMORY_TOTAL, totalMemorySize); List inputArguments = ManagementFactory.getRuntimeMXBean().getInputArguments(); - List maxRAMPrecentageValues = inputArguments.stream().filter(arg -> arg.startsWith("-XX:MaxRAMPercentage")).toList(); - String maxHeapSuffix = "determined at start"; - if (maxRAMPrecentageValues.size() > 1) { // The driver sets this option once - maxHeapSuffix = "set via '%s'".formatted(maxRAMPrecentageValues.get(maxRAMPrecentageValues.size() - 1)); + List maxRAMPercentageValues = inputArguments.stream().filter(arg -> arg.startsWith("-XX:MaxRAMPercentage=") || arg.startsWith("-XX:MaximumHeapSizePercent=")).toList(); + String memoryUsageReason = "unknown"; + if (maxRAMPercentageValues.size() == 1) { // The driver sets one of these options once + memoryUsageReason = System.getProperty(SubstrateOptions.BUILD_MEMORY_USAGE_REASON_TEXT_PROPERTY, "unknown"); + } else if (maxRAMPercentageValues.size() > 1) { + memoryUsageReason = "set via '%s'".formatted(maxRAMPercentageValues.getLast()); } String xmxValueOrNull = inputArguments.stream().filter(arg -> arg.startsWith("-Xmx")).reduce((first, second) -> second).orElse(null); if (xmxValueOrNull != null) { // -Xmx takes precedence over -XX:MaxRAMPercentage - maxHeapSuffix = "set via '%s'".formatted(xmxValueOrNull); + memoryUsageReason = "set via '%s'".formatted(xmxValueOrNull); } int maxNumberOfThreads = NativeImageOptions.getActualNumberOfThreads(); @@ -445,8 +447,7 @@ private void printResourceInfo() { l().printLineSeparator(); l().yellowBold().doclink("Build resources", "#glossary-build-resources").a(":").reset().println(); - l().a(" - %.2fGB of memory (%.1f%% of %.2fGB system memory, %s)", - ByteFormattingUtil.bytesToGiB(maxMemory), Utils.toPercentage(maxMemory, totalMemorySize), ByteFormattingUtil.bytesToGiB(totalMemorySize), maxHeapSuffix).println(); + l().a(" - %.2fGB of memory (%.1f%% of system memory, reason: %s)", ByteFormattingUtil.bytesToGiB(maxMemory), Utils.toPercentage(maxMemory, totalMemorySize), memoryUsageReason).println(); l().a(" - %s thread(s) (%.1f%% of %s available processor(s), %s)", maxNumberOfThreads, Utils.toPercentage(maxNumberOfThreads, availableProcessors), availableProcessors, maxNumberOfThreadsSuffix).println(); } From a37c941ae766151df344a9185df6a9dc708f1aaa Mon Sep 17 00:00:00 2001 From: Fabio Niephaus Date: Fri, 10 Jan 2025 11:50:27 +0100 Subject: [PATCH 04/10] Update documentation about memory limits. --- docs/reference-manual/native-image/BuildOutput.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/reference-manual/native-image/BuildOutput.md b/docs/reference-manual/native-image/BuildOutput.md index ebdd1f181d5f..58d5eb84c87b 100644 --- a/docs/reference-manual/native-image/BuildOutput.md +++ b/docs/reference-manual/native-image/BuildOutput.md @@ -30,7 +30,7 @@ GraalVM Native Image: Generating 'helloworld' (executable)... Garbage collector: Serial GC (max heap size: 80% of RAM) -------------------------------------------------------------------------------- Build resources: - - 13.24GB of memory (42.7% of 31.00GB system memory, determined at start) + - 13.24GB of memory (42.7% of system memory, reason: enough available) - 16 thread(s) (100.0% of 16 available processor(s), determined at start) [2/8] Performing analysis... [****] (4.5s @ 0.54GB) 3,163 reachable types (72.5% of 4,364 total) @@ -142,12 +142,13 @@ The `NATIVE_IMAGE_OPTIONS` environment variable is designed to be used by users, #### Build Resources The memory limit and number of threads used by the build process. -More precisely, the memory limit of the Java heap, so actual memory consumption can be even higher. +More precisely, the memory limit of the Java heap, so actual memory consumption can be higher. Please check the [peak RSS](#glossary-peak-rss) reported at the end of the build to understand how much memory was actually used. -By default, the build process tries to only use free memory (to avoid memory pressure on the build machine), and never more than 32GB of memory. -If less than 8GB of memory are free, the build process falls back to use 85% of total memory. +By default, the build process will use up to 85% of system memory in containers or CI environments, but never more than 32GB of memory. +Otherwise, it tries to use available memory to avoid memory pressure on developer machines. +If less than 8GB of memory are available, the build process falls back to use 85% of system memory. Therefore, consider freeing up memory if your machine is slow during a build, for example, by closing applications that you do not need. -It is possible to overwrite the default behavior, for example with `-J-XX:MaxRAMPercentage=60.0` or `-J-Xmx16g`. +It is possible to override the default behavior and set relative or absolute memory limits, for example with `-J-XX:MaxRAMPercentage=60.0` or `-J-Xmx16g`. By default, the build process uses all available processors to maximize speed, but not more than 32 threads. Use the `--parallelism` option to set the number of threads explicitly (for example, `--parallelism=4`). From e424fb5ec67631467f51f13526435b5959b3facd Mon Sep 17 00:00:00 2001 From: Fabio Niephaus Date: Fri, 10 Jan 2025 11:37:37 +0100 Subject: [PATCH 05/10] Fix and simplify `ByteFormattingUtil`. It shows KB/MB/GB but calculated KiB/MiB/GiB. --- .../oracle/svm/hosted/ByteFormattingUtil.java | 36 +++++++------------ .../oracle/svm/hosted/ProgressReporter.java | 12 +++---- 2 files changed, 18 insertions(+), 30 deletions(-) diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ByteFormattingUtil.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ByteFormattingUtil.java index 9683bdced990..e0865c6ba427 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ByteFormattingUtil.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ByteFormattingUtil.java @@ -25,35 +25,23 @@ package com.oracle.svm.hosted; public class ByteFormattingUtil { - private static final double BYTES_TO_KiB = 1024d; - private static final double BYTES_TO_MiB = 1024d * 1024d; - private static final double BYTES_TO_GiB = 1024d * 1024d * 1024d; + private static final double BYTES_TO_KB = 1000d; + private static final double BYTES_TO_MB = 1000d * 1000d; + private static final double BYTES_TO_GB = 1000d * 1000d * 1000d; public static String bytesToHuman(long bytes) { - return bytesToHuman("%4.2f", bytes); - } - - public static String bytesToHuman(String format, long bytes) { - if (bytes < BYTES_TO_KiB) { - return String.format(format, (double) bytes) + "B"; - } else if (bytes < BYTES_TO_MiB) { - return String.format(format, bytesToKiB(bytes)) + "kB"; - } else if (bytes < BYTES_TO_GiB) { - return String.format(format, bytesToMiB(bytes)) + "MB"; + if (bytes < BYTES_TO_KB) { + return toHuman(bytes, "B"); + } else if (bytes < BYTES_TO_MB) { + return toHuman(bytes / BYTES_TO_KB, "KB"); + } else if (bytes < BYTES_TO_GB) { + return toHuman(bytes / BYTES_TO_MB, "MB"); } else { - return String.format(format, bytesToGiB(bytes)) + "GB"; + return toHuman(bytes / BYTES_TO_GB, "GB"); } } - static double bytesToKiB(long bytes) { - return bytes / BYTES_TO_KiB; - } - - static double bytesToGiB(long bytes) { - return bytes / BYTES_TO_GiB; - } - - static double bytesToMiB(long bytes) { - return bytes / BYTES_TO_MiB; + private static String toHuman(double value, String unit) { + return "%.2f%s".formatted(value, unit); } } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporter.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporter.java index efb2b2893003..8c59b7a52c0d 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporter.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporter.java @@ -447,7 +447,7 @@ private void printResourceInfo() { l().printLineSeparator(); l().yellowBold().doclink("Build resources", "#glossary-build-resources").a(":").reset().println(); - l().a(" - %.2fGB of memory (%.1f%% of system memory, reason: %s)", ByteFormattingUtil.bytesToGiB(maxMemory), Utils.toPercentage(maxMemory, totalMemorySize), memoryUsageReason).println(); + l().a(" - %s of memory (%.1f%% of system memory, reason: %s)", ByteFormattingUtil.bytesToHuman(maxMemory), Utils.toPercentage(maxMemory, totalMemorySize), memoryUsageReason).println(); l().a(" - %s thread(s) (%.1f%% of %s available processor(s), %s)", maxNumberOfThreads, Utils.toPercentage(maxNumberOfThreads, availableProcessors), availableProcessors, maxNumberOfThreadsSuffix).println(); } @@ -838,7 +838,7 @@ private void printResourceStatistics() { .doclink("GCs", "#glossary-garbage-collections"); long peakRSS = ProgressReporterCHelper.getPeakRSS(); if (peakRSS >= 0) { - p.a(" | ").doclink("Peak RSS", "#glossary-peak-rss").a(": ").a("%.2fGB", ByteFormattingUtil.bytesToGiB(peakRSS)); + p.a(" | ").doclink("Peak RSS", "#glossary-peak-rss").a(": ").a(ByteFormattingUtil.bytesToHuman(peakRSS)); } recordJsonMetric(ResourceUsageKey.PEAK_RSS, (peakRSS >= 0 ? peakRSS : UNAVAILABLE_METRIC)); long processCPUTime = getOperatingSystemMXBean().getProcessCpuTime(); @@ -863,7 +863,7 @@ private void checkForExcessiveGarbageCollection() { .a(": %.1fs spent in %d GCs during the last stage, taking up %.2f%% of the time.", Utils.millisToSeconds(gcTimeDeltaMillis), currentGCStats.totalCount - lastGCStats.totalCount, ratio * 100) .println(); - l().a(" Please ensure more than %.2fGB of memory is available for Native Image", ByteFormattingUtil.bytesToGiB(ProgressReporterCHelper.getPeakRSS())).println(); + l().a(" Please ensure more than %s of memory is available for Native Image", ByteFormattingUtil.bytesToHuman(ProgressReporterCHelper.getPeakRSS())).println(); l().a(" to reduce GC overhead and improve image build time.").println(); } lastGCStats = currentGCStats; @@ -899,8 +899,8 @@ private static double nanosToSeconds(double nanos) { return nanos / NANOS_TO_SECONDS; } - private static double getUsedMemory() { - return ByteFormattingUtil.bytesToGiB(Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()); + private static String getUsedMemory() { + return ByteFormattingUtil.bytesToHuman(Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()); } private static String stringFilledWith(int size, String fill) { @@ -1229,7 +1229,7 @@ void end(double totalTime) { a("]").reset(); } - String suffix = String.format("(%.1fs @ %.2fGB)", Utils.millisToSeconds(totalTime), Utils.getUsedMemory()); + String suffix = String.format("(%.1fs @ %s)", Utils.millisToSeconds(totalTime), Utils.getUsedMemory()); int textLength = getCurrentTextLength(); // TODO: `assert textLength > 0;` should be used here but tests do not start stages // properly (GR-35721) From 1cb2660a8d735fb486444e0240878cd89347adef Mon Sep 17 00:00:00 2001 From: Fabio Niephaus Date: Fri, 10 Jan 2025 12:28:37 +0100 Subject: [PATCH 06/10] Re-use `JVM.isContainerized()`. --- substratevm/mx.substratevm/suite.py | 3 +++ .../src/com/oracle/svm/driver/MemoryUtil.java | 19 +++++-------------- 2 files changed, 8 insertions(+), 14 deletions(-) diff --git a/substratevm/mx.substratevm/suite.py b/substratevm/mx.substratevm/suite.py index c1076f561a03..731fea0affcd 100644 --- a/substratevm/mx.substratevm/suite.py +++ b/substratevm/mx.substratevm/suite.py @@ -974,6 +974,9 @@ "java.base" : [ "jdk.internal.jimage", ], + "jdk.jfr": [ + "jdk.jfr.internal", + ], }, "checkstyle": "com.oracle.svm.hosted", "workingSets": "SVM", diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/MemoryUtil.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/MemoryUtil.java index a9fc9c34afd4..8e24e61e2ee9 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/MemoryUtil.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/MemoryUtil.java @@ -28,7 +28,6 @@ import java.io.InputStreamReader; import java.lang.management.ManagementFactory; import java.nio.file.Files; -import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; import java.util.List; @@ -146,19 +145,11 @@ private static List determineReasonableMaxRAMPercentage(Function s.contains("docker") || s.contains("kubepods") || s.contains("containerd")); - } catch (Exception e) { - } - return false; + /* + * [GR-55515]: Using shouldInstrument() as a workaround only to access isContainerized(). + * After dropping JDK 21, use jdk.jfr.internal.JVM.isContainerized() directly. + */ + return jdk.jfr.internal.Utils.shouldInstrument(false, ""); } private static double getAvailableMemorySize() { From 11ee99acaf84625aca01392c236f2e7a586b0fa5 Mon Sep 17 00:00:00 2001 From: Fabio Niephaus Date: Fri, 10 Jan 2025 12:33:11 +0100 Subject: [PATCH 07/10] Make `$CI=false` work as expected. --- docs/reference-manual/native-image/BuildOutput.md | 2 +- .../src/com/oracle/svm/core/SubstrateUtil.java | 4 ++-- .../src/com/oracle/svm/driver/MemoryUtil.java | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/reference-manual/native-image/BuildOutput.md b/docs/reference-manual/native-image/BuildOutput.md index 58d5eb84c87b..1a56f5a165e2 100644 --- a/docs/reference-manual/native-image/BuildOutput.md +++ b/docs/reference-manual/native-image/BuildOutput.md @@ -144,7 +144,7 @@ The memory limit and number of threads used by the build process. More precisely, the memory limit of the Java heap, so actual memory consumption can be higher. Please check the [peak RSS](#glossary-peak-rss) reported at the end of the build to understand how much memory was actually used. -By default, the build process will use up to 85% of system memory in containers or CI environments, but never more than 32GB of memory. +By default, the build process will use up to 85% of system memory in containers or CI environments (when the `$CI` environment variable is set), but never more than 32GB of memory. Otherwise, it tries to use available memory to avoid memory pressure on developer machines. If less than 8GB of memory are available, the build process falls back to use 85% of system memory. Therefore, consider freeing up memory if your machine is slow during a build, for example, by closing applications that you do not need. diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateUtil.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateUtil.java index d7459d547695..399ad0d24d2d 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateUtil.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateUtil.java @@ -38,7 +38,6 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; -import jdk.graal.compiler.word.Word; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; import org.graalvm.nativeimage.c.type.CCharPointer; @@ -60,6 +59,7 @@ import jdk.graal.compiler.java.LambdaUtils; import jdk.graal.compiler.nodes.BreakpointNode; import jdk.graal.compiler.util.Digest; +import jdk.graal.compiler.word.Word; import jdk.vm.ci.meta.ResolvedJavaMethod; import jdk.vm.ci.meta.ResolvedJavaType; import jdk.vm.ci.meta.Signature; @@ -112,7 +112,7 @@ private static boolean isTTY() { } public static boolean isRunningInCI() { - return !isTTY() || System.getenv("CI") != null; + return !isTTY() || "true".equals(System.getenv("CI")); } /** diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/MemoryUtil.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/MemoryUtil.java index 8e24e61e2ee9..8f4b692c1ad6 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/MemoryUtil.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/MemoryUtil.java @@ -106,7 +106,7 @@ private static List determineReasonableMaxRAMPercentage(Function Date: Fri, 10 Jan 2025 17:45:04 +0100 Subject: [PATCH 08/10] Address reviewer feedback. --- .../native-image/BuildOutput.md | 2 +- .../com/oracle/svm/core/SubstrateUtil.java | 6 ++++- .../src/com/oracle/svm/driver/MemoryUtil.java | 24 +++++++++++++++---- 3 files changed, 26 insertions(+), 6 deletions(-) diff --git a/docs/reference-manual/native-image/BuildOutput.md b/docs/reference-manual/native-image/BuildOutput.md index 1a56f5a165e2..290a6d4e5ed4 100644 --- a/docs/reference-manual/native-image/BuildOutput.md +++ b/docs/reference-manual/native-image/BuildOutput.md @@ -144,7 +144,7 @@ The memory limit and number of threads used by the build process. More precisely, the memory limit of the Java heap, so actual memory consumption can be higher. Please check the [peak RSS](#glossary-peak-rss) reported at the end of the build to understand how much memory was actually used. -By default, the build process will use up to 85% of system memory in containers or CI environments (when the `$CI` environment variable is set), but never more than 32GB of memory. +By default, the build process will use up to 85% of system memory in containers or CI environments (when the `$CI` environment variable is set to `true`), but never more than 32GB of memory. Otherwise, it tries to use available memory to avoid memory pressure on developer machines. If less than 8GB of memory are available, the build process falls back to use 85% of system memory. Therefore, consider freeing up memory if your machine is slow during a build, for example, by closing applications that you do not need. diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateUtil.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateUtil.java index 399ad0d24d2d..0bdbb2744b49 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateUtil.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateUtil.java @@ -112,7 +112,11 @@ private static boolean isTTY() { } public static boolean isRunningInCI() { - return !isTTY() || "true".equals(System.getenv("CI")); + return !isTTY() || isCISet(); + } + + public static boolean isCISet() { + return "true".equals(System.getenv("CI")); } /** diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/MemoryUtil.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/MemoryUtil.java index 8f4b692c1ad6..ae07b4fc0e9e 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/MemoryUtil.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/MemoryUtil.java @@ -37,8 +37,13 @@ import com.oracle.svm.core.OS; import com.oracle.svm.core.SubstrateOptions; +import com.oracle.svm.core.SubstrateUtil; import com.oracle.svm.core.util.ExitStatus; +import com.oracle.svm.core.util.VMError; import com.oracle.svm.driver.NativeImage.NativeImageError; +import com.oracle.svm.util.ReflectionUtil; + +import jdk.graal.compiler.serviceprovider.JavaVersionUtil; class MemoryUtil { private static final long KiB_TO_BYTES = 1024L; @@ -106,7 +111,7 @@ private static List determineReasonableMaxRAMPercentage(Function determineReasonableMaxRAMPercentage(Function Date: Fri, 10 Jan 2025 17:53:49 +0100 Subject: [PATCH 09/10] Improve heuristic texts. --- docs/reference-manual/native-image/BuildOutput.md | 2 +- .../src/com/oracle/svm/driver/MemoryUtil.java | 10 +++++----- .../src/com/oracle/svm/hosted/ProgressReporter.java | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/reference-manual/native-image/BuildOutput.md b/docs/reference-manual/native-image/BuildOutput.md index 290a6d4e5ed4..711858237512 100644 --- a/docs/reference-manual/native-image/BuildOutput.md +++ b/docs/reference-manual/native-image/BuildOutput.md @@ -30,7 +30,7 @@ GraalVM Native Image: Generating 'helloworld' (executable)... Garbage collector: Serial GC (max heap size: 80% of RAM) -------------------------------------------------------------------------------- Build resources: - - 13.24GB of memory (42.7% of system memory, reason: enough available) + - 13.24GB of memory (42.7% of system memory, using available memory) - 16 thread(s) (100.0% of 16 available processor(s), determined at start) [2/8] Performing analysis... [****] (4.5s @ 0.54GB) 3,163 reachable types (72.5% of 4,364 total) diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/MemoryUtil.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/MemoryUtil.java index ae07b4fc0e9e..017d121ea034 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/MemoryUtil.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/MemoryUtil.java @@ -57,7 +57,7 @@ class MemoryUtil { private static final double DEDICATED_MODE_TOTAL_MEMORY_RATIO = 0.85D; /* If available memory is below 8GiB, fall back to dedicated mode. */ - private static final long MIN_AVAILABLE_MEMORY_THRESHOLD = 8L * GiB_TO_BYTES; + private static final int MIN_AVAILABLE_MEMORY_THRESHOLD_GB = 8; /* * Builder uses at most 32GB to avoid disabling compressed oops (UseCompressedOops). @@ -113,7 +113,7 @@ private static List determineReasonableMaxRAMPercentage(Function determineReasonableMaxRAMPercentage(Function= MIN_AVAILABLE_MEMORY_THRESHOLD) { - memoryUsageReason = "enough available"; + if (reasonableMaxMemorySize >= MIN_AVAILABLE_MEMORY_THRESHOLD_GB * GiB_TO_BYTES) { + memoryUsageReason = "using available memory"; } else { // fall back to dedicated mode - memoryUsageReason = "not enough available"; + memoryUsageReason = "less than " + MIN_AVAILABLE_MEMORY_THRESHOLD_GB + "GB of memory available"; reasonableMaxMemorySize = dedicatedMemorySize; } } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporter.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporter.java index 8c59b7a52c0d..8d99206e8c04 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporter.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ProgressReporter.java @@ -447,7 +447,7 @@ private void printResourceInfo() { l().printLineSeparator(); l().yellowBold().doclink("Build resources", "#glossary-build-resources").a(":").reset().println(); - l().a(" - %s of memory (%.1f%% of system memory, reason: %s)", ByteFormattingUtil.bytesToHuman(maxMemory), Utils.toPercentage(maxMemory, totalMemorySize), memoryUsageReason).println(); + l().a(" - %s of memory (%.1f%% of system memory, %s)", ByteFormattingUtil.bytesToHuman(maxMemory), Utils.toPercentage(maxMemory, totalMemorySize), memoryUsageReason).println(); l().a(" - %s thread(s) (%.1f%% of %s available processor(s), %s)", maxNumberOfThreads, Utils.toPercentage(maxNumberOfThreads, availableProcessors), availableProcessors, maxNumberOfThreadsSuffix).println(); } From 626346abd7fd12c84e54a3ceeb67743ee7b818a5 Mon Sep 17 00:00:00 2001 From: Fabio Niephaus Date: Fri, 10 Jan 2025 18:13:27 +0100 Subject: [PATCH 10/10] Lookup `isContainerized` at build time. --- .../src/com/oracle/svm/driver/MemoryUtil.java | 46 +++++++++++-------- 1 file changed, 28 insertions(+), 18 deletions(-) diff --git a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/MemoryUtil.java b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/MemoryUtil.java index 017d121ea034..d65fb88b141b 100644 --- a/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/MemoryUtil.java +++ b/substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/MemoryUtil.java @@ -27,6 +27,7 @@ import java.io.BufferedReader; import java.io.InputStreamReader; import java.lang.management.ManagementFactory; +import java.lang.reflect.Method; import java.nio.file.Files; import java.nio.file.Paths; import java.util.ArrayList; @@ -65,6 +66,23 @@ class MemoryUtil { */ private static final long MAX_HEAP_BYTES = 32_000_000_000L; + private static final Method IS_CONTAINERIZED_METHOD; + private static final Object IS_CONTAINERIZED_RECEIVER; + + static { + IS_CONTAINERIZED_METHOD = ReflectionUtil.lookupMethod(jdk.jfr.internal.JVM.class, "isContainerized"); + if (JavaVersionUtil.JAVA_SPEC == 21) { // non-static + var jvmField = ReflectionUtil.lookupField(jdk.jfr.internal.JVM.class, "jvm"); + try { + IS_CONTAINERIZED_RECEIVER = jvmField.get(null); + } catch (IllegalAccessException e) { + throw VMError.shouldNotReachHere(e); + } + } else { + IS_CONTAINERIZED_RECEIVER = null; // static + } + } + public static List determineMemoryFlags(NativeImage.HostFlags hostFlags) { List flags = new ArrayList<>(); if (hostFlags.hasUseParallelGC()) { @@ -77,9 +95,9 @@ public static List determineMemoryFlags(NativeImage.HostFlags hostFlags) * -XX:InitialRAMPercentage or -Xms. */ if (hostFlags.hasMaxRAMPercentage()) { - flags.addAll(determineReasonableMaxRAMPercentage(value -> "-XX:MaxRAMPercentage=" + value)); + flags.addAll(determineMemoryUsageFlags(value -> "-XX:MaxRAMPercentage=" + value)); } else if (hostFlags.hasMaximumHeapSizePercent()) { - flags.addAll(determineReasonableMaxRAMPercentage(value -> "-XX:MaximumHeapSizePercent=" + value.intValue())); + flags.addAll(determineMemoryUsageFlags(value -> "-XX:MaximumHeapSizePercent=" + value.intValue())); } if (hostFlags.hasGCTimeRatio()) { /* @@ -98,13 +116,12 @@ public static List determineMemoryFlags(NativeImage.HostFlags hostFlags) } /** - * Returns a percentage (0.0-100.0) to be used as a value for the -XX:MaxRAMPercentage or - * -XX:MaximumHeapSizePercent flags of the builder process. Dedicated mode uses a fixed - * percentage of total memory and is the default in containers. Shared mode tries to use - * available memory to reduce memory pressure on the host machine. Note that this method uses - * OperatingSystemMXBean, which is container-aware. + * Returns memory usage flags for the build process. Dedicated mode uses a fixed percentage of + * total memory and is the default in containers. Shared mode tries to use available memory to + * reduce memory pressure on the host machine. Note that this method uses OperatingSystemMXBean, + * which is container-aware. */ - private static List determineReasonableMaxRAMPercentage(Function toMemoryFlag) { + private static List determineMemoryUsageFlags(Function toMemoryFlag) { var osBean = (com.sun.management.OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean(); final double totalMemorySize = osBean.getTotalMemorySize(); final double dedicatedMemorySize = totalMemorySize * DEDICATED_MODE_TOTAL_MEMORY_RATIO; @@ -151,18 +168,11 @@ private static List determineReasonableMaxRAMPercentage(Function