[GR-52400] Revise heuristic for memory usage of Native Image build process.

docs/reference-manual/native-image/BuildOutput.md

@@ -30,7 +30,7 @@ GraalVM Native Image: Generating 'helloworld' (executable)...
  Garbage collector: Serial GC (max heap size: 80% of RAM)
 --------------------------------------------------------------------------------
  Build resources:
- - 13.24GB of memory (42.7% of 31.00GB system memory, determined at start)
+ - 13.24GB of memory (42.7% of system memory, reason: enough available)
  - 16 thread(s) (100.0% of 16 available processor(s), determined at start)
 [2/8] Performing analysis...  [****]                             (4.5s @ 0.54GB)
     3,163 reachable types   (72.5% of    4,364 total)
@@ -142,12 +142,13 @@ The `NATIVE_IMAGE_OPTIONS` environment variable is designed to be used by users,
 #### <a name="glossary-build-resources"></a>Build Resources
 The memory limit and number of threads used by the build process.
 
-More precisely, the memory limit of the Java heap, so actual memory consumption can be even higher.
+More precisely, the memory limit of the Java heap, so actual memory consumption can be higher.
 Please check the [peak RSS](#glossary-peak-rss) reported at the end of the build to understand how much memory was actually used.
-By default, the build process tries to only use free memory (to avoid memory pressure on the build machine), and never more than 32GB of memory.
-If less than 8GB of memory are free, the build process falls back to use 85% of total memory.
+By default, the build process will use up to 85% of system memory in containers or CI environments (when the `$CI` environment variable is set), but never more than 32GB of memory.
+Otherwise, it tries to use available memory to avoid memory pressure on developer machines.
+If less than 8GB of memory are available, the build process falls back to use 85% of system memory.
 Therefore, consider freeing up memory if your machine is slow during a build, for example, by closing applications that you do not need.
-It is possible to overwrite the default behavior, for example with `-J-XX:MaxRAMPercentage=60.0` or `-J-Xmx16g`.
+It is possible to override the default behavior and set relative or absolute memory limits, for example with `-J-XX:MaxRAMPercentage=60.0` or `-J-Xmx16g`.
 
 By default, the build process uses all available processors to maximize speed, but not more than 32 threads.
 Use the `--parallelism` option to set the number of threads explicitly (for example, `--parallelism=4`).

substratevm/mx.substratevm/suite.py

@@ -974,6 +974,9 @@
                 "java.base" : [
                     "jdk.internal.jimage",
                 ],
+                "jdk.jfr": [
+                    "jdk.jfr.internal",
+                ],
             },
             "checkstyle": "com.oracle.svm.hosted",
             "workingSets": "SVM",

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateOptions.java

@@ -738,6 +738,8 @@ public static boolean hasColorsEnabled(OptionValues values) {
     @Option(help = "Internal option to forward the value of " + NATIVE_IMAGE_OPTIONS_ENV_VAR)//
     public static final HostedOptionKey<String> BuildOutputNativeImageOptionsEnvVarValue = new HostedOptionKey<>(null);
 
+    public static final String BUILD_MEMORY_USAGE_REASON_TEXT_PROPERTY = "svm.build.memoryUsageReasonText";
+
     /*
      * Object and array allocation options.
      */

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateUtil.java

@@ -38,7 +38,6 @@
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
-import jdk.graal.compiler.word.Word;
 import org.graalvm.nativeimage.Platform;
 import org.graalvm.nativeimage.Platforms;
 import org.graalvm.nativeimage.c.type.CCharPointer;
@@ -60,6 +59,7 @@
 import jdk.graal.compiler.java.LambdaUtils;
 import jdk.graal.compiler.nodes.BreakpointNode;
 import jdk.graal.compiler.util.Digest;
+import jdk.graal.compiler.word.Word;
 import jdk.vm.ci.meta.ResolvedJavaMethod;
 import jdk.vm.ci.meta.ResolvedJavaType;
 import jdk.vm.ci.meta.Signature;
@@ -112,7 +112,7 @@ private static boolean isTTY() {
     }
 
     public static boolean isRunningInCI() {
-        return !isTTY() || System.getenv("CI") != null;
+        return !isTTY() || "true".equals(System.getenv("CI"));
     }
 
     /**

substratevm/src/com.oracle.svm.driver/src/com/oracle/svm/driver/MemoryUtil.java

@@ -24,10 +24,19 @@
  */
 package com.oracle.svm.driver;
 
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
 import java.lang.management.ManagementFactory;
+import java.nio.file.Files;
+import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.function.Function;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
+import com.oracle.svm.core.OS;
+import com.oracle.svm.core.SubstrateOptions;
 import com.oracle.svm.core.util.ExitStatus;
 import com.oracle.svm.driver.NativeImage.NativeImageError;
 
@@ -39,10 +48,12 @@ class MemoryUtil {
     /* Builder needs at least 512MiB for building a helloworld in a reasonable amount of time. */
     private static final long MIN_HEAP_BYTES = 512L * MiB_TO_BYTES;
 
-    /* If free memory is below 8GiB, use 85% of total system memory (e.g., 7GiB * 85% ~ 6GiB). */
-    private static final long DEDICATED_MODE_THRESHOLD = 8L * GiB_TO_BYTES;
+    /* Use 85% of total system memory (e.g., 7GiB * 85% ~ 6GiB) in dedicated mode. */
     private static final double DEDICATED_MODE_TOTAL_MEMORY_RATIO = 0.85D;
 
+    /* If available memory is below 8GiB, fall back to dedicated mode. */
+    private static final long MIN_AVAILABLE_MEMORY_THRESHOLD = 8L * GiB_TO_BYTES;
+
     /*
      * Builder uses at most 32GB to avoid disabling compressed oops (UseCompressedOops).
      * Deliberately use GB (not GiB) to stay well below 32GiB when relative maximum is calculated.
@@ -61,9 +72,9 @@ public static List<String> determineMemoryFlags(NativeImage.HostFlags hostFlags)
          * -XX:InitialRAMPercentage or -Xms.
          */
         if (hostFlags.hasMaxRAMPercentage()) {
-            flags.add("-XX:MaxRAMPercentage=" + determineReasonableMaxRAMPercentage());
+            flags.addAll(determineReasonableMaxRAMPercentage(value -> "-XX:MaxRAMPercentage=" + value));
         } else if (hostFlags.hasMaximumHeapSizePercent()) {
-            flags.add("-XX:MaximumHeapSizePercent=" + (int) determineReasonableMaxRAMPercentage());
+            flags.addAll(determineReasonableMaxRAMPercentage(value -> "-XX:MaximumHeapSizePercent=" + value.intValue()));
         }
         if (hostFlags.hasGCTimeRatio()) {
             /*
@@ -82,23 +93,40 @@ public static List<String> determineMemoryFlags(NativeImage.HostFlags hostFlags)
     }
 
     /**
-     * Returns a percentage (0.0-100.0) to be used as a value for the -XX:MaxRAMPercentage flag of
-     * the builder process. Prefer free memory over total memory to reduce memory pressure on the
-     * host machine. Note that this method uses OperatingSystemMXBean, which is container-aware.
+     * Returns a percentage (0.0-100.0) to be used as a value for the -XX:MaxRAMPercentage or
+     * -XX:MaximumHeapSizePercent flags of the builder process. Dedicated mode uses a fixed
+     * percentage of total memory and is the default in containers. Shared mode tries to use
+     * available memory to reduce memory pressure on the host machine. Note that this method uses
+     * OperatingSystemMXBean, which is container-aware.
      */
-    private static double determineReasonableMaxRAMPercentage() {
+    private static List<String> determineReasonableMaxRAMPercentage(Function<Double, String> toMemoryFlag) {
         var osBean = (com.sun.management.OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean();
         final double totalMemorySize = osBean.getTotalMemorySize();
-        double reasonableMaxMemorySize = osBean.getFreeMemorySize();
+        final double dedicatedMemorySize = totalMemorySize * DEDICATED_MODE_TOTAL_MEMORY_RATIO;
 
-        if (reasonableMaxMemorySize < DEDICATED_MODE_THRESHOLD) {
-            /*
-             * When free memory is low, for example in memory-constrained environments or when a
-             * good amount of memory is used for caching, use a fixed percentage of total memory
-             * rather than free memory. In containerized environments, builds are expected to run
-             * more or less exclusively (builder + driver + optional Gradle/Maven process).
-             */
-            reasonableMaxMemorySize = totalMemorySize * DEDICATED_MODE_TOTAL_MEMORY_RATIO;
+        String memoryUsageReason = "unknown";
+        final boolean isDedicatedMemoryUsage;
+        if ("true".equals(System.getenv("CI"))) {
+            isDedicatedMemoryUsage = true;
+            memoryUsageReason = "$CI set";
+        } else if (isContainerized()) {
+            isDedicatedMemoryUsage = true;
+            memoryUsageReason = "in container";
+        } else {
+            isDedicatedMemoryUsage = false;
+        }
+
+        double reasonableMaxMemorySize;
+        if (isDedicatedMemoryUsage) {
+            reasonableMaxMemorySize = dedicatedMemorySize;
+        } else {
+            reasonableMaxMemorySize = getAvailableMemorySize();
+            if (reasonableMaxMemorySize >= MIN_AVAILABLE_MEMORY_THRESHOLD) {
+                memoryUsageReason = "enough available";
+            } else { // fall back to dedicated mode
+                memoryUsageReason = "not enough available";
+                reasonableMaxMemorySize = dedicatedMemorySize;
+            }
         }
 
         if (reasonableMaxMemorySize < MIN_HEAP_BYTES) {
@@ -111,6 +139,135 @@ private static double determineReasonableMaxRAMPercentage() {
         /* Ensure max memory size does not exceed upper limit. */
         reasonableMaxMemorySize = Math.min(reasonableMaxMemorySize, MAX_HEAP_BYTES);
 
-        return reasonableMaxMemorySize / totalMemorySize * 100;
+        double reasonableMaxRamPercentage = reasonableMaxMemorySize / totalMemorySize * 100;
+        return List.of(toMemoryFlag.apply(reasonableMaxRamPercentage),
+                        "-D" + SubstrateOptions.BUILD_MEMORY_USAGE_REASON_TEXT_PROPERTY + "=" + memoryUsageReason);
+    }
+
+    private static boolean isContainerized() {
+        /*
+         * [GR-55515]: Using shouldInstrument() as a workaround only to access isContainerized().
+         * After dropping JDK 21, use jdk.jfr.internal.JVM.isContainerized() directly.
+         */
+        return jdk.jfr.internal.Utils.shouldInstrument(false, "");
+    }
+
+    private static double getAvailableMemorySize() {
+        return switch (OS.getCurrent()) {
+            case LINUX -> getAvailableMemorySizeLinux();
+            case DARWIN -> getAvailableMemorySizeDarwin();
+            case WINDOWS -> getAvailableMemorySizeWindows();
+        };
+    }
+
+    /**
+     * Returns the total amount of available memory in bytes on Linux based on
+     * <code>/proc/meminfo</code>, otherwise <code>-1</code>. Note that this metric is not
+     * container-aware (does not take cgroups into account) and may report available memory of the
+     * host.
+     *
+     * @see <a href=
+     *      "https://github.com/torvalds/linux/blob/865fdb08197e657c59e74a35fa32362b12397f58/mm/page_alloc.c#L5137">page_alloc.c#L5137</a>
+     */
+    private static long getAvailableMemorySizeLinux() {
+        try {
+            String memAvailableLine = Files.readAllLines(Paths.get("/proc/meminfo")).stream().filter(l -> l.startsWith("MemAvailable")).findFirst().orElse("");
+            Matcher m = Pattern.compile("^MemAvailable:\\s+(\\d+) kB").matcher(memAvailableLine);
+            if (m.matches()) {
+                return Long.parseLong(m.group(1)) * KiB_TO_BYTES;
+            }
+        } catch (Exception e) {
+        }
+        return -1;
+    }
+
+    /**
+     * Returns the total amount of available memory in bytes on Darwin based on
+     * <code>vm_stat</code>, otherwise <code>-1</code>.
+     *
+     * @see <a href=
+     *      "https://opensource.apple.com/source/system_cmds/system_cmds-496/vm_stat.tproj/vm_stat.c.auto.html">vm_stat.c</a>
+     */
+    private static long getAvailableMemorySizeDarwin() {
+        try {
+            Process p = Runtime.getRuntime().exec(new String[]{"vm_stat"});
+            try (BufferedReader reader = new BufferedReader(new InputStreamReader(p.getInputStream()))) {
+                String line1 = reader.readLine();
+                if (line1 == null) {
+                    return -1;
+                }
+                Matcher m1 = Pattern.compile("^Mach Virtual Memory Statistics: \\(page size of (\\d+) bytes\\)").matcher(line1);
+                long pageSize = -1;
+                if (m1.matches()) {
+                    pageSize = Long.parseLong(m1.group(1));
+                }
+                if (pageSize <= 0) {
+                    return -1;
+                }
+                String line2 = reader.readLine();
+                Matcher m2 = Pattern.compile("^Pages free:\\s+(\\d+).").matcher(line2);
+                long freePages = -1;
+                if (m2.matches()) {
+                    freePages = Long.parseLong(m2.group(1));
+                }
+                if (freePages <= 0) {
+                    return -1;
+                }
+                String line3 = reader.readLine();
+                if (!line3.startsWith("Pages active")) {
+                    return -1;
+                }
+                String line4 = reader.readLine();
+                Matcher m4 = Pattern.compile("^Pages inactive:\\s+(\\d+).").matcher(line4);
+                long inactivePages = -1;
+                if (m4.matches()) {
+                    inactivePages = Long.parseLong(m4.group(1));
+                }
+                if (inactivePages <= 0) {
+                    return -1;
+                }
+                assert freePages > 0 && inactivePages > 0 && pageSize > 0;
+                return (freePages + inactivePages) * pageSize;
+            } finally {
+                p.waitFor();
+            }
+        } catch (Exception e) {
+        }
+        return -1;
+    }
+
+    /**
+     * Returns the total amount of available memory in bytes on Windows based on <code>wmic</code>,
+     * otherwise <code>-1</code>.
+     *
+     * @see <a href=
+     *      "https://learn.microsoft.com/en-us/windows/win32/cimwin32prov/win32-operatingsystem">Win32_OperatingSystem
+     *      class</a>
+     */
+    private static long getAvailableMemorySizeWindows() {
+        try {
+            Process p = Runtime.getRuntime().exec(new String[]{"cmd.exe", "/c", "wmic", "OS", "get", "FreePhysicalMemory"});
+            try (BufferedReader reader = new BufferedReader(new InputStreamReader(p.getInputStream()))) {
+                String line1 = reader.readLine();
+                if (line1 == null || !line1.startsWith("FreePhysicalMemory")) {
+                    return -1;
+                }
+                String line2 = reader.readLine();
+                if (line2 == null) {
+                    return -1;
+                }
+                String line3 = reader.readLine();
+                if (line3 == null) {
+                    return -1;
+                }
+                Matcher m = Pattern.compile("^(\\d+)\\s+").matcher(line3);
+                if (m.matches()) {
+                    return Long.parseLong(m.group(1)) * KiB_TO_BYTES;
+                }
+            }
+            p.waitFor();
+        } catch (Exception e) {
+        }
+        return -1;
     }
 }

substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ByteFormattingUtil.java

@@ -25,35 +25,23 @@
 package com.oracle.svm.hosted;
 
 public class ByteFormattingUtil {
-    private static final double BYTES_TO_KiB = 1024d;
-    private static final double BYTES_TO_MiB = 1024d * 1024d;
-    private static final double BYTES_TO_GiB = 1024d * 1024d * 1024d;
+    private static final double BYTES_TO_KB = 1000d;
+    private static final double BYTES_TO_MB = 1000d * 1000d;
+    private static final double BYTES_TO_GB = 1000d * 1000d * 1000d;
 
     public static String bytesToHuman(long bytes) {
-        return bytesToHuman("%4.2f", bytes);
-    }
-
-    public static String bytesToHuman(String format, long bytes) {
-        if (bytes < BYTES_TO_KiB) {
-            return String.format(format, (double) bytes) + "B";
-        } else if (bytes < BYTES_TO_MiB) {
-            return String.format(format, bytesToKiB(bytes)) + "kB";
-        } else if (bytes < BYTES_TO_GiB) {
-            return String.format(format, bytesToMiB(bytes)) + "MB";
+        if (bytes < BYTES_TO_KB) {
+            return toHuman(bytes, "B");
+        } else if (bytes < BYTES_TO_MB) {
+            return toHuman(bytes / BYTES_TO_KB, "KB");
+        } else if (bytes < BYTES_TO_GB) {
+            return toHuman(bytes / BYTES_TO_MB, "MB");
         } else {
-            return String.format(format, bytesToGiB(bytes)) + "GB";
+            return toHuman(bytes / BYTES_TO_GB, "GB");
         }
     }
 
-    static double bytesToKiB(long bytes) {
-        return bytes / BYTES_TO_KiB;
-    }
-
-    static double bytesToGiB(long bytes) {
-        return bytes / BYTES_TO_GiB;
-    }
-
-    static double bytesToMiB(long bytes) {
-        return bytes / BYTES_TO_MiB;
+    private static String toHuman(double value, String unit) {
+        return "%.2f%s".formatted(value, unit);
     }
 }

substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageOptions.java

@@ -186,11 +186,11 @@ public static CStandards getCStandard() {
     }
 
     /**
-     * Configures the number of threads of the common pool (see driver).
+     * Configures the number of threads of the common pool.
      */
     private static final String PARALLELISM_OPTION_NAME = "parallelism";
     @APIOption(name = PARALLELISM_OPTION_NAME)//
-    @Option(help = "The maximum number of threads to use concurrently during native image generation.")//
+    @Option(help = "The maximum number of threads to use concurrently by the build process.")//
     public static final HostedOptionKey<Integer> NumberOfThreads = new HostedOptionKey<>(Math.max(1, Math.min(Runtime.getRuntime().availableProcessors(), 32)), key -> {
         int numberOfThreads = key.getValue();
         if (numberOfThreads < 1) {