Run native tests without running tests on JVM first

common/utils/src/main/java/org/graalvm/buildtools/utils/SharedConstants.java

@@ -78,6 +78,7 @@ public interface SharedConstants {
     String AGENT_OUTPUT_DIRECTORY_MARKER = "{output_dir}";
     String AGENT_OUTPUT_DIRECTORY_OPTION = "config-output-dir=";
     String METADATA_REPO_URL_TEMPLATE = "https://github.com/oracle/graalvm-reachability-metadata/releases/download/%1$s/graalvm-reachability-metadata-%1$s.zip";
+    String SKIP_JVM_TESTS = "skipJVMTests";
     /**
      * The default metadata repository version. Maintained for backwards
      * compatibility.

native-gradle-plugin/src/functionalTest/groovy/org/graalvm/buildtools/gradle/JUnitFunctionalTests.groovy

@@ -73,4 +73,40 @@ class JUnitFunctionalTests extends AbstractFunctionalTest {
 [         0 tests failed          ]
 """.trim()
     }
+
+    def "test if JUnit support works when JVM tests are skipped"() {
+        debug=true
+        given:
+        withSample("junit-tests")
+
+        when:
+        run 'nativeTest', '-DskipJVMTests', '--info'
+
+        then:
+        tasks {
+            succeeded ':testClasses', ':nativeTestCompile', ':nativeTest'
+        }
+        outputContains "VintageTests > testEvery SKIPPED"
+        outputContains "ComplexTest > accessMethodReflectively() SKIPPED"
+        outputContains "ComplexTest > resourceTest() SKIPPED"
+        outputContains "JUnitAnnotationsTests > beforeAndAfterEachTest1() SKIPPED"
+        outputContains "OrderTests > firstTest() SKIPPED"
+        outputDoesNotContain "[junit-platform-native] WARNING: Trying to find test-ids on default locations"
+        outputContains "Running in 'test listener' mode using files matching pattern [junit-platform-unique-ids*] found in folder ["
+        outputContains """
+[        10 containers found      ]
+[         0 containers skipped    ]
+[        10 containers started    ]
+[         0 containers aborted    ]
+[        10 containers successful ]
+[         0 containers failed     ]
+[        24 tests found           ]
+[         1 tests skipped         ]
+[        23 tests started         ]
+[         0 tests aborted         ]
+[        23 tests successful      ]
+[         0 tests failed          ]
+""".trim()
+    }
+
 }

native-gradle-plugin/src/functionalTest/groovy/org/graalvm/buildtools/gradle/JavaApplicationWithTestsFunctionalTest.groovy

@@ -143,6 +143,59 @@ class JavaApplicationWithTestsFunctionalTest extends AbstractFunctionalTest {
         junitVersion = System.getProperty('versions.junit')
     }
 
+    def "can execute tests with -DskipJVMTests in a native image directly"() {
+        given:
+        withSample("java-application-with-tests")
+
+        when:
+        run 'nativeTest', '-DskipJVMTests', '--info'
+
+        then:
+        tasks {
+            succeeded ':testClasses',
+                    ':nativeTestCompile',
+                    ':test',
+                    ':nativeTest'
+            doesNotContain ':build'
+        }
+
+        then:
+        outputDoesNotContain "Running in 'test discovery' mode. Note that this is a fallback mode."
+        outputContains "Running in 'test listener' mode using files matching pattern [junit-platform-unique-ids*] found in folder ["
+        outputContains "CalculatorTest > 1 + 1 = 2 SKIPPED"
+        outputContains "CalculatorTest > 1 + 2 = 3 SKIPPED"
+
+        outputContains """
+[         3 containers found      ]
+[         0 containers skipped    ]
+[         3 containers started    ]
+[         0 containers aborted    ]
+[         3 containers successful ]
+[         0 containers failed     ]
+[         6 tests found           ]
+[         0 tests skipped         ]
+[         6 tests started         ]
+[         0 tests aborted         ]
+[         6 tests successful      ]
+[         0 tests failed          ]
+""".trim()
+
+        and:
+        def results = TestResults.from(file("build/test-results/test/TEST-org.graalvm.demo.CalculatorTest.xml"))
+        def nativeResults = TestResults.from(file("build/test-results/test-native/TEST-junit-jupiter.xml"))
+
+        results == nativeResults
+        results.with {
+            tests == 6
+            failures == 0
+            skipped == 0
+            errors == 0
+        }
+
+        where:
+        junitVersion = System.getProperty('versions.junit')
+    }
+
     @Issue("https://github.com/graalvm/native-build-tools/issues/215")
     @Unroll("can pass environment variables to native test execution with JUnit Platform #junitVersion")
     def "can pass environment variables to native test execution"() {

native-gradle-plugin/src/main/java/org/graalvm/buildtools/gradle/NativeImagePlugin.java

@@ -170,6 +170,7 @@ public class NativeImagePlugin implements Plugin<Project> {
     public static final Attribute<Boolean> JAR_ANALYSIS_ATTRIBUTE = Attribute.of("jar-analysis", Boolean.class);
 
     private static final String JUNIT_PLATFORM_LISTENERS_UID_TRACKING_ENABLED = "junit.platform.listeners.uid.tracking.enabled";
+    private static final String JUNIT_PLATFORM_DRY_RUN_ENABLED = "junit.platform.execution.dryRun.enabled";
     private static final String JUNIT_PLATFORM_LISTENERS_UID_TRACKING_OUTPUT_DIR = "junit.platform.listeners.uid.tracking.output.dir";
     private static final String REPOSITORY_COORDINATES = "org.graalvm.buildtools:graalvm-reachability-metadata:" + VersionInfo.NBT_VERSION + ":repository@zip";
     private static final String DEFAULT_URI = String.format(METADATA_REPO_URL_TEMPLATE, VersionInfo.METADATA_REPO_VERSION);
@@ -679,6 +680,16 @@ public void registerTestBinary(Project project,
             test.getOutputs().dir(testList);
             // Set system property read by the UniqueIdTrackingListener.
             test.systemProperty(JUNIT_PLATFORM_LISTENERS_UID_TRACKING_ENABLED, true);
+
+            // Set system property to skip execution of JVM tests before native tests
+            if (shouldSkipJVMTests()) {
+                if (graalExtension.getAgent().getEnabled().get()) {
+                    throw new IllegalStateException("Native Image Agent and skipJVMTests cannot be used at the same time.");
+                }
+
+                test.systemProperty(JUNIT_PLATFORM_DRY_RUN_ENABLED, true);
+            }
+
             TrackingDirectorySystemPropertyProvider directoryProvider = project.getObjects().newInstance(TrackingDirectorySystemPropertyProvider.class);
             directoryProvider.getDirectory().set(testListDirectory);
             test.getJvmArgumentProviders().add(directoryProvider);
@@ -713,6 +724,11 @@ public void registerTestBinary(Project project,
         });
     }
 
+    private boolean shouldSkipJVMTests() {
+        String option = System.getProperty(SharedConstants.SKIP_JVM_TESTS);
+        return (option != null && option.isEmpty()) || Boolean.parseBoolean(option);
+    }
+
     /**
      * Returns a provider which prefers the CLI arguments over the configured
      * extension value.

native-maven-plugin/src/functionalTest/groovy/org/graalvm/buildtools/maven/JUnitFunctionalTests.groovy

@@ -65,6 +65,36 @@ class JUnitFunctionalTests extends AbstractGraalVMMavenFunctionalTest {
 [         0 tests aborted         ]
 [        23 tests successful      ]
 [         0 tests failed          ]
+""".trim()
+    }
+
+    def "test if JUint support works when JVM tests are skipped"() {
+        withSample("junit-tests")
+
+        when:
+        mvn '-DquickBuild', '-DskipJVMTests', '-Pnative', 'test'
+
+        then:
+        buildSucceeded
+        outputDoesNotContain "[junit-platform-native] WARNING: Trying to find test-ids on default locations"
+        outputContains "[junit-platform-native] Running in 'test listener' mode"
+        outputContainsPattern "Tests run: 4, Failures: 0, Errors: 0, Skipped: 4, .* - in tests.ComplexTest"
+        outputContainsPattern "Tests run: 3, Failures: 0, Errors: 0, Skipped: 3, .* - in tests.OrderTests"
+        outputContainsPattern "Tests run: 14, Failures: 0, Errors: 0, Skipped: 14, .* - in tests.JUnitAnnotationsTests"
+        outputContainsPattern "Tests run: 3, Failures: 0, Errors: 0, Skipped: 3, .* - in tests.VintageTests"
+        outputContains """
+[        10 containers found      ]
+[         0 containers skipped    ]
+[        10 containers started    ]
+[         0 containers aborted    ]
+[        10 containers successful ]
+[         0 containers failed     ]
+[        24 tests found           ]
+[         1 tests skipped         ]
+[        23 tests started         ]
+[         0 tests aborted         ]
+[        23 tests successful      ]
+[         0 tests failed          ]
 """.trim()
     }
 }

native-maven-plugin/src/functionalTest/groovy/org/graalvm/buildtools/maven/JavaApplicationWithTestsFunctionalTest.groovy

@@ -73,6 +73,34 @@ class JavaApplicationWithTestsFunctionalTest extends AbstractGraalVMMavenFunctio
 """.trim()
     }
 
+
+    def "can run tests in a native image with the Maven plugin without running JVM tests first"() {
+        withSample("java-application-with-tests")
+
+        when:
+        mvn '-Pnative', '-DskipJVMTests', '-DquickBuild', 'test'
+
+        then:
+        buildSucceeded
+        outputContains "[junit-platform-native] Running in 'test listener' mode"
+        outputContainsPattern "Tests run: 6, Failures: 0, Errors: 0, Skipped: 6, .* - in org.graalvm.demo.CalculatorTest"
+        outputContains """
+[         3 containers found      ]
+[         0 containers skipped    ]
+[         3 containers started    ]
+[         0 containers aborted    ]
+[         3 containers successful ]
+[         0 containers failed     ]
+[         6 tests found           ]
+[         0 tests skipped         ]
+[         6 tests started         ]
+[         0 tests aborted         ]
+[         6 tests successful      ]
+[         0 tests failed          ]
+""".trim()
+    }
+
+
     def "can run tests in a native image with the Maven plugin using shading"() {
         withSample("java-application-with-tests")
 

native-maven-plugin/src/main/java/org/graalvm/buildtools/maven/MergeAgentFilesMojo.java

@@ -116,7 +116,7 @@ private void mergeForGivenDir(String agentOutputDirectory) throws MojoExecutionE
         File baseDir = new File(agentOutputDirectory);
         if (baseDir.exists()) {
             List<File> sessionDirectories = sessionDirectoriesFrom(baseDir.listFiles()).collect(Collectors.toList());
-            if (sessionDirectories.size() == 0) {
+            if (sessionDirectories.isEmpty()) {
                 sessionDirectories = Collections.singletonList(baseDir);
             }
 

native-maven-plugin/src/main/java/org/graalvm/buildtools/maven/NativeExtension.java

@@ -67,13 +67,14 @@
 import static org.graalvm.buildtools.utils.NativeImageConfigurationUtils.getNativeImage;
 
 /**
- * This extension is responsible for configuring the Surefire plugin to enable
+ * This extension is responsible for configuring the Surefire and the Failsafe plugins to enable
  * the JUnit Platform test listener and registering the native dependency transparently.
  */
 @Component(role = AbstractMavenLifecycleParticipant.class, hint = "native-build-tools")
 public class NativeExtension extends AbstractMavenLifecycleParticipant implements LogEnabled {
 
     private static final String JUNIT_PLATFORM_LISTENERS_UID_TRACKING_ENABLED = "junit.platform.listeners.uid.tracking.enabled";
+    private static final String JUNIT_PLATFORM_DRY_RUN_ENABLED = "junit.platform.execution.dryRun.enabled";
     private static final String JUNIT_PLATFORM_LISTENERS_UID_TRACKING_OUTPUT_DIR = "junit.platform.listeners.uid.tracking.output.dir";
     private static final String NATIVEIMAGE_IMAGECODE = "org.graalvm.nativeimage.imagecode";
 
@@ -134,11 +135,16 @@ public void afterProjectsRead(MavenSession session) {
                     throw new RuntimeException(e);
                 }
 
+                boolean skipJVMTests = shouldSkipJVMTests(session);
+                if (skipJVMTests && agent.isEnabled()) {
+                    throw new IllegalStateException("Native Image Agent and skipJVMTests cannot be used at the same time.");
+                }
+
                 // Test configuration
                 List<String> plugins = List.of("maven-surefire-plugin", "maven-failsafe-plugin");
                 for (String pluginName : plugins) {
                     withPlugin(build, pluginName, plugin -> {
-                        configureJunitListener(plugin, testIdsDir);
+                        configureJunitListener(plugin, testIdsDir, skipJVMTests);
                         if (agent.isEnabled()) {
                             List<String> agentOptions = agent.getAgentCommandLine();
                             configureAgentForPlugin(plugin, buildAgentArgument(target, Context.test, agentOptions));
@@ -188,6 +194,11 @@ public void afterProjectsRead(MavenSession session) {
         }
     }
 
+    private static boolean shouldSkipJVMTests(MavenSession session) {
+        String option = session.getSystemProperties().getProperty(SharedConstants.SKIP_JVM_TESTS);
+        return (option != null && option.isEmpty()) || Boolean.parseBoolean(option);
+    }
+
     private static void setupMergeAgentFiles(PluginExecution exec, Xpp3Dom configuration, Context context) {
         List<String> goals = new ArrayList<>();
         goals.add("merge-agent-files");
@@ -217,12 +228,19 @@ private static void configureAgentForPlugin(Plugin plugin, String agentArgument)
         });
     }
 
-    private static void configureJunitListener(Plugin surefirePlugin, String testIdsDir) {
-        updatePluginConfiguration(surefirePlugin, (exec, configuration) -> {
+    private static void configureJunitListener(Plugin plugin, String testIdsDir, boolean skipJVMTests) {
+        updatePluginConfiguration(plugin, (exec, configuration) -> {
             Xpp3Dom systemProperties = findOrAppend(configuration, "systemProperties");
+
             Xpp3Dom junitTracking = findOrAppend(systemProperties, JUNIT_PLATFORM_LISTENERS_UID_TRACKING_ENABLED);
-            Xpp3Dom testIdsProperty = findOrAppend(systemProperties, JUNIT_PLATFORM_LISTENERS_UID_TRACKING_OUTPUT_DIR);
             junitTracking.setValue("true");
+
+            if (skipJVMTests) {
+                Xpp3Dom junitDryRun = findOrAppend(systemProperties, JUNIT_PLATFORM_DRY_RUN_ENABLED);
+                junitDryRun.setValue("true");
+            }
+
+            Xpp3Dom testIdsProperty = findOrAppend(systemProperties, JUNIT_PLATFORM_LISTENERS_UID_TRACKING_OUTPUT_DIR);
             testIdsProperty.setValue(testIdsDir);
         });
     }

native-maven-plugin/src/main/java/org/graalvm/buildtools/utils/AgentUtils.java

@@ -56,10 +56,13 @@
 import java.util.List;
 import java.util.stream.Collectors;
 
-import static org.graalvm.buildtools.utils.Utils.parseBoolean;
-
 public abstract class AgentUtils {
 
+    private static final String STANDARD = "standard";
+    private static final String CONDITIONAL = "conditional";
+    private static final String DIRECT = "direct";
+    private static final String DISABLED = "disabled";
+
     public static AgentMode getAgentMode(Xpp3Dom agent) throws Exception {
         Xpp3Dom defaultModeNode = Xpp3DomParser.getTagByName(agent, "defaultMode");
         // if default mode is not provided in pom, return Standard mode
@@ -72,13 +75,13 @@ public static AgentMode getAgentMode(Xpp3Dom agent) throws Exception {
         AgentMode agentMode;
         String mode = defaultModeNode.getValue();
         switch (mode.toLowerCase()) {
-            case "standard":
-                agentMode = new StandardAgentMode();
-                break;
-            case "disabled":
+            case DISABLED:
                 agentMode = new DisabledAgentMode();
                 break;
-            case "conditional":
+            case STANDARD:
+                agentMode = new StandardAgentMode();
+                break;
+            case CONDITIONAL:
                 // conditional mode needs few more options declared in xml
                 if (agentModes == null) {
                     throw new RuntimeException("Tag <modes> not provided in agent configuration.");
@@ -91,12 +94,12 @@ public static AgentMode getAgentMode(Xpp3Dom agent) throws Exception {
                     throw new Exception("UserCodeFilterPath must be provided in agent configuration");
                 }
 
-                Boolean parallel = parseBooleanNode(agentModes, "parallel");
+                Boolean parallel = Utils.parseBooleanNode(agentModes, "parallel");
                 agentMode = new ConditionalAgentMode(userCodeFilterPathNode.getValue(),
                                                     extraFilterPathNode != null ? extraFilterPathNode.getValue() : "",
                                                      parallel == null ? false : parallel);
                 break;
-            case "direct":
+            case DIRECT:
                 // direct mode is given
                 if (agentModes == null) {
                     throw new RuntimeException("Tag <modes> not provided in agent configuration.");
@@ -138,11 +141,11 @@ public static AgentConfiguration collectAgentProperties(MavenSession session, Xp
 
         ArrayList<String> callerFilterFiles = (ArrayList<String>) getFilterFiles(options, "callerFilterFiles");
         ArrayList<String> accessFilterFiles = (ArrayList<String>) getFilterFiles(options, "accessFilterFiles");
-        Boolean builtinCallerFilter = parseBooleanNode(options, "builtinCallerFilter");
-        Boolean builtinHeuristicFilter = parseBooleanNode(options, "builtinHeuristicFilter");
-        Boolean enableExperimentalPredefinedClasses = parseBooleanNode(options, "enableExperimentalPredefinedClasses");
-        Boolean enableExperimentalUnsafeAllocationTracing = parseBooleanNode(options, "enableExperimentalUnsafeAllocationTracing");
-        Boolean trackReflectionMetadata = parseBooleanNode(options, "trackReflectionMetadata");
+        Boolean builtinCallerFilter = Utils.parseBooleanNode(options, "builtinCallerFilter");
+        Boolean builtinHeuristicFilter = Utils.parseBooleanNode(options, "builtinHeuristicFilter");
+        Boolean enableExperimentalPredefinedClasses = Utils.parseBooleanNode(options, "enableExperimentalPredefinedClasses");
+        Boolean enableExperimentalUnsafeAllocationTracing = Utils.parseBooleanNode(options, "enableExperimentalUnsafeAllocationTracing");
+        Boolean trackReflectionMetadata = Utils.parseBooleanNode(options, "trackReflectionMetadata");
 
         AgentMode mode;
         try {
@@ -175,7 +178,7 @@ private static Boolean isAgentEnabledInCmd(MavenSession session) {
         String systemProperty = session.getSystemProperties().getProperty("agent");
         if (systemProperty != null) {
             // -Dagent=[true|false] overrides configuration in the POM.
-            return parseBoolean("agent system property", systemProperty);
+            return Boolean.parseBoolean(systemProperty);
         }
 
         return null;
@@ -187,7 +190,7 @@ private static boolean isAgentEnabled(MavenSession session, Xpp3Dom agent) {
             return cmdEnable;
         }
 
-        Boolean val = parseBooleanNode(agent, "enabled");
+        Boolean val = Utils.parseBooleanNode(agent, "enabled");
         if (val == null) {
             return false;
         }
@@ -210,18 +213,4 @@ private static List<String> getFilterFiles(Xpp3Dom root, String type) {
                 .map(Xpp3Dom::getValue)
                 .collect(Collectors.toCollection(ArrayList::new));
     }
-
-    private static Boolean parseBooleanNode(Xpp3Dom root, String name) {
-        if (root == null) {
-            return null;
-        }
-
-        Xpp3Dom node = Xpp3DomParser.getTagByName(root, name);
-        if (node == null) {
-            // if node is not provided, default value is false
-            return null;
-        }
-
-        return Utils.parseBoolean("<" + name + ">", node.getValue());
-    }
 }