Native libraries are searched in lib/polyglot project dir

.gitignore

@@ -110,6 +110,7 @@ bench-report*.xml
 /enso.lib
 
 *.dll
+*.dylib
 *.exe
 *.pdb
 *.so

CHANGELOG.md

@@ -17,11 +17,13 @@
 - A constructor or type definition with a single inline argument definition was
   previously allowed to use spaces in the argument definition without
   parentheses. [This is now a syntax error.][11856]
+- [Native libraries of projects can be added to `polyglot/lib` directory][11874]
 - Symetric, transitive and reflexive [equality for intersection types][11897]
 
 [11777]: https://github.com/enso-org/enso/pull/11777
 [11600]: https://github.com/enso-org/enso/pull/11600
 [11856]: https://github.com/enso-org/enso/pull/11856
+[11874]: https://github.com/enso-org/enso/pull/11874
 [11897]: https://github.com/enso-org/enso/pull/11897
 
 # Next Release

build.sbt

@@ -4487,6 +4487,7 @@ def stdLibComponentRoot(name: String): File =
 val `base-polyglot-root`  = stdLibComponentRoot("Base") / "polyglot" / "java"
 val `table-polyglot-root` = stdLibComponentRoot("Table") / "polyglot" / "java"
 val `image-polyglot-root` = stdLibComponentRoot("Image") / "polyglot" / "java"
+val `image-native-libs`   = stdLibComponentRoot("Image") / "polyglot" / "lib"
 val `google-api-polyglot-root` =
   stdLibComponentRoot("Google_Api") / "polyglot" / "java"
 val `database-polyglot-root` =
@@ -4654,6 +4655,10 @@ lazy val `std-table` = project
   )
   .dependsOn(`std-base` % "provided")
 
+lazy val extractNativeLibs = taskKey[Unit](
+  "Helper task to extract native libraries from OpenCV JAR"
+)
+
 lazy val `std-image` = project
   .in(file("std-bits") / "image")
   .settings(
@@ -4669,15 +4674,21 @@ lazy val `std-image` = project
       "org.netbeans.api"     % "org-openide-util-lookup" % netbeansApiVersion        % "provided",
       "org.openpnp"          % "opencv"                  % opencvVersion
     ),
-    Compile / packageBin := Def.task {
-      val result = (Compile / packageBin).value
-      val _ = StdBits
-        .copyDependencies(
+    // Extract native libraries from opencv.jar, and put them under
+    // Standard/Image/polyglot/lib directory. The minimized opencv.jar will
+    // be put under Standard/Image/polyglot/java directory.
+    extractNativeLibs := {
+      StdBits
+        .extractNativeLibsFromOpenCV(
           `image-polyglot-root`,
-          Seq("std-image.jar"),
-          ignoreScalaLibrary = true
+          `image-native-libs`,
+          opencvVersion
         )
         .value
+    },
+    Compile / packageBin := Def.task {
+      val result = (Compile / packageBin).value
+      val _      = extractNativeLibs.value
       result
     }.value
   )

docs/polyglot/java.md

@@ -44,27 +44,60 @@ The dynamic polyglot system is a dynamic runtime lookup for Java objects,
 allowing Enso code to work with them through a runtime reflection-style
 mechanism. It is comprised of the following components:
 
-- `Java.lookup_class : Class.Path -> Maybe Class`: A function that lets users
-  look up a class by a given name on the runtime classpath.
-- `Polyglot.instantiate : Class -> Object`: A function that lets users
-  instantiate a class into an object.
+- `Java.lookup_class : Text -> Any`: A function that lets users look up a class
+  by a given name on the runtime classpath.
 - A whole host of functions on the polyglot type that let you dynamically work
   with object bindings.
 
 An example can be found below:
 
 ```ruby
+from Standard.Base.Polyglot import Java, polyglot
+
 main =
     class = Java.lookup_class "org.enso.example.TestClass"
-    instance = Polyglot.instantiate1 class (x -> x * 2)
+    instance = class.new (x -> x * 2)
     method = Polyglot.get_member instance "callFunctionAndIncrement"
-    Polyglot.execute1 method 10
+    Polyglot.execute method 10
 ```
 
 > The actionables for this section are:
 >
 > - Expand on the detail when there is time.
 
+## Native libraries
+
+Java can load native libraries using, e.g., the
+[System.loadLibrary](<https://docs.oracle.com/en/java/javase/21/docs/api/java.base/java/lang/System.html#loadLibrary(java.lang.String)>)
+or
+[ClassLoader.findLibrary](<https://docs.oracle.com/en/java/javase/21/docs/api/java.base/java/lang/ClassLoader.html#findLibrary(java.lang.String)>)
+methods. If a Java method loaded from the `polyglot/java` directory in project
+`Proj` tries to load a native library via one of the aforementioned mechanisms,
+the runtime system will look for the native library in the `polyglot/lib`
+directory within the project `Proj`. The runtime system implements this by
+overriding the
+[ClassLoader.findLibrary](<https://docs.oracle.com/en/java/javase/21/docs/api/java.base/java/lang/ClassLoader.html#findLibrary(java.lang.String)>)
+method on the `ClassLoader` used to load the Java class.
+
+The algorithm used to search for the native libraries within the `polyglot/lib`
+directory hierarchy conforms to the
+[NetBeans JNI specification](https://bits.netbeans.org/23/javadoc/org-openide-modules/org/openide/modules/doc-files/api.html#jni):
+Lookup of library with name `native` works roughly in these steps:
+
+- Add platform-specific prefix and/or suffix to the library name, e.g.,
+  `libnative.so` on Linux.
+- Search for the library in the `polyglot/lib` directory.
+- Search for the library in the `polyglot/lib/<arch>` directory, where `<arch>`
+  is the name of the architecture.
+- Search for the library in the `polyglot/lib/<arch>/<os>` directory, where
+  `<os>` is the name of the operating system.
+
+Supported names:
+
+- Names for `<os>` are `linux`, `macos`, `windows`.
+  - Note that for simplicity we omit the versions of the operating systems.
+- Names for architectures `<arch>` are `amd64`, `x86_64`, `x86_32`.
+
 ## Download a Java Library from Maven Central
 
 A typical use-case when bringing in some popular Java library into Enso

engine/runner/src/main/java/org/enso/runner/EnsoLibraryFeature.java

@@ -2,27 +2,25 @@
 
 import static scala.jdk.javaapi.CollectionConverters.asJava;
 
+import java.io.File;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.LinkedHashSet;
 import java.util.TreeSet;
 import org.enso.compiler.core.EnsoParser;
 import org.enso.compiler.core.ir.module.scope.imports.Polyglot;
+import org.enso.filesystem.FileSystem$;
+import org.enso.pkg.NativeLibraryFinder;
 import org.enso.pkg.PackageManager$;
 import org.graalvm.nativeimage.hosted.Feature;
 import org.graalvm.nativeimage.hosted.RuntimeProxyCreation;
 import org.graalvm.nativeimage.hosted.RuntimeReflection;
-import org.graalvm.nativeimage.hosted.RuntimeResourceAccess;
+import org.graalvm.nativeimage.hosted.RuntimeSystemProperties;
 
 public final class EnsoLibraryFeature implements Feature {
   @Override
   public void beforeAnalysis(BeforeAnalysisAccess access) {
-    try {
-      registerOpenCV(access.getApplicationClassLoader());
-    } catch (ReflectiveOperationException ex) {
-      ex.printStackTrace();
-      throw new IllegalStateException(ex);
-    }
+
     var libs = new LinkedHashSet<Path>();
     for (var p : access.getApplicationClassPath()) {
       var p1 = p.getParent();
@@ -53,6 +51,7 @@ public void beforeAnalysis(BeforeAnalysisAccess access) {
     */
 
     var classes = new TreeSet<String>();
+    var nativeLibPaths = new TreeSet<String>();
     try {
       for (var p : libs) {
         var result = PackageManager$.MODULE$.Default().loadPackage(p.toFile());
@@ -90,46 +89,31 @@ public void beforeAnalysis(BeforeAnalysisAccess access) {
               }
             }
           }
+          if (pkg.nativeLibraryDir().exists()) {
+            var nativeLibs =
+                NativeLibraryFinder.listAllNativeLibraries(pkg, FileSystem$.MODULE$.defaultFs());
+            for (var nativeLib : nativeLibs) {
+              assert nativeLib.exists() && nativeLib.isFile();
+              var dir = nativeLib.toPath().getParent().toFile();
+              assert dir.exists() && dir.isDirectory();
+              nativeLibPaths.add(dir.getAbsolutePath());
+              var current = System.getProperty("java.library.path");
+              RuntimeSystemProperties.register(
+                  "java.library.path", current + File.pathSeparator + dir.getAbsolutePath());
+            }
+          }
         }
       }
     } catch (Exception ex) {
-      ex.printStackTrace();
+      ex.printStackTrace(System.err);
       throw new IllegalStateException(ex);
     }
     System.err.println("Summary for polyglot import java:");
     for (var className : classes) {
       System.err.println("  " + className);
     }
     System.err.println("Registered " + classes.size() + " classes for reflection");
-  }
-
-  private static void registerOpenCV(ClassLoader cl) throws ReflectiveOperationException {
-    var moduleOpenCV = cl.getUnnamedModule();
-    var currentOS = System.getProperty("os.name").toUpperCase().replaceAll(" .*$", "");
-
-    var libOpenCV =
-        switch (currentOS) {
-          case "LINUX" -> "nu/pattern/opencv/linux/x86_64/libopencv_java470.so";
-          case "WINDOWS" -> "nu/pattern/opencv/windows/x86_64/opencv_java470.dll";
-          case "MAC" -> {
-            var arch = System.getProperty("os.arch").toUpperCase();
-            yield switch (arch) {
-              case "X86_64" -> "nu/pattern/opencv/osx/x86_64/libopencv_java470.dylib";
-              case "AARCH64" -> "nu/pattern/opencv/osx/ARMv8/libopencv_java470.dylib";
-              default -> null;
-            };
-          }
-          default -> null;
-        };
-
-    if (libOpenCV != null) {
-      var verify = cl.getResource(libOpenCV);
-      if (verify == null) {
-        throw new IllegalStateException("Cannot find " + libOpenCV + " resource in " + cl);
-      }
-      RuntimeResourceAccess.addResource(moduleOpenCV, libOpenCV);
-    } else {
-      throw new IllegalStateException("No resource suggested for " + currentOS);
-    }
+    System.err.println(
+        "Registered native libraries into runtime's java.library.path: " + nativeLibPaths);
   }
 }

engine/runtime-integration-tests/src/test/java/org/enso/compiler/test/SerializationManagerTest.java

@@ -34,7 +34,7 @@ public class SerializationManagerTest {
 
   @Before
   public void setup() {
-    packageManager = new PackageManager<>(new TruffleFileSystem());
+    packageManager = new PackageManager<>(TruffleFileSystem.INSTANCE);
     interpreterContext = new InterpreterContext(x -> x);
     ensoContext =
         interpreterContext

engine/runtime-integration-tests/src/test/java/org/enso/interpreter/test/NativeLibraryFinderTest.java

@@ -0,0 +1,108 @@
+package org.enso.interpreter.test;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.is;
+
+import com.oracle.truffle.api.TruffleFile;
+import java.io.IOException;
+import java.nio.file.FileVisitResult;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.util.ArrayList;
+import java.util.List;
+import org.enso.editions.LibraryName;
+import org.enso.interpreter.runtime.util.TruffleFileSystem;
+import org.enso.pkg.NativeLibraryFinder;
+import org.enso.pkg.Package;
+import org.enso.test.utils.ContextUtils;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TestRule;
+import org.junit.runner.Description;
+import org.junit.runners.model.Statement;
+
+public class NativeLibraryFinderTest {
+
+  @Rule public final TestRule printContextRule = new PrintSystemInfoRule();
+  private Package<TruffleFile> stdImgPkg;
+
+  @Test
+  public void standardImageShouldHaveNativeLib() {
+    try (var ctx = ContextUtils.createDefaultContext()) {
+      // Evaluate dummy sources to force loading Standard.Image
+      ContextUtils.evalModule(
+          ctx, """
+          from Standard.Image import all
+          main = 42
+          """);
+      var ensoCtx = ContextUtils.leakContext(ctx);
+      var stdImg =
+          ensoCtx
+              .getPackageRepository()
+              .getPackageForLibraryJava(LibraryName.apply("Standard", "Image"));
+      assertThat(stdImg.isPresent(), is(true));
+      this.stdImgPkg = stdImg.get();
+      var nativeLibs =
+          NativeLibraryFinder.listAllNativeLibraries(stdImg.get(), TruffleFileSystem.INSTANCE);
+      assertThat(
+          "There should be just single native lib in Standard.Image", nativeLibs.size(), is(1));
+    }
+  }
+
+  public final class PrintSystemInfoRule implements TestRule {
+    @Override
+    public Statement apply(Statement base, Description description) {
+      return new Statement() {
+        @Override
+        public void evaluate() {
+          try {
+            base.evaluate();
+          } catch (Throwable e) {
+            var sb = new StringBuilder();
+            sb.append(System.lineSeparator());
+            sb.append("  os.name: ")
+                .append(System.getProperty("os.name"))
+                .append(System.lineSeparator());
+            sb.append("  os.arch: ")
+                .append(System.getProperty("os.arch"))
+                .append(System.lineSeparator());
+            var mappedLibName = System.mapLibraryName("opencv_java470");
+            sb.append("  Mapped library name: ")
+                .append(mappedLibName)
+                .append(System.lineSeparator());
+            if (stdImgPkg != null) {
+              sb.append("  Contents of Standard.Image native library dir:")
+                  .append(System.lineSeparator());
+              var nativeLibDir = stdImgPkg.nativeLibraryDir();
+              var nativeLibPath = Path.of(nativeLibDir.getAbsoluteFile().getPath());
+              var contents = contentsOfDir(nativeLibPath);
+              contents.forEach(
+                  path -> sb.append("    ").append(path).append(System.lineSeparator()));
+            }
+            throw new AssertionError(sb.toString(), e);
+          }
+        }
+      };
+    }
+  }
+
+  private static List<String> contentsOfDir(Path dir) {
+    var contents = new ArrayList<String>();
+    var fileVisitor =
+        new SimpleFileVisitor<Path>() {
+          @Override
+          public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) {
+            contents.add(file.toAbsolutePath().toString());
+            return FileVisitResult.CONTINUE;
+          }
+        };
+    try {
+      Files.walkFileTree(dir, fileVisitor);
+    } catch (IOException e) {
+      throw new AssertionError(e);
+    }
+    return contents;
+  }
+}

engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoContext.java

@@ -170,7 +170,7 @@ public EnsoContext(
 
   /** Perform expensive initialization logic for the context. */
   public void initialize() {
-    TruffleFileSystem fs = new TruffleFileSystem();
+    TruffleFileSystem fs = TruffleFileSystem.INSTANCE;
     PackageManager<TruffleFile> packageManager = new PackageManager<>(fs);
 
     Optional<TruffleFile> projectRoot = OptionsHelper.getProjectRoot(environment);