Add ServiceLoader auto-registration for ServerSidePlanningClientFactory

Enable automatic discovery and registration of ServerSidePlanningClientFactory
implementations using Java's ServiceLoader mechanism. When delta-iceberg JAR
is on the classpath, IcebergRESTCatalogPlanningClientFactory is automatically
registered, eliminating the need for manual registration.

Changes:
- Add META-INF/services file for IcebergRESTCatalogPlanningClientFactory
- Implement ServiceLoader discovery in ServerSidePlanningClientFactory
- Include serverSidePlanning package in delta-iceberg JAR (build.sbt)
- Add helper methods for factory registration introspection

Author: murali-db

build.sbt

@@ -1105,6 +1105,8 @@ lazy val testDeltaIcebergJar = (project in file("testDeltaIcebergJar"))
 val deltaIcebergSparkIncludePrefixes = Seq(
   // We want everything from this package
   "org/apache/spark/sql/delta/icebergShaded",
+  // Server-side planning support for Unity Catalog
+  "org/apache/spark/sql/delta/serverSidePlanning",
 
   // We only want the files in this project from this package. e.g. we want to exclude
   // org/apache/spark/sql/delta/commands/convert/ConvertTargetFile.class (from delta-spark project).

iceberg/src/main/resources/META-INF/services/org.apache.spark.sql.delta.serverSidePlanning.ServerSidePlanningClientFactory

@@ -0,0 +1 @@
+org.apache.spark.sql.delta.serverSidePlanning.IcebergRESTCatalogPlanningClientFactory

spark/src/main/scala/org/apache/spark/sql/delta/serverSidePlanning/ServerSidePlanningClient.scala

@@ -70,41 +70,113 @@ private[serverSidePlanning] trait ServerSidePlanningClientFactory {
 }
 
 /**
- * Registry for client factories. Can be configured for testing or to provide
- * production implementations (e.g., IcebergRESTCatalogPlanningClientFactory).
+ * Registry for client factories. Automatically discovers and registers implementations
+ * using Java's ServiceLoader mechanism at first access.
  *
- * By default, no factory is registered. Production code should register an appropriate
- * factory implementation before attempting to create clients.
+ * When delta-iceberg JAR is on the classpath, IcebergRESTCatalogPlanningClientFactory
+ * is automatically registered via META-INF/services discovery. Manual registration
+ * using setFactory() is only needed for testing or to override the auto-discovered factory.
  */
 private[serverSidePlanning] object ServerSidePlanningClientFactory {
   @volatile private var registeredFactory: Option[ServerSidePlanningClientFactory] = None
 
+  // ========== SERVICE LOADER AUTO-REGISTRATION ==========
+  // This block runs ONCE when ServerSidePlanningClientFactory is first accessed.
+  // It uses Java's ServiceLoader to discover and register factories declared in
+  // META-INF/services/org.apache.spark.sql.delta.serverSidePlanning.ServerSidePlanningClientFactory
+  {
+    import java.util.ServiceLoader
+    import scala.jdk.CollectionConverters._
+
+    try {
+      // Use ServiceLoader to discover all implementations on the classpath
+      val loader = ServiceLoader.load(
+        classOf[ServerSidePlanningClientFactory],
+        Thread.currentThread().getContextClassLoader)
+
+      // Convert Java Iterator to Scala and collect all factories
+      val factories = loader.iterator().asScala.toList
+
+      if (factories.nonEmpty) {
+        // Use the first discovered factory
+        // In practice, there should only be one (IcebergRESTCatalogPlanningClientFactory)
+        registeredFactory = Some(factories.head)
+
+        // Optional: Log successful registration for debugging
+        // Uncomment for visibility during development
+        // System.err.println(s"[Delta] Auto-registered ${factories.head.getClass.getName} " +
+        //   s"via ServiceLoader (found ${factories.size} implementation(s))")
+
+        // Optional: Warn if multiple implementations found
+        if (factories.size > 1) {
+          // scalastyle:off println
+          System.err.println(
+            s"[Delta] Warning: Multiple ServerSidePlanningClientFactory implementations found. " +
+            s"Using ${factories.head.getClass.getName}. " +
+            s"Others: ${factories.tail.map(_.getClass.getName).mkString(", ")}")
+          // scalastyle:on println
+        }
+      } else {
+        // No factories discovered - delta-iceberg not on classpath
+        // This is fine, server-side planning just won't be available
+        // System.err.println("[Delta] No ServerSidePlanningClientFactory found, FGAC disabled")
+      }
+
+    } catch {
+      case e: Exception =>
+        // Unexpected error during service loading - log but don't fail
+        // Delta should still work for non-FGAC tables
+        // scalastyle:off println
+        System.err.println(
+          s"[Delta] Warning: Failed to auto-discover server-side planning factory: ${e.getMessage}")
+        // scalastyle:on println
+    }
+  }
+  // ========== END SERVICE LOADER AUTO-REGISTRATION ==========
+
   /**
-   * Set a factory for production use or testing.
+   * Set a factory, overriding any auto-registered factory.
+   * Primarily useful for testing or providing custom implementations.
    */
   private[serverSidePlanning] def setFactory(factory: ServerSidePlanningClientFactory): Unit = {
     registeredFactory = Some(factory)
   }
 
   /**
-   * Clear the registered factory.
+   * Clear the registered factory. Primarily useful for testing to reset state between tests.
+   * Note: This does NOT re-trigger ServiceLoader discovery. The factory must be manually
+   * set again via setFactory() if needed.
    */
   private[serverSidePlanning] def clearFactory(): Unit = {
     registeredFactory = None
   }
 
   /**
    * Get the currently registered factory.
-   * Throws IllegalStateException if no factory has been registered.
+   * Throws IllegalStateException if no factory has been registered (either via ServiceLoader
+   * auto-discovery or explicit setFactory() call).
    */
   def getFactory(): ServerSidePlanningClientFactory = {
     registeredFactory.getOrElse {
       throw new IllegalStateException(
         "No ServerSidePlanningClientFactory has been registered. " +
-        "Call ServerSidePlanningClientFactory.setFactory() to register an implementation.")
+        "Ensure delta-iceberg JAR is on the classpath for auto-registration, " +
+        "or call ServerSidePlanningClientFactory.setFactory() to register manually.")
     }
   }
 
+  /**
+   * Check if a factory is currently registered (either via ServiceLoader or setFactory()).
+   * Useful for testing or conditional logic.
+   */
+  def isFactoryRegistered(): Boolean = registeredFactory.isDefined
+
+  /**
+   * Get information about the currently registered factory for debugging/logging.
+   * Returns None if no factory is registered.
+   */
+  def getFactoryInfo(): Option[String] = registeredFactory.map(_.getClass.getName)
+
   /**
    * Convenience method to create a client from metadata using the registered factory.
    */