Support providing root client ID via env. variables when bootstrapping

extension/persistence/eclipselink/src/main/java/org/apache/polaris/extension/persistence/impl/eclipselink/PolarisEclipseLinkMetaStoreSessionImpl.java

@@ -63,6 +63,7 @@
 import org.apache.polaris.core.exceptions.AlreadyExistsException;
 import org.apache.polaris.core.persistence.PolarisMetaStoreManagerImpl;
 import org.apache.polaris.core.persistence.PolarisMetaStoreSession;
+import org.apache.polaris.core.persistence.PrincipalSecretsGenerator;
 import org.apache.polaris.core.persistence.RetryOnConcurrencyException;
 import org.apache.polaris.core.persistence.models.ModelEntity;
 import org.apache.polaris.core.persistence.models.ModelEntityActive;
@@ -645,13 +646,16 @@ public int lookupEntityGrantRecordsVersion(
   /** {@inheritDoc} */
   @Override
   public @NotNull PolarisPrincipalSecrets generateNewPrincipalSecrets(
-      @NotNull PolarisCallContext callCtx, @NotNull String principalName, long principalId) {
+      @NotNull PolarisCallContext callCtx,
+      @NotNull String principalName,
+      long principalId,
+      PrincipalSecretsGenerator generator) {
     // ensure principal client id is unique
     PolarisPrincipalSecrets principalSecrets;
     ModelPrincipalSecrets lookupPrincipalSecrets;
     do {
       // generate new random client id and secrets
-      principalSecrets = new PolarisPrincipalSecrets(principalId);
+      principalSecrets = generator.produceSecrets(principalName, principalId);
 
       // load the existing secrets
       lookupPrincipalSecrets =

polaris-core/src/main/java/org/apache/polaris/core/persistence/LocalPolarisMetaStoreManagerFactory.java

@@ -21,6 +21,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.function.Function;
 import java.util.function.Supplier;
 import org.apache.polaris.core.PolarisCallContext;
 import org.apache.polaris.core.PolarisDefaultDiagServiceImpl;
@@ -75,17 +76,20 @@ private void initializeForRealm(RealmContext realmContext) {
   }
 
   @Override
-  public synchronized Map<String, PolarisMetaStoreManager.PrincipalSecretsResult> bootstrapRealms(
-      List<String> realms) {
+  public Map<String, PolarisMetaStoreManager.PrincipalSecretsResult> bootstrapRealms(
+      List<String> realms, Function<String, PrincipalSecretsGenerator> rootSecretsPerRealm) {
     Map<String, PolarisMetaStoreManager.PrincipalSecretsResult> results = new HashMap<>();
 
     for (String realm : realms) {
       RealmContext realmContext = () -> realm;
+      PrincipalSecretsGenerator rootSecrets = rootSecretsPerRealm.apply(realm);
       if (!metaStoreManagerMap.containsKey(realmContext.getRealmIdentifier())) {
         initializeForRealm(realmContext);
         PolarisMetaStoreManager.PrincipalSecretsResult secretsResult =
             bootstrapServiceAndCreatePolarisPrincipalForRealm(
-                realmContext, metaStoreManagerMap.get(realmContext.getRealmIdentifier()));
+                realmContext,
+                metaStoreManagerMap.get(realmContext.getRealmIdentifier()),
+                rootSecrets);
         results.put(realmContext.getRealmIdentifier(), secretsResult);
       }
     }
@@ -159,7 +163,9 @@ public void setStorageIntegrationProvider(PolarisStorageIntegrationProvider stor
    */
   private PolarisMetaStoreManager.PrincipalSecretsResult
       bootstrapServiceAndCreatePolarisPrincipalForRealm(
-          RealmContext realmContext, PolarisMetaStoreManager metaStoreManager) {
+          RealmContext realmContext,
+          PolarisMetaStoreManager metaStoreManager,
+          PrincipalSecretsGenerator rootSecretsGenerator) {
     // While bootstrapping we need to act as a fake privileged context since the real
     // CallContext hasn't even been resolved yet.
     PolarisCallContext polarisContext =
@@ -182,7 +188,7 @@ public void setStorageIntegrationProvider(PolarisStorageIntegrationProvider stor
       throw new IllegalArgumentException(overrideMessage);
     }
 
-    metaStoreManager.bootstrapPolarisService(polarisContext);
+    metaStoreManager.bootstrapPolarisService(polarisContext, rootSecretsGenerator);
 
     PolarisMetaStoreManager.EntityResult rootPrincipalLookup =
         metaStoreManager.readEntityByName(

polaris-core/src/main/java/org/apache/polaris/core/persistence/MetaStoreManagerFactory.java

@@ -22,6 +22,7 @@
 import io.dropwizard.jackson.Discoverable;
 import java.util.List;
 import java.util.Map;
+import java.util.function.Function;
 import java.util.function.Supplier;
 import org.apache.polaris.core.context.RealmContext;
 import org.apache.polaris.core.monitor.PolarisMetricRegistry;
@@ -45,7 +46,8 @@ public interface MetaStoreManagerFactory extends Discoverable {
 
   void setMetricRegistry(PolarisMetricRegistry metricRegistry);
 
-  Map<String, PolarisMetaStoreManager.PrincipalSecretsResult> bootstrapRealms(List<String> realms);
+  Map<String, PolarisMetaStoreManager.PrincipalSecretsResult> bootstrapRealms(
+      List<String> realms, Function<String, PrincipalSecretsGenerator> rootSecretsPerRealm);
 
   /** Purge all metadata for the realms provided */
   void purgeRealms(List<String> realms);

polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManager.java

@@ -186,7 +186,8 @@ public boolean alreadyExists() {
    * @return the result of the bootstrap attempt
    */
   @NotNull
-  BaseResult bootstrapPolarisService(@NotNull PolarisCallContext callCtx);
+  BaseResult bootstrapPolarisService(
+      @NotNull PolarisCallContext callCtx, PrincipalSecretsGenerator rootSecretsGenerator);
 
   /**
    * Purge all metadata associated with the Polaris service, resetting the metastore to the state it

polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreManagerImpl.java

@@ -18,6 +18,8 @@
  */
 package org.apache.polaris.core.persistence;
 
+import static org.apache.polaris.core.persistence.PrincipalSecretsGenerator.RANDOM_SECRETS;
+
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.core.type.TypeReference;
 import com.fasterxml.jackson.databind.JsonMappingException;
@@ -636,9 +638,12 @@ private void revokeGrantRecord(
    *
    * @param callCtx call context
    * @param ms meta store in read/write mode
+   * @param rootSecretsGenerator
    */
   private void bootstrapPolarisService(
-      @NotNull PolarisCallContext callCtx, @NotNull PolarisMetaStoreSession ms) {
+      @NotNull PolarisCallContext callCtx,
+      @NotNull PolarisMetaStoreSession ms,
+      PrincipalSecretsGenerator rootSecretsGenerator) {
 
     // Create a root container entity that can represent the securable for any top-level grants.
     PolarisBaseEntity rootContainer =
@@ -665,7 +670,7 @@ private void bootstrapPolarisService(
             PolarisEntityConstants.getRootPrincipalName());
 
     // create this principal
-    this.createPrincipal(callCtx, ms, rootPrincipal);
+    this.createPrincipal(callCtx, ms, rootPrincipal, rootSecretsGenerator);
 
     // now create the account admin principal role
     long serviceAdminPrincipalRoleId = ms.generateNewId(callCtx);
@@ -698,12 +703,14 @@ private void bootstrapPolarisService(
 
   /** {@inheritDoc} */
   @Override
-  public @NotNull BaseResult bootstrapPolarisService(@NotNull PolarisCallContext callCtx) {
+  public @NotNull BaseResult bootstrapPolarisService(
+      @NotNull PolarisCallContext callCtx, PrincipalSecretsGenerator rootSecretsGenerator) {
     // get meta store we should be using
     PolarisMetaStoreSession ms = callCtx.getMetaStore();
 
     // run operation in a read/write transaction
-    ms.runActionInTransaction(callCtx, () -> this.bootstrapPolarisService(callCtx, ms));
+    ms.runActionInTransaction(
+        callCtx, () -> this.bootstrapPolarisService(callCtx, ms, rootSecretsGenerator));
 
     // all good
     return new BaseResult(ReturnStatus.SUCCESS);
@@ -879,7 +886,8 @@ public Map<String, String> deserializeProperties(PolarisCallContext callCtx, Str
   private @NotNull CreatePrincipalResult createPrincipal(
       @NotNull PolarisCallContext callCtx,
       @NotNull PolarisMetaStoreSession ms,
-      @NotNull PolarisBaseEntity principal) {
+      @NotNull PolarisBaseEntity principal,
+      PrincipalSecretsGenerator generator) {
     // validate input
     callCtx.getDiagServices().checkNotNull(principal, "unexpected_null_principal");
 
@@ -956,7 +964,7 @@ public Map<String, String> deserializeProperties(PolarisCallContext callCtx, Str
 
     // generate new secretes for this principal
     PolarisPrincipalSecrets principalSecrets =
-        ms.generateNewPrincipalSecrets(callCtx, principal.getName(), principal.getId());
+        ms.generateNewPrincipalSecrets(callCtx, principal.getName(), principal.getId(), generator);
 
     // generate properties
     Map<String, String> internalProperties = getInternalPropertyMap(callCtx, principal);
@@ -981,7 +989,8 @@ public Map<String, String> deserializeProperties(PolarisCallContext callCtx, Str
     PolarisMetaStoreSession ms = callCtx.getMetaStore();
 
     // need to run inside a read/write transaction
-    return ms.runInTransaction(callCtx, () -> this.createPrincipal(callCtx, ms, principal));
+    return ms.runInTransaction(
+        callCtx, () -> this.createPrincipal(callCtx, ms, principal, RANDOM_SECRETS));
   }
 
   /** See {@link #loadPrincipalSecrets(PolarisCallContext, String)} */

polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisMetaStoreSession.java

@@ -435,7 +435,10 @@ PolarisPrincipalSecrets loadPrincipalSecrets(
    */
   @NotNull
   PolarisPrincipalSecrets generateNewPrincipalSecrets(
-      @NotNull PolarisCallContext callCtx, @NotNull String principalName, long principalId);
+      @NotNull PolarisCallContext callCtx,
+      @NotNull String principalName,
+      long principalId,
+      PrincipalSecretsGenerator generator);
 
   /**
    * Rotate the secrets of a principal entity, i.e. make the specified main secrets the secondary

polaris-core/src/main/java/org/apache/polaris/core/persistence/PolarisTreeMapMetaStoreSessionImpl.java

@@ -448,13 +448,16 @@ public int lookupEntityGrantRecordsVersion(
   /** {@inheritDoc} */
   @Override
   public @NotNull PolarisPrincipalSecrets generateNewPrincipalSecrets(
-      @NotNull PolarisCallContext callCtx, @NotNull String principalName, long principalId) {
+      @NotNull PolarisCallContext callCtx,
+      @NotNull String principalName,
+      long principalId,
+      PrincipalSecretsGenerator generator) {
     // ensure principal client id is unique
     PolarisPrincipalSecrets principalSecrets;
     PolarisPrincipalSecrets lookupPrincipalSecrets;
     do {
       // generate new random client id and secrets
-      principalSecrets = new PolarisPrincipalSecrets(principalId);
+      principalSecrets = generator.produceSecrets(principalName, principalId);
 
       // load the existing secrets
       lookupPrincipalSecrets =

polaris-core/src/main/java/org/apache/polaris/core/persistence/PrincipalSecretsGenerator.java

@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.core.persistence;
+
+import java.util.Locale;
+import java.util.function.Function;
+import org.apache.polaris.core.entity.PolarisPrincipalSecrets;
+import org.jetbrains.annotations.NotNull;
+
+@FunctionalInterface
+public interface PrincipalSecretsGenerator {
+
+  PrincipalSecretsGenerator RANDOM_SECRETS = (name, id) -> new PolarisPrincipalSecrets(id);
+
+  PolarisPrincipalSecrets produceSecrets(@NotNull String principalName, long principalId);
+
+  static Function<String, PrincipalSecretsGenerator> bootstrap(Function<String, String> config) {
+    return (realmName) ->
+        (principalName, principalId) -> {
+          String propId =
+              String.format("POLARIS_BOOTSTRAP_%s_%s_CLIENT_ID", realmName, principalName);
+          String propSecret =
+              String.format("POLARIS_BOOTSTRAP_%s_%s_CLIENT_SECRET", realmName, principalName);
+          String clientId = config.apply(propId.toUpperCase(Locale.ROOT));
+          String secret = config.apply(propSecret.toUpperCase(Locale.ROOT));
+          if (clientId == null || secret == null) {
+            return new PolarisPrincipalSecrets(principalId); // random secrets
+          } else {
+            // Use the same secondary secret for bootstrapping. It can be rotated later, if required
+            return new PolarisPrincipalSecrets(principalId, clientId, secret, secret);
+          }
+        };
+  }
+}

polaris-core/src/main/java/org/apache/polaris/core/persistence/TransactionWorkspaceMetaStoreManager.java

@@ -71,7 +71,8 @@ public List<EntityWithPath> getPendingUpdates() {
   }
 
   @Override
-  public BaseResult bootstrapPolarisService(@NotNull PolarisCallContext callCtx) {
+  public @NotNull BaseResult bootstrapPolarisService(
+      @NotNull PolarisCallContext callCtx, PrincipalSecretsGenerator rootSecretsGenerator) {
     callCtx
         .getDiagServices()
         .fail("illegal_method_in_transaction_workspace", "bootstrapPolarisService");

polaris-core/src/test/java/org/apache/polaris/core/persistence/PrincipalSecretsGeneratorTest.java

@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.core.persistence;
+
+import static org.apache.polaris.core.persistence.PrincipalSecretsGenerator.bootstrap;
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.util.Map;
+import org.apache.polaris.core.entity.PolarisPrincipalSecrets;
+import org.junit.jupiter.api.Test;
+
+class PrincipalSecretsGeneratorTest {
+
+  @Test
+  void testRandomSecrets() {
+    PolarisPrincipalSecrets s =
+        bootstrap((name) -> null).apply("test").produceSecrets("name1", 123);
+    assertThat(s).isNotNull();
+    assertThat(s.getPrincipalId()).isEqualTo(123);
+    assertThat(s.getPrincipalClientId()).isNotNull();
+    assertThat(s.getMainSecret()).isNotNull();
+    assertThat(s.getSecondarySecret()).isNotNull();
+  }
+
+  @Test
+  void testSecretOverride() {
+    PolarisPrincipalSecrets s =
+        bootstrap(
+                Map.of(
+                        "POLARIS_BOOTSTRAP_TEST-REALM_USER1_CLIENT_ID",
+                        "client1",
+                        "POLARIS_BOOTSTRAP_TEST-REALM_USER1_CLIENT_SECRET",
+                        "sec2")
+                    ::get)
+            .apply("test-Realm")
+            .produceSecrets("user1", 123);
+    assertThat(s).isNotNull();
+    assertThat(s.getPrincipalId()).isEqualTo(123);
+    assertThat(s.getPrincipalClientId()).isEqualTo("client1");
+    assertThat(s.getMainSecret()).isEqualTo("sec2");
+    assertThat(s.getSecondarySecret()).isEqualTo("sec2");
+  }
+}

polaris-core/src/testFixtures/java/org/apache/polaris/core/persistence/PolarisTestMetaStoreManager.java

@@ -18,6 +18,8 @@
  */
 package org.apache.polaris.core.persistence;
 
+import static org.apache.polaris.core.persistence.PrincipalSecretsGenerator.RANDOM_SECRETS;
+
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
@@ -69,7 +71,7 @@ public PolarisTestMetaStoreManager(
 
     // bootstrap the Polaris service
     polarisMetaStoreManager.purge(polarisCallContext);
-    polarisMetaStoreManager.bootstrapPolarisService(polarisCallContext);
+    polarisMetaStoreManager.bootstrapPolarisService(polarisCallContext, RANDOM_SECRETS);
   }
 
   public void forceRetry() {

polaris-service/src/main/java/org/apache/polaris/service/BootstrapRealmsCommand.java

@@ -25,6 +25,7 @@
 import org.apache.polaris.core.PolarisConfigurationStore;
 import org.apache.polaris.core.persistence.MetaStoreManagerFactory;
 import org.apache.polaris.core.persistence.PolarisMetaStoreManager;
+import org.apache.polaris.core.persistence.PrincipalSecretsGenerator;
 import org.apache.polaris.service.config.ConfigurationStoreAware;
 import org.apache.polaris.service.config.PolarisApplicationConfig;
 import org.apache.polaris.service.context.CallContextResolver;
@@ -62,7 +63,9 @@ protected void run(
 
     // Execute the bootstrap
     Map<String, PolarisMetaStoreManager.PrincipalSecretsResult> results =
-        metaStoreManagerFactory.bootstrapRealms(configuration.getDefaultRealms());
+        metaStoreManagerFactory.bootstrapRealms(
+            configuration.getDefaultRealms(),
+            PrincipalSecretsGenerator.bootstrap(System.getenv()::get));
 
     // Log any errors:
     boolean success = true;

polaris-service/src/main/java/org/apache/polaris/service/persistence/InMemoryPolarisMetaStoreManagerFactory.java

@@ -31,6 +31,7 @@
 import org.apache.polaris.core.persistence.PolarisMetaStoreSession;
 import org.apache.polaris.core.persistence.PolarisTreeMapMetaStoreSessionImpl;
 import org.apache.polaris.core.persistence.PolarisTreeMapStore;
+import org.apache.polaris.core.persistence.PrincipalSecretsGenerator;
 import org.jetbrains.annotations.NotNull;
 
 @JsonTypeName("in-memory")
@@ -71,7 +72,9 @@ public synchronized Supplier<PolarisMetaStoreSession> getOrCreateSessionSupplier
 
   private void bootstrapRealmAndPrintCredentials(String realmId) {
     Map<String, PolarisMetaStoreManager.PrincipalSecretsResult> results =
-        this.bootstrapRealms(Collections.singletonList(realmId));
+        this.bootstrapRealms(
+            Collections.singletonList(realmId),
+            PrincipalSecretsGenerator.bootstrap(System.getenv()::get));
     bootstrappedRealms.add(realmId);
 
     PolarisMetaStoreManager.PrincipalSecretsResult principalSecrets = results.get(realmId);