[b456377401] Refactor Cloudera Manager authentication (#1062)

1. Replace ClouderaManagerLoginHelper with ClouderaHttpClientFactory to manage both session-based (Cloudera Manager) and basic-auth (Knox/YARN) HTTP clients.
2. Update ClouderaManagerHandle to manage separate clients for different authentication needs, ensuring correct credentials are used for both CM and cluster services.
3. Update Cloudera Manager tasks and tests to utilize the refactored HTTP client management and ensure proper resource cleanup.

Author: dominik-burdzy

dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/AbstractClouderaTimeSeriesTask.java

@@ -74,7 +74,7 @@ protected JsonNode requestTimeSeriesChart(ClouderaManagerHandle handle, String q
     uriBuilder.addParameter("to", endDate.format(isoDateTimeFormatter));
     URI tsURI = uriBuilder.build();
 
-    CloseableHttpClient httpClient = handle.getHttpClient();
+    CloseableHttpClient httpClient = handle.getClouderaManagerHttpClient();
     JsonNode chartInJson;
     try (CloseableHttpResponse chart = httpClient.execute(new HttpGet(tsURI))) {
       int statusCode = chart.getStatusLine().getStatusCode();

dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/AbstractClouderaYarnApplicationTask.java

@@ -75,7 +75,7 @@ class PaginatedClouderaYarnApplicationsLoader {
 
     public PaginatedClouderaYarnApplicationsLoader(ClouderaManagerHandle handle, int limit) {
       this.apiURI = handle.getApiURI();
-      this.httpClient = handle.getHttpClient();
+      this.httpClient = handle.getClouderaManagerHttpClient();
       this.limit = limit;
 
       final DateTimeFormatter dtFormatter = DateTimeFormatter.ofPattern(ISO_DATETIME_FORMAT);

dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaAPIHostsTask.java

@@ -48,7 +48,7 @@ public ClouderaAPIHostsTask() {
   protected void doRun(
       TaskRunContext context, @Nonnull ByteSink sink, @Nonnull ClouderaManagerHandle handle)
       throws Exception {
-    CloseableHttpClient httpClient = handle.getHttpClient();
+    CloseableHttpClient httpClient = handle.getClouderaManagerHttpClient();
     List<ClouderaClusterDTO> clusters = handle.getClusters();
     if (clusters == null) {
       throw new MetadataDumperUsageException(

dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaCMFHostsTask.java

@@ -55,7 +55,7 @@ public TaskCategory getCategory() {
   protected void doRun(
       TaskRunContext context, @Nonnull ByteSink sink, @Nonnull ClouderaManagerHandle handle)
       throws Exception {
-    CloseableHttpClient httpClient = handle.getHttpClient();
+    CloseableHttpClient httpClient = handle.getClouderaManagerHttpClient();
     List<ClouderaClusterDTO> clusters = handle.getClusters();
     if (clusters == null) {
       throw new IllegalStateException(

dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaClustersTask.java

@@ -50,7 +50,7 @@ public ClouderaClustersTask() {
   protected void doRun(
       TaskRunContext context, @Nonnull ByteSink sink, @Nonnull ClouderaManagerHandle handle)
       throws Exception {
-    CloseableHttpClient httpClient = handle.getHttpClient();
+    CloseableHttpClient httpClient = handle.getClouderaManagerHttpClient();
 
     ApiClusterListDTO clusterList;
 

dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaConnectorVerifier.java

@@ -53,7 +53,7 @@ private static void verifyClusterExists(ClouderaManagerHandle handle, String clu
     String endpoint = String.format("%s/clusters/%s", handle.getApiURI(), clusterName);
     HttpGet httpGet = new HttpGet(endpoint);
 
-    try (CloseableHttpResponse response = handle.getHttpClient().execute(httpGet)) {
+    try (CloseableHttpResponse response = handle.getClouderaManagerHttpClient().execute(httpGet)) {
 
       int statusCode = response.getStatusLine().getStatusCode();
 

dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaHostComponentsTask.java

@@ -51,7 +51,7 @@ public TaskCategory getCategory() {
   protected void doRun(
       TaskRunContext context, @Nonnull ByteSink sink, @Nonnull ClouderaManagerHandle handle)
       throws Exception {
-    CloseableHttpClient httpClient = handle.getHttpClient();
+    CloseableHttpClient httpClient = handle.getClouderaManagerHttpClient();
     List<ClouderaHostDTO> hosts = handle.getHosts();
     if (hosts == null) {
       throw new IllegalStateException(

dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaHttpClientFactory.java

@@ -0,0 +1,187 @@
+/*
+ * Copyright 2022-2025 Google LLC
+ * Copyright 2013-2021 CompilerWorks
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.google.edwmigration.dumper.application.dumper.connector.cloudera.manager;
+
+import com.google.edwmigration.dumper.application.dumper.MetadataDumperUsageException;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.http.Header;
+import org.apache.http.HttpHeaders;
+import org.apache.http.HttpStatus;
+import org.apache.http.NameValuePair;
+import org.apache.http.client.CredentialsProvider;
+import org.apache.http.client.config.RequestConfig;
+import org.apache.http.client.entity.UrlEncodedFormEntity;
+import org.apache.http.client.methods.CloseableHttpResponse;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.conn.ssl.NoopHostnameVerifier;
+import org.apache.http.conn.ssl.TrustAllStrategy;
+import org.apache.http.impl.client.CloseableHttpClient;
+import org.apache.http.impl.client.HttpClientBuilder;
+import org.apache.http.impl.client.HttpClients;
+import org.apache.http.message.BasicHeader;
+import org.apache.http.message.BasicNameValuePair;
+import org.apache.http.ssl.SSLContextBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Factory for creating specialized {@link CloseableHttpClient} instances for Cloudera ecosystem
+ * interactions.
+ *
+ * <p>This factory provides two distinct client types to handle the different authentication
+ * mechanisms present in Cloudera environments:
+ *
+ * <ul>
+ *   <li><b>Cloudera Manager Client:</b> Stateful, cookie-aware client that performs a Form-Based
+ *       login handshake.
+ *   <li><b>Basic Auth Client:</b> Stateless, isolated client for services like Spark History Server
+ *       or YARN.
+ * </ul>
+ *
+ * <p>Both clients are configured to trust all SSL certificates by default, as internal cluster
+ * certificates are frequently self-signed.
+ */
+public final class ClouderaHttpClientFactory {
+
+  private static final Logger logger = LoggerFactory.getLogger(ClouderaHttpClientFactory.class);
+
+  /**
+   * Connection Timeout (5 seconds) Time to wait for the TCP handshake. Since Cloudera clusters are
+   * on internal networks, if a host is reachable, it should connect almost instantly. If it takes
+   * >5s, the host is likely down or firewalled, and we shouldn't block the thread longer.
+   */
+  private static final int CONNECTION_TIMEOUT_MS = 5000;
+
+  /**
+   * Socket Timeout (30 seconds) Time to wait for data packets after connection. Spark History
+   * Server and YARN are heavy Java applications prone to Garbage Collection pauses and slow I/O
+   * (parsing large HDFS logs). We give them 30s to respond before giving up.
+   */
+  private static final int SOCKET_TIMEOUT_MS = 30000;
+
+  /**
+   * Creates a {@link CloseableHttpClient} configured for Cloudera Manager and performs the login
+   * handshake.
+   *
+   * <p>This method builds a client with cookie management enabled, executes the Spring Security
+   * form login request (`/j_spring_security_check`), and verifies the session by accessing the home
+   * page.
+   *
+   * @param apiUri the API URI of the Cloudera Manager instance
+   * @param username the username for Cloudera Manager
+   * @param password the password for Cloudera Manager
+   * @return a ready-to-use, authenticated http client
+   * @throws Exception if the login request fails, the credentials are invalid, or the home page is
+   *     inaccessible
+   */
+  public static CloseableHttpClient createClouderaManagerClient(
+      URI apiUri, String username, String password) throws Exception {
+
+    HttpClientBuilder builder = HttpClients.custom();
+    configureTrustAllSSL(builder);
+    CloseableHttpClient client = builder.build();
+
+    try {
+      authenticateViaForm(apiUri, client, username, password);
+    } catch (Exception e) {
+      // If login fails, close the client to avoid leaking resources
+      try {
+        client.close();
+      } catch (Exception suppressed) {
+        e.addSuppressed(suppressed);
+      }
+      throw e;
+    }
+
+    return client;
+  }
+
+  /**
+   * Creates a dedicated {@link CloseableHttpClient} configured with Basic Authentication.
+   *
+   * <p>This client is isolated from the application's global state and uses a dedicated {@link
+   * CredentialsProvider}. It is suitable for stateless REST APIs such as the Spark History Server
+   * or YARN ResourceManager.
+   *
+   * <p>Timeouts are explicitly configured (5s connect, 30s read) to handle potential latency in
+   * overloaded cluster services.
+   *
+   * @param username the username for the Basic Auth header
+   * @param password the password for the Basic Auth header
+   * @return a configured http client that sends credentials with every request
+   * @throws Exception if SSL configuration fails
+   */
+  public static CloseableHttpClient createBasicAuthClient(String username, String password)
+      throws Exception {
+    HttpClientBuilder builder = HttpClients.custom();
+    configureTrustAllSSL(builder);
+
+    String authHeader =
+        "Basic "
+            + java.util.Base64.getEncoder()
+                .encodeToString(
+                    (username + ":" + password).getBytes(java.nio.charset.StandardCharsets.UTF_8));
+    List<Header> headers = new ArrayList<>();
+    headers.add(new BasicHeader(HttpHeaders.AUTHORIZATION, authHeader));
+    builder.setDefaultHeaders(headers);
+
+    RequestConfig config =
+        RequestConfig.custom()
+            .setConnectTimeout(CONNECTION_TIMEOUT_MS)
+            .setSocketTimeout(SOCKET_TIMEOUT_MS)
+            .setCircularRedirectsAllowed(true) // Helpful for some SSO redirects
+            .build();
+    builder.setDefaultRequestConfig(config);
+
+    return builder.build();
+  }
+
+  private static HttpClientBuilder configureTrustAllSSL(HttpClientBuilder builder)
+      throws Exception {
+    builder.setSSLContext(
+        new SSLContextBuilder().loadTrustMaterial(null, TrustAllStrategy.INSTANCE).build());
+    builder.setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE);
+    return builder;
+  }
+
+  private static void authenticateViaForm(
+      URI apiUri, CloseableHttpClient httpClient, String username, String password)
+      throws Exception {
+    URI baseUri = apiUri.resolve("/");
+    HttpPost post = new HttpPost(baseUri + "/j_spring_security_check");
+    List<NameValuePair> urlParameters = new ArrayList<>();
+    urlParameters.add(new BasicNameValuePair("j_username", username));
+    urlParameters.add(new BasicNameValuePair("j_password", password));
+
+    post.setEntity(new UrlEncodedFormEntity(urlParameters));
+
+    try (CloseableHttpResponse loginResponse = httpClient.execute(post)) {
+      // Check Home Page to verify the session cookie is valid
+      try (CloseableHttpResponse homeResponse =
+          httpClient.execute(new HttpGet(baseUri + "/cmf/home"))) {
+        if (HttpStatus.SC_OK != homeResponse.getStatusLine().getStatusCode()) {
+          logger.error("Login failed. Home Page response: {}", homeResponse.getStatusLine());
+          throw new MetadataDumperUsageException("Cloudera Manager login failed: " + loginResponse);
+        }
+      }
+    }
+    logger.info("Successfully logged into Cloudera Manager.");
+  }
+}

dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaManagerConnector.java

@@ -39,12 +39,7 @@
 import java.util.List;
 import java.util.function.Supplier;
 import javax.annotation.Nonnull;
-import org.apache.http.conn.ssl.NoopHostnameVerifier;
-import org.apache.http.conn.ssl.TrustAllStrategy;
 import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.HttpClientBuilder;
-import org.apache.http.impl.client.HttpClients;
-import org.apache.http.ssl.SSLContextBuilder;
 
 @AutoService({Connector.class})
 @Description("Dumps metadata from Cloudera Manager.")
@@ -122,13 +117,16 @@ public void addTasksTo(@Nonnull List<? super Task<?>> out, @Nonnull ConnectorArg
   @Nonnull
   @Override
   public ClouderaManagerHandle open(@Nonnull ConnectorArguments arguments) throws Exception {
-    URI uri = new URI(arguments.getUri());
-    CloseableHttpClient httpClient = disableSSLVerification(HttpClients.custom()).build();
-    ClouderaManagerHandle handle = new ClouderaManagerHandle(uri, httpClient);
-
+    URI apiUri = new URI(arguments.getUri());
     String user = arguments.getUser();
     String password = arguments.getPasswordOrPrompt();
-    doClouderaManagerLogin(handle.getBaseURI(), httpClient, user, password);
+
+    CloseableHttpClient clouderaManagerClient =
+        ClouderaHttpClientFactory.createClouderaManagerClient(apiUri, user, password);
+    CloseableHttpClient basicAuthClient =
+        ClouderaHttpClientFactory.createBasicAuthClient(user, password);
+    ClouderaManagerHandle handle =
+        new ClouderaManagerHandle(apiUri, clouderaManagerClient, basicAuthClient);
 
     ClouderaConnectorVerifier.verify(handle, arguments);
 
@@ -146,19 +144,4 @@ public final void validate(@Nonnull ConnectorArguments arguments) {
 
     validateDateRange(arguments);
   }
-
-  private void doClouderaManagerLogin(
-      URI baseURI, CloseableHttpClient httpClient, String user, String password) throws Exception {
-    ClouderaManagerLoginHelper.login(baseURI, httpClient, user, password);
-  }
-
-  private HttpClientBuilder disableSSLVerification(HttpClientBuilder builder) throws Exception {
-    // Cloudera Manager API SSL certificate is not in list of know certificates.
-    // So, switch off SSL certificate validation.
-    // It is  expected that Dumper will work  in internal private network (probably localhost calls)
-    builder.setSSLContext(
-        new SSLContextBuilder().loadTrustMaterial(null, TrustAllStrategy.INSTANCE).build());
-    builder.setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE);
-    return builder;
-  }
 }

dumper/app/src/main/java/com/google/edwmigration/dumper/application/dumper/connector/cloudera/manager/ClouderaManagerHandle.java

@@ -33,19 +33,25 @@
 public class ClouderaManagerHandle implements Handle {
 
   private final URI apiURI;
-  private final CloseableHttpClient httpClient;
+  private final CloseableHttpClient clouderaManagerHttpClient;
+  private final CloseableHttpClient basicAuthHttpClient;
 
   private ImmutableList<ClouderaClusterDTO> clusters;
   private ImmutableList<ClouderaHostDTO> hosts;
   private ImmutableList<ClouderaYarnApplicationDTO> sparkYarnApplications;
 
-  public ClouderaManagerHandle(URI apiURI, CloseableHttpClient httpClient) {
+  public ClouderaManagerHandle(
+      URI apiURI,
+      CloseableHttpClient clouderaManagerHttpClient,
+      CloseableHttpClient basicAuthHttpClient) {
     Preconditions.checkNotNull(apiURI, "Cloudera's apiURI can't be null.");
-    Preconditions.checkNotNull(httpClient, "httpClient can't be null.");
+    Preconditions.checkNotNull(clouderaManagerHttpClient, "ClouderaManager client can't be null.");
+    Preconditions.checkNotNull(basicAuthHttpClient, "BasicAuth client can't be null.");
 
     // Always add trailing slash for safety
     this.apiURI = unify(apiURI);
-    this.httpClient = httpClient;
+    this.clouderaManagerHttpClient = clouderaManagerHttpClient;
+    this.basicAuthHttpClient = basicAuthHttpClient;
   }
 
   /** 1. Remove query params and url fragments 2. Add trailing slash for safety */
@@ -73,8 +79,12 @@ public URI getBaseURI() {
     return apiURI.resolve("/");
   }
 
-  public CloseableHttpClient getHttpClient() {
-    return httpClient;
+  public CloseableHttpClient getClouderaManagerHttpClient() {
+    return clouderaManagerHttpClient;
+  }
+
+  public CloseableHttpClient getBasicAuthHttpClient() {
+    return basicAuthHttpClient;
   }
 
   @CheckForNull
@@ -121,14 +131,13 @@ public synchronized void initSparkYarnApplications(
 
   @Override
   public void close() throws IOException {
-    if (httpClient != null) {
-      try {
-        httpClient.close();
-      } catch (IOException ignore) {
-        // The intention is to do graceful shutdown and try to release the resource.
-        // In case of errors we do not need to interrupt the execution flow
-        // because the e2e use case might be successful
-      }
+    try {
+      clouderaManagerHttpClient.close();
+      basicAuthHttpClient.close();
+    } catch (IOException ignore) {
+      // The intention is to do graceful shutdown and try to release the resource.
+      // In case of errors we do not need to interrupt the execution flow
+      // because the e2e use case might be successful
     }
   }