[KYUUBI #6875] [K8S][HELM] Add Hadoop configuration files support

### Why are the changes needed?
The PR adds support for Hadoop configuration files to be used by Apache Kyuubi, Apache Spark etc.

The PR is continuation of PR #6521 and relates to the issue #6123.

### How was this patch tested?
1. Create `hadoop-configs.yaml` file (ConfigMap with `core-site.xml` and `hive-site.xml` entries):
```yaml
apiVersion: v1
kind: ConfigMap
metadata:
  name: hadoop-configs
data:
  'core-site.xml': |
    <?xml version="1.0"?>
    <?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
    <configuration>
        <property>
            <name>hadoop.pr.test</name>
            <value>configmap</value>
        </property>
    </configuration>
  'hive-site.xml': |
    <?xml version="1.0"?>
    <?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
    <configuration>
        <property>
            <name>hive.pr.test</name>
            <value>configmap</value>
        </property>
    </configuration>

```

2. Create ConfigMap from `hadoop-configs.yaml` file:
```shell
kubectl create -f hadoop-configs.yaml
```

3. Create custom `values-hadoop.yaml` (overwrites `core-site.xml`):
```yaml
image:
  repository: apache/kyuubi
  tag: 1.10.0-spark

rbac:
  create: true
  rules:
    - apiGroups: [""]
      resources: ["pods", "configmaps", "services"]
      verbs: ["create", "list", "delete", "watch", "deletecollection", "get"]

hadoopConf:
  files:
   'core-site.xml': |
     <?xml version="1.0"?>
     <?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
     <configuration>
       <property>
         <name>hadoop.pr.test</name>
         <value>values</value>
       </property>
     </configuration>
   'hdfs-site.xml': |
     <?xml version="1.0"?>
     <?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
     <configuration>
       <property>
         <name>hdfs.pr.test</name>
         <value>values</value>
       </property>
     </configuration>
  filesFrom:
  - configMap:
      name: hadoop-configs

sparkConf:
  files:
    'spark-defaults.conf': |
      spark.submit.deployMode=client
      spark.kubernetes.container.image=apache/spark:3.5.2
      spark.kubernetes.authenticate.driver.serviceAccountName=kyuubi
```

4. Install the chart
```shell
helm install kyuubi charts/kyuubi -f values-hadoop.yaml
```

5. Check there are 3 files in the Hadoop configuration directory:
```shell
kubectl exec kyuubi-0 -- ls /opt/hadoop/conf

core-site.xml
hdfs-site.xml
hive-site.xml
```
6. Check `/opt/hadoop/conf/core-site.xml` has content from ConfigMap:
```shell
kubectl exec kyuubi-0 -- cat /opt/hadoop/conf/core-site.xml

<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
    <property>
        <name>hadoop.pr.test</name>
        <value>configmap</value>
    </property>
</configuration>
```

7.  Check `/opt/hadoop/conf/hdfs-site.xml` has content from values:
```shell
kubectl exec kyuubi-0 -- cat /opt/hadoop/conf/hdfs-site.xml

<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
  <property>
    <name>hdfs.pr.test</name>
    <value>values</value>
  </property>
</configuration>
```

8.  Check `/opt/hadoop/conf/hive-site.xml` has content from ConfigMap:
```shell
kubectl exec kyuubi-0 -- cat /opt/hadoop/conf/hive-site.xml

<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
    <property>
        <name>hive.pr.test</name>
        <value>configmap</value>
    </property>
</configuration>
```

9. Check configuration values from Spark session:
```shell
kubectl exec kyuubi-0 -- ./bin/beeline -u 'jdbc:hive2://kyuubi-thrift-binary:10009' -e 'set hadoop.pr.test;'
+-----------------+------------+
|       key       |   value    |
+-----------------+------------+
| hadoop.pr.test  | configmap  |
+-----------------+------------

kubectl exec kyuubi-0 -- ./bin/beeline -u 'jdbc:hive2://kyuubi-thrift-binary:10009' -e 'set hdfs.pr.test;'
+---------------+---------+
|      key      |  value  |
+---------------+---------+
| hdfs.pr.test  | values  |
+---------------+---------

kubectl exec kyuubi-0 -- ./bin/beeline -u 'jdbc:hive2://kyuubi-thrift-binary:10009' -e 'set hive.pr.test;'
+---------------+------------+
|      key      |   value    |
+---------------+------------+
| hive.pr.test  | configmap  |
+---------------+------------
```

### Was this patch authored or co-authored using generative AI tooling?
No

Closes #6875 from dnskr/helm-add-hadoop-configs-support.

Closes #6875

8c8665f [dnskr] [K8S][HELM] Add Hadoop configuration files support

Authored-by: dnskr <[email protected]>
Signed-off-by: dnskr <[email protected]>

Author: dnskr

charts/kyuubi/templates/kyuubi-hadoop-configmap.yaml

@@ -0,0 +1,27 @@
+{{/*
+  Licensed to the Apache Software Foundation (ASF) under one or more
+  contributor license agreements.  See the NOTICE file distributed with
+  this work for additional information regarding copyright ownership.
+  The ASF licenses this file to You under the Apache License, Version 2.0
+  (the "License"); you may not use this file except in compliance with
+  the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+*/}}
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ .Release.Name }}-hadoop
+  labels:
+    {{- include "kyuubi.labels" . | nindent 4 }}
+data:
+  {{- with .Values.hadoopConf.files }}
+    {{- tpl (toYaml .) $ | nindent 2 }}
+  {{- end }}

charts/kyuubi/templates/kyuubi-statefulset.yaml

@@ -39,6 +39,7 @@ spec:
         {{- include "kyuubi.selectorLabels" . | nindent 8 }}
       annotations:
         checksum/conf: {{ include (print $.Template.BasePath "/kyuubi-configmap.yaml") . | sha256sum }}
+        checksum/conf-hadoop: {{ include (print $.Template.BasePath "/kyuubi-hadoop-configmap.yaml") . | sha256sum }}
     spec:
       {{- with .Values.imagePullSecrets }}
       imagePullSecrets: {{- toYaml . | nindent 8 }}
@@ -79,6 +80,8 @@ spec:
           env:
             - name: KYUUBI_CONF_DIR
               value: {{ .Values.kyuubiConf.dir }}
+            - name: HADOOP_CONF_DIR
+              value: {{ .Values.hadoopConf.dir }}
             - name: SPARK_CONF_DIR
               value: {{ .Values.sparkConf.dir }}
             {{- with .Values.env }}
@@ -124,6 +127,8 @@ spec:
           volumeMounts:
             - name: conf
               mountPath: {{ .Values.kyuubiConf.dir }}
+            - name: conf-hadoop
+              mountPath: {{ .Values.hadoopConf.dir }}
             - name: conf-spark
               mountPath: {{ .Values.sparkConf.dir }}
             {{- with .Values.volumeMounts }}
@@ -141,6 +146,14 @@ spec:
               {{- with .Values.kyuubiConf.filesFrom }}
                 {{- tpl (toYaml .) $ | nindent 14 }}
               {{- end }}
+        - name: conf-hadoop
+          projected:
+            sources:
+              - configMap:
+                  name: {{ .Release.Name }}-hadoop
+              {{- with .Values.hadoopConf.filesFrom }}
+                {{- tpl (toYaml .) $ | nindent 14 }}
+              {{- end }}
         - name: conf-spark
           projected:
             sources:

charts/kyuubi/values.yaml

@@ -171,6 +171,38 @@ kyuubiConf:
   #      - key: trust-store
   #        path: certs/truststore.jks
 
+# Hadoop configuration files
+hadoopConf:
+  # $HADOOP_CONF_DIR directory
+  dir: /opt/hadoop/conf
+  # Configuration files from the specified keys (file name) and values (file content)
+  files: ~
+  #files:
+  # 'core-site.xml': |
+  #   <?xml version="1.0"?>
+  #   <?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+  #   <configuration>
+  #     <property>
+  #       <name>hadoop.tmp.dir</name>
+  #       <value>/tmp/hadoop-${user.name}</value>
+  #     </property>
+  #   </configuration>
+
+  # Configuration files from the list of existing ConfigMaps and Secrets
+  filesFrom: []
+  #filesFrom:
+  #- configMap:
+  #    name: hadoop-configs
+  #- secret:
+  #    name: hadoop-secrets
+  #- secret:
+  #    name: ssl-secrets
+  #    items:
+  #      - key: key-store
+  #        path: certs/keystore.jks
+  #      - key: trust-store
+  #        path: certs/truststore.jks
+
 # Spark configuration, see https://github.com/apache/spark/tree/master/conf and Spark documentation for more details
 sparkConf:
   # $SPARK_CONF_DIR directory