Merge branch 'apache:master' into jdbc-oracle

.github/PULL_REQUEST_TEMPLATE

@@ -1,38 +1,33 @@
-# :mag: Description
-## Issue References 🔗
-<!-- Append the issue number after #. If there is no issue for you to link create one or -->
-<!-- If there are no issues to link, please provide details here. -->
+<!--
+Thanks for sending a pull request!
+
+Here are some tips for you:
+  1. If this is your first time, please read our contributor guidelines: https://kyuubi.readthedocs.io/en/master/contributing/code/index.html
+  2. If the PR is related to an issue in https://github.com/apache/kyuubi/issues, add '[KYUUBI #XXXX]' in your PR title, e.g., '[KYUUBI #XXXX] Your PR title ...'.
+  3. If the PR is unfinished, add '[WIP]' in your PR title, e.g., '[WIP][KYUUBI #XXXX] Your PR title ...'.
+-->
+
+### Why are the changes needed?
+<!--
+Please clarify why the changes are needed. For instance,
+  1. If you propose a new API, clarify the use case for a new API.
+  2. If you fix a bug, you can clarify why it is a bug, and what versions are affected.
+-->
+
+
+### How was this patch tested?
+<!--
+If tests were added, say they were added here. Please make sure to add some test cases that check the changes thoroughly including negative and positive cases if possible.
+If it was tested in a way different from regular unit tests, please clarify how you tested step by step, ideally copy and paste-able, so that other reviewers can test and check, and descendants can verify in the future.
+If tests were not added, please describe why they were not added and/or why it was difficult to add.
+-->
+
+
+### Was this patch authored or co-authored using generative AI tooling?
+<!--
+If a generative AI tooling has been used in the process of authoring this patch, please include
+phrase 'Generated-by: ' followed by the name of the tool and its version.
+If no, write 'No'.
+Please refer to the [ASF Generative Tooling Guidance](https://www.apache.org/legal/generative-tooling.html) for details.
+-->
 
-This pull request fixes #
-
-## Describe Your Solution 🔧
-
-Please include a summary of the change and which issue is fixed. Please also include relevant motivation and context. List any dependencies that are required for this change.
-
-
-## Types of changes :bookmark:
-<!--- What types of changes does your code introduce? Put an `x` in all the boxes that apply: -->
-- [ ] Bugfix (non-breaking change which fixes an issue)
-- [ ] New feature (non-breaking change which adds functionality)
-- [ ] Breaking change (fix or feature that would cause existing functionality to change)
-
-## Test Plan 🧪
-
-#### Behavior Without This Pull Request :coffin:
-
-
-#### Behavior With This Pull Request :tada:
-
-
-#### Related Unit Tests
-
-
----
-
-# Checklist 📝
-<!--- Go over all the following points, and put an `x` in all the boxes that apply. -->
-<!--- If you're unsure about any of these, don't hesitate to ask. We're here to help! -->
-
-- [ ] This patch was not authored or co-authored using [Generative Tooling](https://www.apache.org/legal/generative-tooling.html)
-
-**Be nice. Be informative.**

charts/kyuubi/templates/kyuubi-configmap.yaml

@@ -22,30 +22,6 @@ metadata:
   labels:
     {{- include "kyuubi.labels" . | nindent 4 }}
 data:
-  {{- with .Values.kyuubiConf.kyuubiEnv }}
-  kyuubi-env.sh: |
-    {{- tpl . $ | nindent 4 }}
-  {{- end }}
-  kyuubi-defaults.conf: |
-    ## Helm chart provided Kyuubi configurations
-    kyuubi.kubernetes.namespace={{ .Release.Namespace }}
-    kyuubi.frontend.connection.url.use.hostname=false
-    kyuubi.frontend.thrift.binary.bind.port={{ .Values.server.thriftBinary.port }}
-    kyuubi.frontend.thrift.http.bind.port={{ .Values.server.thriftHttp.port }}
-    kyuubi.frontend.rest.bind.port={{ .Values.server.rest.port }}
-    kyuubi.frontend.mysql.bind.port={{ .Values.server.mysql.port }}
-    kyuubi.frontend.protocols={{ include "kyuubi.frontend.protocols" . }}
-
-    # Kyuubi Metrics
-    kyuubi.metrics.enabled={{ .Values.metrics.enabled }}
-    kyuubi.metrics.reporters={{ .Values.metrics.reporters }}
-    kyuubi.metrics.prometheus.port={{ .Values.metrics.prometheusPort }}
-
-    ## User provided Kyuubi configurations
-    {{- with .Values.kyuubiConf.kyuubiDefaults }}
-      {{- tpl . $ | nindent 4 }}
-    {{- end }}
-  {{- with .Values.kyuubiConf.log4j2 }}
-  log4j2.xml: |
-    {{- tpl . $ | nindent 4 }}
+  {{- with .Values.kyuubiConf.files }}
+    {{- tpl (toYaml .) $ | nindent 2 }}
   {{- end }}

charts/kyuubi/templates/kyuubi-spark-configmap.yaml

@@ -22,19 +22,6 @@ metadata:
   labels:
     {{- include "kyuubi.labels" . | nindent 4 }}
 data:
-  {{- with .Values.sparkConf.sparkEnv }}
-  spark-env.sh: |
-    {{- tpl . $ | nindent 4 }}
-  {{- end }}
-  {{- with .Values.sparkConf.sparkDefaults }}
-  spark-defaults.conf: |
-    {{- tpl . $ | nindent 4 }}
-  {{- end }}
-  {{- with .Values.sparkConf.log4j2 }}
-  log4j2.properties: |
-    {{- tpl . $ | nindent 4 }}
-  {{- end }}
-  {{- with .Values.sparkConf.metrics }}
-  metrics.properties: |
-    {{- tpl . $ | nindent 4 }}
+  {{- with .Values.sparkConf.files }}
+    {{- tpl (toYaml .) $ | nindent 2 }}
   {{- end }}

charts/kyuubi/templates/kyuubi-statefulset.yaml

@@ -59,14 +59,28 @@ spec:
           {{- with .Values.command }}
           command: {{- tpl (toYaml .) $ | nindent 12 }}
           {{- end }}
-          {{- with .Values.args }}
+          {{- if .Values.args }}
           args: {{- tpl (toYaml .) $ | nindent 12 }}
+          {{- else }}
+          args:
+            - ./bin/kyuubi
+            - run
+            - --conf kyuubi.kubernetes.namespace={{ .Release.Namespace }}
+            - --conf kyuubi.frontend.connection.url.use.hostname=false
+            - --conf kyuubi.frontend.thrift.binary.bind.port={{ .Values.server.thriftBinary.port }}
+            - --conf kyuubi.frontend.thrift.http.bind.port={{ .Values.server.thriftHttp.port }}
+            - --conf kyuubi.frontend.rest.bind.port={{ .Values.server.rest.port }}
+            - --conf kyuubi.frontend.mysql.bind.port={{ .Values.server.mysql.port }}
+            - --conf kyuubi.frontend.protocols={{ include "kyuubi.frontend.protocols" . }}
+            - --conf kyuubi.metrics.enabled={{ .Values.metrics.enabled }}
+            - --conf kyuubi.metrics.reporters={{ .Values.metrics.reporters }}
+            - --conf kyuubi.metrics.prometheus.port={{ .Values.metrics.prometheusPort }}
           {{- end }}
           env:
             - name: KYUUBI_CONF_DIR
-              value: {{ .Values.kyuubiConfDir }}
+              value: {{ .Values.kyuubiConf.dir }}
             - name: SPARK_CONF_DIR
-              value: {{ .Values.sparkConfDir }}
+              value: {{ .Values.sparkConf.dir }}
             {{- with .Values.env }}
               {{- tpl (toYaml .) $ | nindent 12 }}
             {{- end }}
@@ -109,9 +123,9 @@ spec:
           {{- end }}
           volumeMounts:
             - name: conf
-              mountPath: {{ .Values.kyuubiConfDir }}
+              mountPath: {{ .Values.kyuubiConf.dir }}
             - name: conf-spark
-              mountPath: {{ .Values.sparkConfDir }}
+              mountPath: {{ .Values.sparkConf.dir }}
             {{- with .Values.volumeMounts }}
               {{- tpl (toYaml .) $ | nindent 12 }}
             {{- end }}
@@ -120,11 +134,21 @@ spec:
         {{- end }}
       volumes:
         - name: conf
-          configMap:
-            name: {{ .Release.Name }}
+          projected:
+            sources:
+              - configMap:
+                  name: {{ .Release.Name }}
+              {{- with .Values.kyuubiConf.filesFrom }}
+                {{- tpl (toYaml .) $ | nindent 14 }}
+              {{- end }}
         - name: conf-spark
-          configMap:
-            name: {{ .Release.Name }}-spark
+          projected:
+            sources:
+              - configMap:
+                  name: {{ .Release.Name }}-spark
+              {{- with .Values.sparkConf.filesFrom }}
+                {{- tpl (toYaml .) $ | nindent 14 }}
+              {{- end }}
         {{- with .Values.volumes }}
           {{- tpl (toYaml .) $ | nindent 8 }}
         {{- end }}

charts/kyuubi/values.yaml

@@ -145,53 +145,40 @@ server:
       #    clientIP:
       #      timeoutSeconds: 10800
 
-# $KYUUBI_CONF_DIR directory
-kyuubiConfDir: /opt/kyuubi/conf
 # Kyuubi configuration files
 kyuubiConf:
-  # The value (templated string) is used for kyuubi-env.sh file
-  # See example at conf/kyuubi-env.sh.template and https://kyuubi.readthedocs.io/en/master/configuration/settings.html#environments for more details
-  kyuubiEnv: ~
-  #  kyuubiEnv: |
-  #    #!/usr/bin/env bash
-  #    export JAVA_HOME=/usr/jdk64/jdk1.8.0_152
-  #    export SPARK_HOME=/opt/spark
-  #    export FLINK_HOME=/opt/flink
-  #    export HIVE_HOME=/opt/hive
-
-  # The value (templated string) is used for kyuubi-defaults.conf file
-  # See https://kyuubi.readthedocs.io/en/master/configuration/settings.html#kyuubi-configurations for more details
-  kyuubiDefaults: ~
-  #  kyuubiDefaults: |
+  # $KYUUBI_CONF_DIR directory
+  dir: /opt/kyuubi/conf
+  # Configuration files from the specified keys (file name) and values (file content)
+  files: ~
+  #files:
+  #  'kyuubi-defaults.conf': |
   #    kyuubi.authentication=NONE
-  #    kyuubi.frontend.bind.host=10.0.0.1
-  #    kyuubi.engine.type=SPARK_SQL
   #    kyuubi.engine.share.level=USER
-  #    kyuubi.session.engine.initialize.timeout=PT3M
-  #    kyuubi.ha.addresses=zk1:2181,zk2:2181,zk3:2181
-  #    kyuubi.ha.namespace=kyuubi
 
-  # The value (templated string) is used for log4j2.xml file
-  # See example at conf/log4j2.xml.template https://kyuubi.readthedocs.io/en/master/configuration/settings.html#logging for more details
-  log4j2: ~
-
-# $SPARK_CONF_DIR directory
-sparkConfDir: /opt/spark/conf
-# Spark configuration files
+  # Configuration files from the list of existing ConfigMaps and Secrets
+  filesFrom: []
+  #filesFrom:
+  #- configMap:
+  #    name: kyuubi-configs
+  #- secret:
+  #    name: kyuubi-secrets
+  #- secret:
+  #    name: ssl-secrets
+  #    items:
+  #      - key: key-store
+  #        path: certs/keystore.jks
+  #      - key: trust-store
+  #        path: certs/truststore.jks
+
+# Spark configuration, see https://github.com/apache/spark/tree/master/conf and Spark documentation for more details
 sparkConf:
-  # The value (templated string) is used for spark-env.sh file
-  # See example at https://github.com/apache/spark/blob/master/conf/spark-env.sh.template and Spark documentation for more details
-  sparkEnv: ~
-  #  sparkEnv: |
-  #    #!/usr/bin/env bash
-  #    export JAVA_HOME=/usr/jdk64/jdk1.8.0_152
-  #    export SPARK_LOG_DIR=/opt/spark/logs
-  #    export SPARK_LOG_MAX_FILES=5
-
-  # The value (templated string) is used for spark-defaults.conf file
-  # See example at https://github.com/apache/spark/blob/master/conf/spark-defaults.conf.template and Spark documentation for more details
-  sparkDefaults: ~
-  #  sparkDefaults: |
+  # $SPARK_CONF_DIR directory
+  dir: /opt/spark/conf
+  # Configuration files from the specified keys (file name) and values (file content)
+  files: ~
+  #files:
+  #  'spark-defaults.conf': |
   #    spark.submit.deployMode=cluster
   #    spark.kubernetes.container.image=apache/spark:3.5.0
   #    spark.kubernetes.authenticate.driver.serviceAccountName=spark
@@ -207,13 +194,20 @@ sparkConf:
   #    spark.hadoop.fs.s3a.path.style.access=true
   #    spark.hadoop.fs.s3a.fast.upload=true
 
-  # The value (templated string) is used for log4j2.properties file
-  # See example at https://github.com/apache/spark/blob/master/conf/log4j2.properties.template and Spark documentation for more details
-  log4j2: ~
-
-  # The value (templated string) is used for metrics.properties file
-  # See example at https://github.com/apache/spark/blob/master/conf/metrics.properties.template and Spark documentation for more details
-  metrics: ~
+  # Configuration files from the list of existing ConfigMaps and Secrets
+  filesFrom: []
+  #filesFrom:
+  #- configMap:
+  #    name: spark-configs
+  #- secret:
+  #    name: spark-secrets
+  #- secret:
+  #    name: ssl-secrets
+  #    items:
+  #      - key: key-store
+  #        path: certs/keystore.jks
+  #      - key: trust-store
+  #        path: certs/truststore.jks
 
 # Command to launch Kyuubi server (templated)
 command: ~

extensions/flink/kyuubi-flink-token-provider/pom.xml

@@ -69,9 +69,12 @@
             <plugin>
                 <groupId>net.alchim31.maven</groupId>
                 <artifactId>scala-maven-plugin</artifactId>
-                <configuration>
-                    <skip>true</skip>
-                </configuration>
+                <executions>
+                    <execution>
+                        <id>attach-scaladocs</id>
+                        <phase>none</phase>
+                    </execution>
+                </executions>
             </plugin>
         </plugins>
     </build>

extensions/server/kyuubi-server-plugin/pom.xml

@@ -35,9 +35,12 @@
             <plugin>
                 <groupId>net.alchim31.maven</groupId>
                 <artifactId>scala-maven-plugin</artifactId>
-                <configuration>
-                    <skip>true</skip>
-                </configuration>
+                <executions>
+                    <execution>
+                        <id>attach-scaladocs</id>
+                        <phase>none</phase>
+                    </execution>
+                </executions>
             </plugin>
         </plugins>
     </build>