Merge branch 'cloudera-labs:main' into feature-selection

cloudera-labs · Nov 9, 2023 · 63efb1b · 63efb1b
2 parents 957ad0c + d25a610
commit 63efb1b
Show file tree

Hide file tree

Showing 6 changed files with 86 additions and 3 deletions.
diff --git a/galaxy.yml b/galaxy.yml
@@ -16,7 +16,7 @@
 
 namespace:  cloudera
 name:       cluster
-version:    4.0.1
+version:    4.1.1
 readme:     README.md
 authors:    []
 

diff --git a/roles/config/cluster/base/templates/configs/databases-7.1.9.j2 b/roles/config/cluster/base/templates/configs/databases-7.1.9.j2
@@ -0,0 +1,69 @@
+---
+DAS:
+  SERVICEWIDE:
+    data_analytics_studio_database_host: {{ databases.DAS.host }}
+    data_analytics_studio_database_type: {{ databases.DAS.type | cloudera.cluster.format_database_type }}
+    data_analytics_studio_database_port: {{ databases.DAS.port }}
+    data_analytics_studio_database_name: {{ databases.DAS.name }}
+    data_analytics_studio_database_username: {{ databases.DAS.user }}
+    data_analytics_studio_database_password: {{ databases.DAS.password }}
+RANGER:
+  SERVICEWIDE:
+    ranger_database_host: {{ databases.RANGER.host }}
+    ranger_database_port: {{ databases.RANGER.port }}
+    ranger_database_type: {{ databases.RANGER.type | cloudera.cluster.format_database_type }}
+    ranger_database_name: {{ databases.RANGER.name }}
+    ranger_database_user: {{ databases.RANGER.user }}
+    ranger_database_password: {{ databases.RANGER.password }}
+RANGER_RMS:
+  SERVICEWIDE:
+    ranger_rms_database_host: {{ databases.RANGER.host }}
+    ranger_rms_database_port: {{ databases.RANGER.port }}
+    ranger_rms_database_type: {{ databases.RANGER.type | cloudera.cluster.format_database_type }}
+    ranger_rms_database_name: {{ databases.RANGER.name }}
+    ranger_rms_database_user: {{ databases.RANGER.user }}
+    ranger_rms_database_password: {{ databases.RANGER.password }}
+SCHEMAREGISTRY:
+  SERVICEWIDE:
+    database_host: {{ databases.SCHEMAREGISTRY.host }}
+    database_port: {{ databases.SCHEMAREGISTRY.port }}
+    database_type: {{ databases.SCHEMAREGISTRY.type | cloudera.cluster.format_database_type }}
+    database_name: {{ databases.SCHEMAREGISTRY.name }}
+    database_user: {{ databases.SCHEMAREGISTRY.user }}
+    database_password: {{ databases.SCHEMAREGISTRY.password }}
+SQL_STREAM_BUILDER:
+  SERVICEWIDE:
+    database_host: {{ databases.SQL_STREAM_BUILDER.host }}
+    database_port: {{ databases.SQL_STREAM_BUILDER.port }}
+    database_type: {{ databases.SQL_STREAM_BUILDER.type | cloudera.cluster.format_database_type }}
+    database_schema: {{ databases.SQL_STREAM_BUILDER.name }}
+    database_user: {{ databases.SQL_STREAM_BUILDER.user }}
+    database_password: {{ databases.SQL_STREAM_BUILDER.password }}
+  MATERIALIZED_VIEW_ENGINE:
+    ssb.mve.datasource.url: jdbc:{{ databases.SQL_STREAM_BUILDER_MVE.type | cloudera.cluster.format_database_type }}://{{ databases.SQL_STREAM_BUILDER_MVE.host }}:{{ databases.SQL_STREAM_BUILDER_MVE.port }}/{{ databases.SQL_STREAM_BUILDER_MVE.name }}
+    ssb.mve.datasource.username: {{ databases.SQL_STREAM_BUILDER_MVE.user }}
+    ssb.mve.datasource.password: {{ databases.SQL_STREAM_BUILDER_MVE.password }}
+STREAMS_MESSAGING_MANAGER:
+  SERVICEWIDE:
+    smm_database_host: {{ databases.STREAMS_MESSAGING_MANAGER.host }}
+    smm_database_port: {{ databases.STREAMS_MESSAGING_MANAGER.port }}
+    smm_database_type: {{ databases.STREAMS_MESSAGING_MANAGER.type | cloudera.cluster.format_database_type }}
+    smm_database_name: {{ databases.STREAMS_MESSAGING_MANAGER.name }}
+    smm_database_user: {{ databases.STREAMS_MESSAGING_MANAGER.user }}
+    smm_database_password: {{ databases.STREAMS_MESSAGING_MANAGER.password }}
+QUERY_PROCESSOR:
+  SERVICEWIDE:
+    query_processor_database_host: {{ databases.QUERY_PROCESSOR.host }}
+    query_processor_database_port: {{ databases.QUERY_PROCESSOR.port }}
+    query_processor_database_type: {{ databases.QUERY_PROCESSOR.type | cloudera.cluster.format_database_type }}
+    query_processor_database_name: {{ databases.QUERY_PROCESSOR.name }}
+    query_processor_database_username: {{ databases.QUERY_PROCESSOR.user }}
+    query_processor_database_password: {{ databases.QUERY_PROCESSOR.password }}
+QUEUE_MANAGER:
+  SERVICEWIDE:
+    queue_manager_database_host: {{ databases.QUEUE_MANAGER.host }}
+    queue_manager_database_port: {{ databases.QUEUE_MANAGER.port }}
+    queue_manager_database_type: {{ databases.QUEUE_MANAGER.type | cloudera.cluster.format_database_type }}
+    queue_manager_database_name: {{ databases.QUEUE_MANAGER.name }}
+    queue_manager_database_username: {{ databases.QUEUE_MANAGER.user }}
+    queue_manager_database_password: {{ databases.QUEUE_MANAGER.password }}
diff --git a/roles/config/cluster/base/templates/configs/inter-service-dependencies.j2 b/roles/config/cluster/base/templates/configs/inter-service-dependencies.j2
@@ -28,7 +28,9 @@ HIVE:
 {% if 'ATLAS' in cluster.services and not (cdh_cdp_upgrade|default(false)|bool) %}
     atlas_service: atlas
 {% endif %}
+{% if cloudera_runtime_version is version('7.0.0','<=') %}
     spark_on_yarn_service: spark_on_yarn
+{% endif %}
     zookeeper_service: zookeeper
 
 HIVE_ON_TEZ:

diff --git a/roles/config/cluster/base/vars/main.yml b/roles/config/cluster/base/vars/main.yml
@@ -21,7 +21,9 @@ custom_config_templates:
 # Custom configurations for databases
   - template: configs/databases.j2
   - template: configs/databases-7.1.0.j2
-    condition: "{{ cloudera_runtime_version is version('7.1.0','>=') }}"
+    condition: "{{ cloudera_runtime_version is version('7.1.0','>=') and cloudera_runtime_version is version('7.1.9','<') }}"
+  - template: configs/databases-7.1.9.j2
+    condition: "{{ cloudera_runtime_version is version('7.1.9','>=') }}"
 # Custom configurations for Infra Solr
   - template: configs/infra-solr.j2
     condition: "{{ 'INFRA_SOLR' in cluster.services }}"

diff --git a/roles/deployment/cluster/templates/upgrade_scripts/fs2cs.j2 b/roles/deployment/cluster/templates/upgrade_scripts/fs2cs.j2
@@ -48,6 +48,6 @@ mkdir fs2cs
 cp ${FAIR_SCHEDULER} fs2cs/fair-scheduler.xml
 cp ${CORE_SITE} fs2cs/core-site.xml
 
-/opt/cloudera/parcels/CDH-{{ new_parcel_version }}/bin/yarn fs2cs ${RESOURCE_OPTS}  --no-terminal-rule-check --yarnsiteconfig ${YARN_SITE}  --fsconfig fs2cs/fair-scheduler.xml --output-directory {{ fs2cs_temp_dir }}
+/opt/cloudera/parcels/CDH-{{ new_parcel_version }}/bin/yarn fs2cs ${RESOURCE_OPTS}  --no-terminal-rule-check --yarnsiteconfig ${YARN_SITE} --fsconfig $(realpath fs2cs/fair-scheduler.xml) --output-directory {{ fs2cs_temp_dir }}
 
 exit $?
diff --git a/roles/deployment/definition/defaults/main.yml b/roles/deployment/definition/defaults/main.yml
@@ -121,6 +121,16 @@ database_defaults:
     user: queryprocessor
     password: "{{ database_default_password }}"
 
+#New in 7.1.9, postgresql only until CHF2
+  QUEUE_MANAGER:
+    host: "{{ database_host }}"
+    port: "{{ database_type | cloudera.cluster.default_database_port }}"
+    type: postgresql
+    name: queuemanager
+    user: queuemanager
+    password: "{{ database_default_password }}"
+
+
 databases_cm_svcs:
   ACTIVITYMONITOR:
     host: "{{ database_host }}"