From 6278053c2e07d23eaaf60287642ff00a7e49a572 Mon Sep 17 00:00:00 2001 From: anisf Date: Thu, 2 Nov 2023 23:32:46 +0400 Subject: [PATCH 1/6] Resolve full path of configuration for the fair-scheduler generation script (#119) Signed-off-by: Anis FATHALLAH --- roles/deployment/cluster/templates/upgrade_scripts/fs2cs.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/deployment/cluster/templates/upgrade_scripts/fs2cs.j2 b/roles/deployment/cluster/templates/upgrade_scripts/fs2cs.j2 index 5f692423..db5a1ed7 100644 --- a/roles/deployment/cluster/templates/upgrade_scripts/fs2cs.j2 +++ b/roles/deployment/cluster/templates/upgrade_scripts/fs2cs.j2 @@ -48,6 +48,6 @@ mkdir fs2cs cp ${FAIR_SCHEDULER} fs2cs/fair-scheduler.xml cp ${CORE_SITE} fs2cs/core-site.xml -/opt/cloudera/parcels/CDH-{{ new_parcel_version }}/bin/yarn fs2cs ${RESOURCE_OPTS} --no-terminal-rule-check --yarnsiteconfig ${YARN_SITE} --fsconfig fs2cs/fair-scheduler.xml --output-directory {{ fs2cs_temp_dir }} +/opt/cloudera/parcels/CDH-{{ new_parcel_version }}/bin/yarn fs2cs ${RESOURCE_OPTS} --no-terminal-rule-check --yarnsiteconfig ${YARN_SITE} --fsconfig $(realpath fs2cs/fair-scheduler.xml) --output-directory {{ fs2cs_temp_dir }} exit $? \ No newline at end of file From 9f3faa044b778491427981a2b8c8cc4057d4d4b1 Mon Sep 17 00:00:00 2001 From: anisf Date: Thu, 2 Nov 2023 23:38:38 +0400 Subject: [PATCH 2/6] Remove spark_on_yarn_service dependency for Hive for CDP 7.0.0 or higher (#123) Signed-off-by: Anis FATHALLAH --- .../base/templates/configs/inter-service-dependencies.j2 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/roles/config/cluster/base/templates/configs/inter-service-dependencies.j2 b/roles/config/cluster/base/templates/configs/inter-service-dependencies.j2 index f7d8c3c1..481a9aae 100644 --- a/roles/config/cluster/base/templates/configs/inter-service-dependencies.j2 +++ b/roles/config/cluster/base/templates/configs/inter-service-dependencies.j2 @@ -28,7 +28,9 @@ HIVE: {% if 'ATLAS' in cluster.services and not (cdh_cdp_upgrade|default(false)|bool) %} atlas_service: atlas {% endif %} +{% if cloudera_runtime_version is version('7.0.0','<=') %} spark_on_yarn_service: spark_on_yarn +{% endif %} zookeeper_service: zookeeper HIVE_ON_TEZ: From 00c71a1b054c46016c7d5b42ccc24fce8c771b1c Mon Sep 17 00:00:00 2001 From: Chuck Levesque Date: Thu, 2 Nov 2023 16:17:42 -0400 Subject: [PATCH 3/6] Add 7.1.9 QueueManager for postgresql (#152) In 7.1.9 QueueManager uses RDBMS. postgresql only supported in 7.1.9.0 & CHF1. Switch to back to legacy H2 included in CHF2, other db's later Signed-off-by: Chuck Levesque --- .../base/templates/configs/databases-7.1.9.j2 | 69 +++++++++++++++++++ roles/config/cluster/base/vars/main.yml | 4 +- roles/deployment/definition/defaults/main.yml | 10 +++ 3 files changed, 82 insertions(+), 1 deletion(-) create mode 100644 roles/config/cluster/base/templates/configs/databases-7.1.9.j2 diff --git a/roles/config/cluster/base/templates/configs/databases-7.1.9.j2 b/roles/config/cluster/base/templates/configs/databases-7.1.9.j2 new file mode 100644 index 00000000..9f786e35 --- /dev/null +++ b/roles/config/cluster/base/templates/configs/databases-7.1.9.j2 @@ -0,0 +1,69 @@ +--- +DAS: + SERVICEWIDE: + data_analytics_studio_database_host: {{ databases.DAS.host }} + data_analytics_studio_database_type: {{ databases.DAS.type | cloudera.cluster.format_database_type }} + data_analytics_studio_database_port: {{ databases.DAS.port }} + data_analytics_studio_database_name: {{ databases.DAS.name }} + data_analytics_studio_database_username: {{ databases.DAS.user }} + data_analytics_studio_database_password: {{ databases.DAS.password }} +RANGER: + SERVICEWIDE: + ranger_database_host: {{ databases.RANGER.host }} + ranger_database_port: {{ databases.RANGER.port }} + ranger_database_type: {{ databases.RANGER.type | cloudera.cluster.format_database_type }} + ranger_database_name: {{ databases.RANGER.name }} + ranger_database_user: {{ databases.RANGER.user }} + ranger_database_password: {{ databases.RANGER.password }} +RANGER_RMS: + SERVICEWIDE: + ranger_rms_database_host: {{ databases.RANGER.host }} + ranger_rms_database_port: {{ databases.RANGER.port }} + ranger_rms_database_type: {{ databases.RANGER.type | cloudera.cluster.format_database_type }} + ranger_rms_database_name: {{ databases.RANGER.name }} + ranger_rms_database_user: {{ databases.RANGER.user }} + ranger_rms_database_password: {{ databases.RANGER.password }} +SCHEMAREGISTRY: + SERVICEWIDE: + database_host: {{ databases.SCHEMAREGISTRY.host }} + database_port: {{ databases.SCHEMAREGISTRY.port }} + database_type: {{ databases.SCHEMAREGISTRY.type | cloudera.cluster.format_database_type }} + database_name: {{ databases.SCHEMAREGISTRY.name }} + database_user: {{ databases.SCHEMAREGISTRY.user }} + database_password: {{ databases.SCHEMAREGISTRY.password }} +SQL_STREAM_BUILDER: + SERVICEWIDE: + database_host: {{ databases.SQL_STREAM_BUILDER.host }} + database_port: {{ databases.SQL_STREAM_BUILDER.port }} + database_type: {{ databases.SQL_STREAM_BUILDER.type | cloudera.cluster.format_database_type }} + database_schema: {{ databases.SQL_STREAM_BUILDER.name }} + database_user: {{ databases.SQL_STREAM_BUILDER.user }} + database_password: {{ databases.SQL_STREAM_BUILDER.password }} + MATERIALIZED_VIEW_ENGINE: + ssb.mve.datasource.url: jdbc:{{ databases.SQL_STREAM_BUILDER_MVE.type | cloudera.cluster.format_database_type }}://{{ databases.SQL_STREAM_BUILDER_MVE.host }}:{{ databases.SQL_STREAM_BUILDER_MVE.port }}/{{ databases.SQL_STREAM_BUILDER_MVE.name }} + ssb.mve.datasource.username: {{ databases.SQL_STREAM_BUILDER_MVE.user }} + ssb.mve.datasource.password: {{ databases.SQL_STREAM_BUILDER_MVE.password }} +STREAMS_MESSAGING_MANAGER: + SERVICEWIDE: + smm_database_host: {{ databases.STREAMS_MESSAGING_MANAGER.host }} + smm_database_port: {{ databases.STREAMS_MESSAGING_MANAGER.port }} + smm_database_type: {{ databases.STREAMS_MESSAGING_MANAGER.type | cloudera.cluster.format_database_type }} + smm_database_name: {{ databases.STREAMS_MESSAGING_MANAGER.name }} + smm_database_user: {{ databases.STREAMS_MESSAGING_MANAGER.user }} + smm_database_password: {{ databases.STREAMS_MESSAGING_MANAGER.password }} +QUERY_PROCESSOR: + SERVICEWIDE: + query_processor_database_host: {{ databases.QUERY_PROCESSOR.host }} + query_processor_database_port: {{ databases.QUERY_PROCESSOR.port }} + query_processor_database_type: {{ databases.QUERY_PROCESSOR.type | cloudera.cluster.format_database_type }} + query_processor_database_name: {{ databases.QUERY_PROCESSOR.name }} + query_processor_database_username: {{ databases.QUERY_PROCESSOR.user }} + query_processor_database_password: {{ databases.QUERY_PROCESSOR.password }} +QUEUE_MANAGER: + SERVICEWIDE: + queue_manager_database_host: {{ databases.QUEUE_MANAGER.host }} + queue_manager_database_port: {{ databases.QUEUE_MANAGER.port }} + queue_manager_database_type: {{ databases.QUEUE_MANAGER.type | cloudera.cluster.format_database_type }} + queue_manager_database_name: {{ databases.QUEUE_MANAGER.name }} + queue_manager_database_username: {{ databases.QUEUE_MANAGER.user }} + queue_manager_database_password: {{ databases.QUEUE_MANAGER.password }} diff --git a/roles/config/cluster/base/vars/main.yml b/roles/config/cluster/base/vars/main.yml index 72e4fbfa..846fb6ab 100644 --- a/roles/config/cluster/base/vars/main.yml +++ b/roles/config/cluster/base/vars/main.yml @@ -21,7 +21,9 @@ custom_config_templates: # Custom configurations for databases - template: configs/databases.j2 - template: configs/databases-7.1.0.j2 - condition: "{{ cloudera_runtime_version is version('7.1.0','>=') }}" + condition: "{{ cloudera_runtime_version is version('7.1.0','>=') and cloudera_runtime_version is version('7.1.9','<') }}"" + - template: configs/databases-7.1.9.j2 + condition: "{{ cloudera_runtime_version is version('7.1.9','>=') }}" # Custom configurations for Infra Solr - template: configs/infra-solr.j2 condition: "{{ 'INFRA_SOLR' in cluster.services }}" diff --git a/roles/deployment/definition/defaults/main.yml b/roles/deployment/definition/defaults/main.yml index 57715363..e82840ae 100644 --- a/roles/deployment/definition/defaults/main.yml +++ b/roles/deployment/definition/defaults/main.yml @@ -121,6 +121,16 @@ database_defaults: user: queryprocessor password: "{{ database_default_password }}" +#New in 7.1.9, postgresql only until CHF2 + QUEUE_MANAGER: + host: "{{ database_host }}" + port: "{{ database_type | cloudera.cluster.default_database_port }}" + type: postgresql + name: queuemanager + user: queuemanager + password: "{{ database_default_password }}" + + databases_cm_svcs: ACTIVITYMONITOR: host: "{{ database_host }}" From 3667a8923381be402da96495ad9f7cb49b3040d4 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Thu, 2 Nov 2023 17:19:48 -0400 Subject: [PATCH 4/6] Update collection version to 4.1.0 Signed-off-by: Webster Mudge --- galaxy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/galaxy.yml b/galaxy.yml index d96637a9..13289f30 100644 --- a/galaxy.yml +++ b/galaxy.yml @@ -16,7 +16,7 @@ namespace: cloudera name: cluster -version: 4.0.1 +version: 4.1.0 readme: README.md authors: [] From 96295f296d035727106c6c8d54e027572102bec9 Mon Sep 17 00:00:00 2001 From: Jim Enright Date: Tue, 7 Nov 2023 17:14:06 +0000 Subject: [PATCH 5/6] Remove extra quote from databases-7.1.0 config condition (#159) Signed-off-by: Jim Enright --- roles/config/cluster/base/vars/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/config/cluster/base/vars/main.yml b/roles/config/cluster/base/vars/main.yml index 846fb6ab..02cf6975 100644 --- a/roles/config/cluster/base/vars/main.yml +++ b/roles/config/cluster/base/vars/main.yml @@ -21,7 +21,7 @@ custom_config_templates: # Custom configurations for databases - template: configs/databases.j2 - template: configs/databases-7.1.0.j2 - condition: "{{ cloudera_runtime_version is version('7.1.0','>=') and cloudera_runtime_version is version('7.1.9','<') }}"" + condition: "{{ cloudera_runtime_version is version('7.1.0','>=') and cloudera_runtime_version is version('7.1.9','<') }}" - template: configs/databases-7.1.9.j2 condition: "{{ cloudera_runtime_version is version('7.1.9','>=') }}" # Custom configurations for Infra Solr From d25a610122287315e78cd33b988730e3a1567381 Mon Sep 17 00:00:00 2001 From: Webster Mudge Date: Tue, 7 Nov 2023 12:16:33 -0500 Subject: [PATCH 6/6] Increment collection version to 4.1.1 Signed-off-by: Webster Mudge --- galaxy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/galaxy.yml b/galaxy.yml index 13289f30..6a1a49cb 100644 --- a/galaxy.yml +++ b/galaxy.yml @@ -16,7 +16,7 @@ namespace: cloudera name: cluster -version: 4.1.0 +version: 4.1.1 readme: README.md authors: []