upgrade citus to v11

Vonng · Jun 19, 2022 · 87036e8 · 87036e8
1 parent a9616e5
commit 87036e8
Show file tree

Hide file tree

Showing 12 changed files with 93 additions and 73 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -8,11 +8,14 @@ Please upgrade postgres to 14.4 ASAP.
 
 ### Software Upgrade
 
-* upgrade postgres to 14.4
-* Upgrade haproxy to 2.6.0
-* Upgrade grafana to 9.0.0
+* upgrade postgres to 14.4 (important bug fix)
+* upgrade citus to 11.0-2 (with enterprise features)
+* upgrade timescaledb to 2.7 (more continuous aggregates) 
+* Upgrade patroni to 2.1.4 (new sync health-check)
+* Upgrade haproxy to 2.6.0 (cli, reload, ssl,...)
+* Upgrade grafana to 9.0.0 (new ui)
 * Upgrade prometheus 2.36.0
-* Upgrade patroni to 2.1.4
+
 
 ### Bug fix:
 
@@ -21,10 +24,12 @@ Please upgrade postgres to 14.4 ASAP.
 * remove i686 packages when using repotrack under el7
 * Fix redis service systemctl enabled issue
 * Fix patroni systemctl service enabled=no by default issue
+* stop vip-manager when purging existing postgres
 
 ### API Changes
 
 * Mark `grafana_database` and `grafana_pgurl` as obsolete
+* Add some new etcd & pgsql alias (optional)
 
 ### New Apps
 

diff --git a/files/conf/pigsty-auto.yml b/files/conf/pigsty-auto.yml
@@ -197,8 +197,11 @@ all:
       - name: timescaledb
         description: TimescaleDB for RHEL/CentOS $releasever - $basearch
         gpgcheck: no
-        baseurl:
-          - https://packagecloud.io/timescale/timescaledb/el/7/$basearch
+        baseurl: https://packagecloud.io/timescale/timescaledb/el/$releasever/$basearch
+      - name: citus
+        description: Citus Community
+        gpgcheck: no
+        baseurl: https://repos.citusdata.com/community/el/$releasever/$basearch
       - name: centos-sclo
         description: CentOS-$releasever - SCLo
         gpgcheck: no
@@ -244,7 +247,7 @@ all:
       - python3 python3-psycopg2 python36-requests python3-etcd python3-consul python36-urllib3 python36-idna python36-pyOpenSSL python36-cryptography
       - patroni patroni-consul patroni-etcd pgbouncer pg_cli pgbadger pg_activity tail_n_mail                           # -- pgsql common - #
       - pgcenter boxinfo check_postgres emaj pgbconsole pg_bloat_check pgquarrel barman barman-cli pgloader pgFormatter pitrery pspg pgxnclient PyGreSQL
-      - postgresql14* postgis32_14* citus_14* pglogical_14* timescaledb-2-postgresql-14 pg_repack_14 wal2json_14        # -- pg14 packages -#
+      - postgresql14* postgis32_14* pglogical_14* citus110_14 timescaledb-2-postgresql-14 pg_repack_14 wal2json_14        # -- pg14 packages -#
       - pg_qualstats_14 pg_stat_kcache_14 pg_stat_monitor_14 pg_top_14 pg_track_settings_14 pg_wait_sampling_14 pg_probackup-std-14
       - pg_statement_rollback_14 system_stats_14 plproxy_14 plsh_14 pldebugger_14 plpgsql_check_14 pgmemcache_14 # plr_14
       - mysql_fdw_14 ogr_fdw_14 tds_fdw_14 sqlite_fdw_14 firebird_fdw_14 hdfs_fdw_14 mongo_fdw_14 osm_fdw_14 pgbouncer_fdw_14
@@ -513,16 +516,14 @@ all:
     pg_bin_dir: /usr/pgsql/bin        # postgres binary dir, default is /usr/pgsql/bin, which use /usr/pgsql -> /usr/pgsql-{ver}
     pg_packages:                      # postgresql related packages. `${pg_version} will be replaced by `pg_version`
       - postgresql${pg_version}*      # postgresql kernel packages
-      - postgis32_${pg_version}*      # postgis
-      - citus_${pg_version}*          # citus
-      - timescaledb-2-postgresql-${pg_version}    # timescaledb
+      - postgis32_${pg_version}* citus110_${pg_version} timescaledb-2-postgresql-${pg_version}
       - pgbouncer pg_exporter pgbadger pg_activity node_exporter consul haproxy vip-manager
       - patroni patroni-consul patroni-etcd python3 python3-psycopg2 python36-requests python3-etcd
       - python3-consul python36-urllib3 python36-idna python36-pyOpenSSL python36-cryptography
     pg_extensions:                   # postgresql extensions, `${pg_version} will be replaced by actual `pg_version`
       - pg_repack_${pg_version} pg_qualstats_${pg_version} pg_stat_kcache_${pg_version} pg_stat_monitor_${pg_version} wal2json_${pg_version}
     # - ogr_fdw${pg_version} mysql_fdw_${pg_version} redis_fdw_${pg_version} mongo_fdw${pg_version} hdfs_fdw_${pg_version}
-    # - count_distinct${version}  ddlx_${version}  geoip${version}  orafce${version}
+    # - count_distinct${version}  ddlx_${version}  geoip${version}  orafce${version}  pg_ivm_${version}  multicorm2_${version}
     # - hypopg_${version}  ip4r${version}  jsquery_${version}  logerrors_${version}  periods_${version}  pg_auto_failover_${version}  pg_catcheck${version}
     # - pg_fkpart${version}  pg_jobmon${version}  pg_partman${version}  pg_prioritize_${version}  pg_track_settings${version}  pgaudit15_${version}
     # - pgcryptokey${version}  pgexportdoc${version}  pgimportdoc${version}  pgmemcache-${version}  pgmp${version}  pgq-${version}  pgquarrel pgrouting_${version}

diff --git a/files/conf/pigsty-citus.yml b/files/conf/pigsty-citus.yml
@@ -65,7 +65,7 @@ all:
         pg_cluster: pg-meta
         vip_address: 10.10.10.2
         pg_users: [ { name: citus , password: citus , pgbouncer: true , roles: [ dbrole_admin ] } ]
-        pg_databases: [ { name: meta , owner: citus , extensions: [ { name: citus } ] } ]
+        pg_databases: [ { name: meta , owner: citus , extensions: [ { name: citus }, { name: postgis, schema: public } ] } ]
 
     #----------------------------------#
     # cluster: citus data nodes
@@ -77,7 +77,7 @@ all:
         pg_cluster: pg-node1
         vip_address: 10.10.10.3
         pg_users: [ { name: citus , password: citus , pgbouncer: true , roles: [ dbrole_admin ] } ]
-        pg_databases: [ { name: meta , owner: citus , extensions: [ { name: citus } ] } ]
+        pg_databases: [ { name: meta , owner: citus , extensions: [ { name: citus }, { name: postgis, schema: public } ] } ]
 
     pg-node2:
       hosts:
@@ -86,7 +86,7 @@ all:
         pg_cluster: pg-node2
         vip_address: 10.10.10.4
         pg_users: [ { name: citus , password: citus , pgbouncer: true , roles: [ dbrole_admin ] } ]
-        pg_databases: [ { name: meta , owner: citus , extensions: [ { name: citus } ] } ]
+        pg_databases: [ { name: meta , owner: citus , extensions: [ { name: citus }, { name: postgis, schema: public } ] } ]
 
     pg-node3:
       hosts:
@@ -95,7 +95,7 @@ all:
         pg_cluster: pg-node3
         vip_address: 10.10.10.5
         pg_users: [ { name: citus , password: citus , pgbouncer: true , roles: [ dbrole_admin ] } ]
-        pg_databases: [ { name: meta , owner: citus , extensions: [ { name: citus } ] } ]
+        pg_databases: [ { name: meta , owner: citus , extensions: [ { name: citus }, { name: postgis, schema: public } ] } ]
 
 
   ####################################################################
@@ -217,8 +217,11 @@ all:
       - name: timescaledb
         description: TimescaleDB for RHEL/CentOS $releasever - $basearch
         gpgcheck: no
-        baseurl:
-          - https://packagecloud.io/timescale/timescaledb/el/7/$basearch
+        baseurl: https://packagecloud.io/timescale/timescaledb/el/$releasever/$basearch
+      - name: citus
+        description: Citus Community
+        gpgcheck: no
+        baseurl: https://repos.citusdata.com/community/el/$releasever/$basearch
       - name: centos-sclo
         description: CentOS-$releasever - SCLo
         gpgcheck: no
@@ -264,7 +267,7 @@ all:
       - python3 python3-psycopg2 python36-requests python3-etcd python3-consul python36-urllib3 python36-idna python36-pyOpenSSL python36-cryptography
       - patroni patroni-consul patroni-etcd pgbouncer pg_cli pgbadger pg_activity tail_n_mail                           # -- pgsql common - #
       - pgcenter boxinfo check_postgres emaj pgbconsole pg_bloat_check pgquarrel barman barman-cli pgloader pgFormatter pitrery pspg pgxnclient PyGreSQL
-      - postgresql14* postgis32_14* citus_14* pglogical_14* timescaledb-2-postgresql-14 pg_repack_14 wal2json_14        # -- pg14 packages -#
+      - postgresql14* postgis32_14* pglogical_14* citus110_14 timescaledb-2-postgresql-14 pg_repack_14 wal2json_14        # -- pg14 packages -#
       - pg_qualstats_14 pg_stat_kcache_14 pg_stat_monitor_14 pg_top_14 pg_track_settings_14 pg_wait_sampling_14 pg_probackup-std-14
       - pg_statement_rollback_14 system_stats_14 plproxy_14 plsh_14 pldebugger_14 plpgsql_check_14 pgmemcache_14 # plr_14
       - mysql_fdw_14 ogr_fdw_14 tds_fdw_14 sqlite_fdw_14 firebird_fdw_14 hdfs_fdw_14 mongo_fdw_14 osm_fdw_14 pgbouncer_fdw_14
@@ -389,8 +392,9 @@ all:
     # NODE_DNS
     #-----------------------------------------------------------------
     node_etc_hosts_default:           # static dns records in /etc/hosts
-      - 10.10.10.10 meta pigsty c.pigsty g.pigsty l.pigsty p.pigsty a.pigsty cli.pigsty lab.pigsty api.pigsty
-    node_etc_hosts:
+      - 10.10.10.10 meta pigsty p.pigsty g.pigsty a.pigsty c.pigsty l.pigsty
+      - 10.10.10.10 api.pigsty adm.pigsty cli.pigsty ddl.pigsty lab.pigsty git.pigsty sss.pigsty
+    node_etc_hosts:                   # extra static dns records in /etc/hosts
       - 10.10.10.10 meta
       - 10.10.10.11 node-1
       - 10.10.10.12 node-2
@@ -556,16 +560,14 @@ all:
     pg_bin_dir: /usr/pgsql/bin        # postgres binary dir, default is /usr/pgsql/bin, which use /usr/pgsql -> /usr/pgsql-{ver}
     pg_packages:                      # postgresql related packages. `${pg_version} will be replaced by `pg_version`
       - postgresql${pg_version}*      # postgresql kernel packages
-      - postgis32_${pg_version}*      # postgis
-      - citus_${pg_version}*          # citus
-      - timescaledb-2-postgresql-${pg_version}    # timescaledb
+      - postgis32_${pg_version}* citus110_${pg_version} timescaledb-2-postgresql-${pg_version}
       - pgbouncer pg_exporter pgbadger pg_activity node_exporter consul haproxy vip-manager
       - patroni patroni-consul patroni-etcd python3 python3-psycopg2 python36-requests python3-etcd
       - python3-consul python36-urllib3 python36-idna python36-pyOpenSSL python36-cryptography
     pg_extensions:                   # postgresql extensions, `${pg_version} will be replaced by actual `pg_version`
       - pg_repack_${pg_version} pg_qualstats_${pg_version} pg_stat_kcache_${pg_version} pg_stat_monitor_${pg_version} wal2json_${pg_version}
     # - ogr_fdw${pg_version} mysql_fdw_${pg_version} redis_fdw_${pg_version} mongo_fdw${pg_version} hdfs_fdw_${pg_version}
-    # - count_distinct${version}  ddlx_${version}  geoip${version}  orafce${version}
+    # - count_distinct${version}  ddlx_${version}  geoip${version}  orafce${version}  pg_ivm_${version}  multicorm2_${version}
     # - hypopg_${version}  ip4r${version}  jsquery_${version}  logerrors_${version}  periods_${version}  pg_auto_failover_${version}  pg_catcheck${version}
     # - pg_fkpart${version}  pg_jobmon${version}  pg_partman${version}  pg_prioritize_${version}  pg_track_settings${version}  pgaudit15_${version}
     # - pgcryptokey${version}  pgexportdoc${version}  pgimportdoc${version}  pgmemcache-${version}  pgmp${version}  pgq-${version}  pgquarrel pgrouting_${version}

diff --git a/files/conf/pigsty-dcs3.yml b/files/conf/pigsty-dcs3.yml
@@ -224,8 +224,11 @@ all:
       - name: timescaledb
         description: TimescaleDB for RHEL/CentOS $releasever - $basearch
         gpgcheck: no
-        baseurl:
-          - https://packagecloud.io/timescale/timescaledb/el/7/$basearch
+        baseurl: https://packagecloud.io/timescale/timescaledb/el/$releasever/$basearch
+      - name: citus
+        description: Citus Community
+        gpgcheck: no
+        baseurl: https://repos.citusdata.com/community/el/$releasever/$basearch
       - name: centos-sclo
         description: CentOS-$releasever - SCLo
         gpgcheck: no
@@ -271,7 +274,7 @@ all:
       - python3 python3-psycopg2 python36-requests python3-etcd python3-consul python36-urllib3 python36-idna python36-pyOpenSSL python36-cryptography
       - patroni patroni-consul patroni-etcd pgbouncer pg_cli pgbadger pg_activity tail_n_mail                           # -- pgsql common - #
       - pgcenter boxinfo check_postgres emaj pgbconsole pg_bloat_check pgquarrel barman barman-cli pgloader pgFormatter pitrery pspg pgxnclient PyGreSQL
-      - postgresql14* postgis32_14* citus_14* pglogical_14* timescaledb-2-postgresql-14 pg_repack_14 wal2json_14        # -- pg14 packages -#
+      - postgresql14* postgis32_14* pglogical_14* citus110_14 timescaledb-2-postgresql-14 pg_repack_14 wal2json_14        # -- pg14 packages -#
       - pg_qualstats_14 pg_stat_kcache_14 pg_stat_monitor_14 pg_top_14 pg_track_settings_14 pg_wait_sampling_14 pg_probackup-std-14
       - pg_statement_rollback_14 system_stats_14 plproxy_14 plsh_14 pldebugger_14 plpgsql_check_14 pgmemcache_14 # plr_14
       - mysql_fdw_14 ogr_fdw_14 tds_fdw_14 sqlite_fdw_14 firebird_fdw_14 hdfs_fdw_14 mongo_fdw_14 osm_fdw_14 pgbouncer_fdw_14
@@ -333,9 +336,6 @@ all:
     grafana_endpoint: http://10.10.10.10:3000     # grafana endpoint url
     grafana_admin_username: admin     # default grafana admin username
     grafana_admin_password: pigsty    # default grafana admin password
-    grafana_database: sqlite3         # default grafana database type: sqlite3|postgres
-    grafana_pgurl: postgres://dbuser_grafana:DBUser.Grafana@meta:5436/grafana
-
     grafana_plugin_method: install    # none|install|always, none will skip plugin install
     grafana_plugin_cache: /www/pigsty/plugins.tgz # path to grafana plugins cache tarball
     grafana_plugin_list:              # plugins that will be downloaded via grafana-cli
@@ -548,16 +548,14 @@ all:
     pg_bin_dir: /usr/pgsql/bin        # postgres binary dir, default is /usr/pgsql/bin, which use /usr/pgsql -> /usr/pgsql-{ver}
     pg_packages:                      # postgresql related packages. `${pg_version} will be replaced by `pg_version`
       - postgresql${pg_version}*      # postgresql kernel packages
-      - postgis32_${pg_version}*      # postgis
-      - citus_${pg_version}*          # citus
-      - timescaledb-2-postgresql-${pg_version}    # timescaledb
+      - postgis32_${pg_version}* citus110_${pg_version} timescaledb-2-postgresql-${pg_version}
       - pgbouncer pg_exporter pgbadger pg_activity node_exporter consul haproxy vip-manager
       - patroni patroni-consul patroni-etcd python3 python3-psycopg2 python36-requests python3-etcd
       - python3-consul python36-urllib3 python36-idna python36-pyOpenSSL python36-cryptography
     pg_extensions:                   # postgresql extensions, `${pg_version} will be replaced by actual `pg_version`
       - pg_repack_${pg_version} pg_qualstats_${pg_version} pg_stat_kcache_${pg_version} pg_stat_monitor_${pg_version} wal2json_${pg_version}
     # - ogr_fdw${pg_version} mysql_fdw_${pg_version} redis_fdw_${pg_version} mongo_fdw${pg_version} hdfs_fdw_${pg_version}
-    # - count_distinct${version}  ddlx_${version}  geoip${version}  orafce${version}
+    # - count_distinct${version}  ddlx_${version}  geoip${version}  orafce${version}  pg_ivm_${version}  multicorm2_${version}
     # - hypopg_${version}  ip4r${version}  jsquery_${version}  logerrors_${version}  periods_${version}  pg_auto_failover_${version}  pg_catcheck${version}
     # - pg_fkpart${version}  pg_jobmon${version}  pg_partman${version}  pg_prioritize_${version}  pg_track_settings${version}  pgaudit15_${version}
     # - pgcryptokey${version}  pgexportdoc${version}  pgimportdoc${version}  pgmemcache-${version}  pgmp${version}  pgq-${version}  pgquarrel pgrouting_${version}

diff --git a/files/conf/pigsty-demo.yml b/files/conf/pigsty-demo.yml
@@ -335,8 +335,11 @@ all:
       - name: timescaledb
         description: TimescaleDB for RHEL/CentOS $releasever - $basearch
         gpgcheck: no
-        baseurl:
-          - https://packagecloud.io/timescale/timescaledb/el/7/$basearch
+        baseurl: https://packagecloud.io/timescale/timescaledb/el/$releasever/$basearch
+      - name: citus
+        description: Citus Community
+        gpgcheck: no
+        baseurl: https://repos.citusdata.com/community/el/$releasever/$basearch
       - name: centos-sclo
         description: CentOS-$releasever - SCLo
         gpgcheck: no
@@ -382,7 +385,7 @@ all:
       - python3 python3-psycopg2 python36-requests python3-etcd python3-consul python36-urllib3 python36-idna python36-pyOpenSSL python36-cryptography
       - patroni patroni-consul patroni-etcd pgbouncer pg_cli pgbadger pg_activity tail_n_mail                           # -- pgsql common - #
       - pgcenter boxinfo check_postgres emaj pgbconsole pg_bloat_check pgquarrel barman barman-cli pgloader pgFormatter pitrery pspg pgxnclient PyGreSQL
-      - postgresql14* postgis32_14* citus_14* pglogical_14* timescaledb-2-postgresql-14 pg_repack_14 wal2json_14        # -- pg14 packages -#
+      - postgresql14* postgis32_14* pglogical_14* citus110_14 timescaledb-2-postgresql-14 pg_repack_14 wal2json_14        # -- pg14 packages -#
       - pg_qualstats_14 pg_stat_kcache_14 pg_stat_monitor_14 pg_top_14 pg_track_settings_14 pg_wait_sampling_14 pg_probackup-std-14
       - pg_statement_rollback_14 system_stats_14 plproxy_14 plsh_14 pldebugger_14 plpgsql_check_14 pgmemcache_14 # plr_14
       - mysql_fdw_14 ogr_fdw_14 tds_fdw_14 sqlite_fdw_14 firebird_fdw_14 hdfs_fdw_14 mongo_fdw_14 osm_fdw_14 pgbouncer_fdw_14
@@ -656,16 +659,14 @@ all:
     pg_bin_dir: /usr/pgsql/bin        # postgres binary dir, default is /usr/pgsql/bin, which use /usr/pgsql -> /usr/pgsql-{ver}
     pg_packages:                      # postgresql related packages. `${pg_version} will be replaced by `pg_version`
       - postgresql${pg_version}*      # postgresql kernel packages
-      - postgis32_${pg_version}*      # postgis
-      - citus_${pg_version}*          # citus
-      - timescaledb-2-postgresql-${pg_version}    # timescaledb
+      - postgis32_${pg_version}* citus110_${pg_version} timescaledb-2-postgresql-${pg_version}
       - pgbouncer pg_exporter pgbadger pg_activity node_exporter consul haproxy vip-manager
       - patroni patroni-consul patroni-etcd python3 python3-psycopg2 python36-requests python3-etcd
       - python3-consul python36-urllib3 python36-idna python36-pyOpenSSL python36-cryptography
     pg_extensions:                   # postgresql extensions, `${pg_version} will be replaced by actual `pg_version`
       - pg_repack_${pg_version} pg_qualstats_${pg_version} pg_stat_kcache_${pg_version} pg_stat_monitor_${pg_version} wal2json_${pg_version}
     # - ogr_fdw${pg_version} mysql_fdw_${pg_version} redis_fdw_${pg_version} mongo_fdw${pg_version} hdfs_fdw_${pg_version}
-    # - count_distinct${version}  ddlx_${version}  geoip${version}  orafce${version}
+    # - count_distinct${version}  ddlx_${version}  geoip${version}  orafce${version}  pg_ivm_${version}  multicorm2_${version}
     # - hypopg_${version}  ip4r${version}  jsquery_${version}  logerrors_${version}  periods_${version}  pg_auto_failover_${version}  pg_catcheck${version}
     # - pg_fkpart${version}  pg_jobmon${version}  pg_partman${version}  pg_prioritize_${version}  pg_track_settings${version}  pgaudit15_${version}
     # - pgcryptokey${version}  pgexportdoc${version}  pgimportdoc${version}  pgmemcache-${version}  pgmp${version}  pgq-${version}  pgquarrel pgrouting_${version}
@@ -780,7 +781,7 @@ all:
           - host    all     +dbrole_offline               172.16.0.0/12     md5
           - host    all     +dbrole_offline               192.168.0.0/16    md5
 
-    pgbouncer_hba_rules:             # pgbouncer host-based authentication rules
+    pgbouncer_hba_rules:              # pgbouncer host-based authentication rules
       - title: local password access
         role: common
         rules: