From 2b12e87535df2d45c84b9308bc242b0e913f31af Mon Sep 17 00:00:00 2001 From: to-bar <46519524+to-bar@users.noreply.github.com> Date: Tue, 7 Dec 2021 14:48:35 +0100 Subject: [PATCH] Remove PgBouncer standalone installation (#2770) * Remove standalone PgBouncer * Update changelog --- .../roles/filebeat/templates/filebeat.yml.j2 | 1 - .../roles/postgresql/defaults/main.yml | 17 --- .../postgresql/molecule/add-repos-redhat.sh | 1 - .../roles/postgresql/molecule/vars-repmgr.yml | 5 - .../postgresql/molecule/verify-common.yml | 19 --- .../tasks/extensions/pgbouncer/extension.yml | 60 --------- .../tasks/extensions/pgbouncer/packages.yml | 11 -- .../upgrade/extensions/pgbouncer/packages.yml | 11 -- .../upgrade/nodes/common/prepare-upgrade.yml | 10 -- .../tasks/upgrade/nodes/common/set-facts.yml | 1 - .../postgresql/tasks/upgrade/run-upgrade.yml | 1 - .../templates/logrotate-pgbouncer.conf.j2 | 18 --- .../roles/preflight/defaults/main.yml | 1 - .../centos-7/add-repositories.multiarch.sh | 1 - .../centos-7/requirements.aarch64.txt | 1 - .../centos-7/requirements.x86_64.txt | 2 - .../redhat-7/add-repositories.multiarch.sh | 1 - .../redhat-7/requirements.x86_64.txt | 2 - .../ubuntu-18.04/requirements.x86_64.txt | 1 - docs/changelogs/CHANGELOG-1.3.md | 6 + docs/home/ARM.md | 4 - docs/home/COMPONENTS.md | 1 - docs/home/RESOURCES.md | 2 +- docs/home/SECURITY.md | 1 - docs/home/howto/DATABASES.md | 27 +--- docs/home/howto/KUBERNETES.md | 2 +- docs/home/howto/MODULES.md | 3 - .../defaults/configuration/firewall.yml | 1 - .../defaults/configuration/postgresql.yml | 5 - .../validation/configuration/postgresql.yml | 12 -- tests/spec/spec/postgresql/postgresql_spec.rb | 127 +----------------- 31 files changed, 11 insertions(+), 344 deletions(-) delete mode 100644 ansible/playbooks/roles/postgresql/tasks/extensions/pgbouncer/extension.yml delete mode 100644 ansible/playbooks/roles/postgresql/tasks/extensions/pgbouncer/packages.yml delete mode 100644 ansible/playbooks/roles/postgresql/tasks/upgrade/extensions/pgbouncer/packages.yml delete mode 100644 ansible/playbooks/roles/postgresql/templates/logrotate-pgbouncer.conf.j2 diff --git a/ansible/playbooks/roles/filebeat/templates/filebeat.yml.j2 b/ansible/playbooks/roles/filebeat/templates/filebeat.yml.j2 index c2698d3d13..f59e8bdfdd 100644 --- a/ansible/playbooks/roles/filebeat/templates/filebeat.yml.j2 +++ b/ansible/playbooks/roles/filebeat/templates/filebeat.yml.j2 @@ -99,7 +99,6 @@ filebeat.inputs: - type: log enabled: true paths: - - {{ postgresql_defaults.pgbouncer.logfile[ansible_os_family] }}* - /var/log/postgresql/repmgr.log* exclude_files: [".gz$"] {% endif %} diff --git a/ansible/playbooks/roles/postgresql/defaults/main.yml b/ansible/playbooks/roles/postgresql/defaults/main.yml index 043095d44e..4339dc91ad 100644 --- a/ansible/playbooks/roles/postgresql/defaults/main.yml +++ b/ansible/playbooks/roles/postgresql/defaults/main.yml @@ -56,20 +56,3 @@ repmgr: version: Debian: "5.2.1" RedHat: "5.2.1" - -pgbouncer: - group: - Debian: postgres - RedHat: pgbouncer - logfile: - Debian: /var/log/postgresql/pgbouncer.log - RedHat: /var/log/pgbouncer/pgbouncer.log - pidfile: - Debian: /var/run/postgresql/pgbouncer.pid - RedHat: /var/run/pgbouncer/pgbouncer.pid - user: - Debian: postgres - RedHat: pgbouncer - version: - Debian: "1.16.0" - RedHat: "1.16.0" diff --git a/ansible/playbooks/roles/postgresql/molecule/add-repos-redhat.sh b/ansible/playbooks/roles/postgresql/molecule/add-repos-redhat.sh index 38c32d51ba..8896662b73 100644 --- a/ansible/playbooks/roles/postgresql/molecule/add-repos-redhat.sh +++ b/ansible/playbooks/roles/postgresql/molecule/add-repos-redhat.sh @@ -148,6 +148,5 @@ EOF ) add_repo_as_file 'postgresql-13' "$POSTGRESQL_REPO_CONF" -add_repo_as_file 'postgresql-common' "$POSTGRESQL_COMMON_REPO_CONF" # for pgbouncer add_repo_from_script 'https://dl.2ndquadrant.com/default/release/get/13/rpm' # for repmgr disable_repo '2ndquadrant-dl-default-release-pg13-debug' # script adds 2 repositories, only 1 is required diff --git a/ansible/playbooks/roles/postgresql/molecule/vars-repmgr.yml b/ansible/playbooks/roles/postgresql/molecule/vars-repmgr.yml index 603fbfbcb5..35ce97aeab 100644 --- a/ansible/playbooks/roles/postgresql/molecule/vars-repmgr.yml +++ b/ansible/playbooks/roles/postgresql/molecule/vars-repmgr.yml @@ -82,8 +82,6 @@ specification: pgaudit.log_relation: 'on # separate log entry for each relation' pgaudit.log_statement_once: off pgaudit.log_parameter: on - pgbouncer: - enabled: true replication: replication_user_name: epi_repmgr replication_user_password: PASSWORD_TO_CHANGE @@ -93,9 +91,6 @@ specification: shared_preload_libraries: - repmgr logrotate: - pgbouncer: - period: weekly - rotations: 5 postgresql: |- /var/log/postgresql/postgresql*.log { maxsize 10M diff --git a/ansible/playbooks/roles/postgresql/molecule/verify-common.yml b/ansible/playbooks/roles/postgresql/molecule/verify-common.yml index ecd7de39c0..8aaa732a9e 100644 --- a/ansible/playbooks/roles/postgresql/molecule/verify-common.yml +++ b/ansible/playbooks/roles/postgresql/molecule/verify-common.yml @@ -13,15 +13,6 @@ - "{{ _services[ansible_os_family] in ansible_facts.services }}" - "{{ ansible_facts.services[_services[ansible_os_family]].state == 'running' }}" -# PgBouncer is installed only on one of nodes -- name: Verify that pgbouncer service exists and is running - delegate_to: groups.postgresql[0] - run_once: true - assert: - that: - - "{{ 'pgbouncer.service' in ansible_facts.services }}" - - "{{ ansible_facts.services['pgbouncer.service'].state == 'running' }}" - # required for 'listen_ports_facts' module - name: Ensure net-tools package is installed package: @@ -38,13 +29,3 @@ | selectattr('port', 'equalto', 5432) | selectattr('address', 'equalto', '0.0.0.0') | length == 1 }}" - -- name: Verify PgBouncer port - delegate_to: groups.postgresql[0] - run_once: true - assert: - that: - - "{{ ansible_facts.tcp_listen - | selectattr('port', 'equalto', 6432) - | selectattr('address', 'equalto', '127.0.0.1') - | length == 1 }}" diff --git a/ansible/playbooks/roles/postgresql/tasks/extensions/pgbouncer/extension.yml b/ansible/playbooks/roles/postgresql/tasks/extensions/pgbouncer/extension.yml deleted file mode 100644 index 045861430e..0000000000 --- a/ansible/playbooks/roles/postgresql/tasks/extensions/pgbouncer/extension.yml +++ /dev/null @@ -1,60 +0,0 @@ ---- -- name: Extensions | configure PgBouncer - when: groups.postgresql[0] == inventory_hostname - block: - # Avoid wrong process identification as interpreted script is used - # See https://chris-lamb.co.uk/posts/start-stop-daemon-exec-vs-startas - - name: Extensions | PgBouncer | Debian | Change daemon start stop option from 'exec' to 'startas' - when: ansible_os_family == 'Debian' - lineinfile: - path: /etc/init.d/pgbouncer - regexp: "^SSD=" - line: 'SSD="start-stop-daemon --pidfile $PIDFILE --startas $DAEMON --quiet"' - - - name: Extensions | PgBouncer | Force systemd to reread configs - systemd: - daemon_reload: true - - - name: Extensions | PgBouncer | Ensure that systemd service is started - systemd: - name: pgbouncer - state: started - - - name: Extensions | PgBouncer | Change pgbouncer configuration - lineinfile: - path: /etc/pgbouncer/pgbouncer.ini - regexp: "^postgres = host=" - line: postgres = host=127.0.0.1 port=5432 dbname=postgres - insertafter: '\[databases\]' - backup: true - register: db_connection_line - - - name: Extensions | PgBouncer | Change pgbouncer users configuration - lineinfile: - path: /etc/pgbouncer/userlist.txt - line: '"postgres" "*"' - create: true - mode: u=rw,g=,o= - owner: "{{ pgbouncer.user[ansible_os_family] }}" - group: "{{ pgbouncer.group[ansible_os_family] }}" - backup: true - register: db_user_line - - - name: Extensions | PgBouncer | Create logrotate configuration file - template: - src: logrotate-pgbouncer.conf.j2 - dest: /etc/logrotate.d/pgbouncer - owner: root - group: root - mode: u=rw,go=r - - - name: Extensions | PgBouncer | Restart systemd service - when: db_connection_line.changed or db_user_line.changed - systemd: - name: pgbouncer - state: restarted - - - name: Extensions | PgBouncer | Ensure that systemd service is enabled - systemd: - name: pgbouncer - enabled: true diff --git a/ansible/playbooks/roles/postgresql/tasks/extensions/pgbouncer/packages.yml b/ansible/playbooks/roles/postgresql/tasks/extensions/pgbouncer/packages.yml deleted file mode 100644 index 25b884b46d..0000000000 --- a/ansible/playbooks/roles/postgresql/tasks/extensions/pgbouncer/packages.yml +++ /dev/null @@ -1,11 +0,0 @@ ---- -- name: Extensions | PgBouncer | Install package(s) - package: - name: "{{ _packages[ansible_os_family] }}" - state: present - vars: - _packages: - Debian: "pgbouncer={{ pgbouncer.version.Debian + '-*' }}" - RedHat: "pgbouncer-{{ pgbouncer.version.RedHat }}" - module_defaults: - yum: { lock_timeout: "{{ yum_lock_timeout }}" } diff --git a/ansible/playbooks/roles/postgresql/tasks/upgrade/extensions/pgbouncer/packages.yml b/ansible/playbooks/roles/postgresql/tasks/upgrade/extensions/pgbouncer/packages.yml deleted file mode 100644 index 8aa084071b..0000000000 --- a/ansible/playbooks/roles/postgresql/tasks/upgrade/extensions/pgbouncer/packages.yml +++ /dev/null @@ -1,11 +0,0 @@ ---- -- name: Extensions | PgBouncer | Install package(s) - package: - name: "{{ _packages[ansible_os_family] }}" - state: present - vars: - _packages: - Debian: "pgbouncer={{ new_version.pgbouncer.version.Debian + '-*' }}" - RedHat: "pgbouncer-{{ new_version.pgbouncer.version.RedHat }}" - module_defaults: - yum: {lock_timeout: "{{ yum_lock_timeout }}"} diff --git a/ansible/playbooks/roles/postgresql/tasks/upgrade/nodes/common/prepare-upgrade.yml b/ansible/playbooks/roles/postgresql/tasks/upgrade/nodes/common/prepare-upgrade.yml index 219c287db1..8132e612d3 100644 --- a/ansible/playbooks/roles/postgresql/tasks/upgrade/nodes/common/prepare-upgrade.yml +++ b/ansible/playbooks/roles/postgresql/tasks/upgrade/nodes/common/prepare-upgrade.yml @@ -56,16 +56,6 @@ include_tasks: upgrade/extensions/pgaudit/packages.yml when: is_pgaudit_used -- name: Extensions | PgBouncer | Upgrade - when: is_pgbouncer_used - block: - - include_tasks: upgrade/extensions/pgbouncer/packages.yml - - - name: Extensions | PgBouncer | Ensure that systemd service is started - systemd: - name: pgbouncer - state: started - - name: Extensions | repmgr | Include package(s) installation tasks include_tasks: upgrade/extensions/replication/pg-new/packages.yml when: is_repmgr_used diff --git a/ansible/playbooks/roles/postgresql/tasks/upgrade/nodes/common/set-facts.yml b/ansible/playbooks/roles/postgresql/tasks/upgrade/nodes/common/set-facts.yml index 35f8ced385..9d5b8563dc 100644 --- a/ansible/playbooks/roles/postgresql/tasks/upgrade/nodes/common/set-facts.yml +++ b/ansible/playbooks/roles/postgresql/tasks/upgrade/nodes/common/set-facts.yml @@ -9,7 +9,6 @@ - name: Set facts on installed extensions set_fact: is_pgaudit_used: "{{ ansible_facts.packages.keys() | intersect(_packages.pgaudit) | count > 0 }}" - is_pgbouncer_used: "{{ ansible_facts.packages.pgbouncer is defined }}" # package name is the same for all versions is_repmgr_used: "{{ ansible_facts.packages.keys() | intersect(_packages.repmgr) | count > 0 }}" vars: _packages: diff --git a/ansible/playbooks/roles/postgresql/tasks/upgrade/run-upgrade.yml b/ansible/playbooks/roles/postgresql/tasks/upgrade/run-upgrade.yml index 299e709a7e..4fae797395 100644 --- a/ansible/playbooks/roles/postgresql/tasks/upgrade/run-upgrade.yml +++ b/ansible/playbooks/roles/postgresql/tasks/upgrade/run-upgrade.yml @@ -21,7 +21,6 @@ # - pg_primary_node # 2) extensions # - is_pgaudit_used -# - is_pgbouncer_used # - is_repmgr_used - include_tasks: upgrade/nodes/common/set-facts.yml diff --git a/ansible/playbooks/roles/postgresql/templates/logrotate-pgbouncer.conf.j2 b/ansible/playbooks/roles/postgresql/templates/logrotate-pgbouncer.conf.j2 deleted file mode 100644 index 78e761bbe5..0000000000 --- a/ansible/playbooks/roles/postgresql/templates/logrotate-pgbouncer.conf.j2 +++ /dev/null @@ -1,18 +0,0 @@ -# {{ ansible_managed }} - -{{ pgbouncer.logfile[ansible_os_family] }} { - missingok - copytruncate - # delaycompress is for Filebeat - delaycompress - compress - notifempty - sharedscripts - create 0640 {{ pgbouncer.user[ansible_os_family] }} {{ pgbouncer.group[ansible_os_family] }} - nodateext - {{ specification.logrotate.pgbouncer.period }} - rotate {{ specification.logrotate.pgbouncer.rotations }} - postrotate - /bin/kill -HUP `cat {{ pgbouncer.pidfile[ansible_os_family] }} 2>/dev/null` 2> /dev/null || true - endscript -} diff --git a/ansible/playbooks/roles/preflight/defaults/main.yml b/ansible/playbooks/roles/preflight/defaults/main.yml index d74f88fd03..65c9ca0112 100644 --- a/ansible/playbooks/roles/preflight/defaults/main.yml +++ b/ansible/playbooks/roles/preflight/defaults/main.yml @@ -102,4 +102,3 @@ unsupported_postgres_extensions: x86_64: [] aarch64: - replication - - pgbouncer diff --git a/ansible/playbooks/roles/repository/files/download-requirements/centos-7/add-repositories.multiarch.sh b/ansible/playbooks/roles/repository/files/download-requirements/centos-7/add-repositories.multiarch.sh index b073a42838..c59a6a3e74 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/centos-7/add-repositories.multiarch.sh +++ b/ansible/playbooks/roles/repository/files/download-requirements/centos-7/add-repositories.multiarch.sh @@ -114,5 +114,4 @@ add_repo_as_file 'elasticsearch-curator-5' "$ELASTICSEARCH_CURATOR_REPO_CONF" add_repo_as_file 'kubernetes' "$KUBERNETES_REPO_CONF" add_repo_as_file 'opendistroforelasticsearch' "$OPENDISTRO_REPO_CONF" add_repo_as_file 'postgresql-13' "$POSTGRESQL_REPO_CONF" -add_repo_as_file 'postgresql-common' "$POSTGRESQL_COMMON_REPO_CONF" # for pgbouncer add_repo_as_file 'rabbitmq' "$RABBITMQ_SERVER_REPO_CONF" diff --git a/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.aarch64.txt b/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.aarch64.txt index 3bef74f7b7..0e4dea9459 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.aarch64.txt +++ b/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.aarch64.txt @@ -23,7 +23,6 @@ https://github.com/google/go-containerregistry/releases/download/v0.4.1/go-conta [packages] audit # for docker-ce bash-completion -#c-ares # for pgbouncer ca-certificates cifs-utils conntrack-tools # for kubelet diff --git a/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.x86_64.txt b/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.x86_64.txt index 49bd853cb1..65ff13f669 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.x86_64.txt +++ b/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.x86_64.txt @@ -23,7 +23,6 @@ https://github.com/google/go-containerregistry/releases/download/v0.4.1/go-conta [packages] audit # for docker-ce bash-completion -c-ares # for pgbouncer ca-certificates cifs-utils conntrack-tools # for kubelet @@ -80,7 +79,6 @@ perl-Pod-Perldoc # for vim perl-Pod-Simple # for vim perl-Pod-Usage # for vim pgaudit15_13-1.5.0 -pgbouncer-1.16.0 policycoreutils-python # for container-selinux pyldb # for cifs-utils python-cffi # for python2-cryptography diff --git a/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/add-repositories.multiarch.sh b/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/add-repositories.multiarch.sh index b073a42838..c59a6a3e74 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/add-repositories.multiarch.sh +++ b/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/add-repositories.multiarch.sh @@ -114,5 +114,4 @@ add_repo_as_file 'elasticsearch-curator-5' "$ELASTICSEARCH_CURATOR_REPO_CONF" add_repo_as_file 'kubernetes' "$KUBERNETES_REPO_CONF" add_repo_as_file 'opendistroforelasticsearch' "$OPENDISTRO_REPO_CONF" add_repo_as_file 'postgresql-13' "$POSTGRESQL_REPO_CONF" -add_repo_as_file 'postgresql-common' "$POSTGRESQL_COMMON_REPO_CONF" # for pgbouncer add_repo_as_file 'rabbitmq' "$RABBITMQ_SERVER_REPO_CONF" diff --git a/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/requirements.x86_64.txt b/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/requirements.x86_64.txt index bb3f34812b..c31fda3232 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/requirements.x86_64.txt +++ b/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/requirements.x86_64.txt @@ -21,7 +21,6 @@ https://github.com/google/go-containerregistry/releases/download/v0.4.1/go-conta [packages] audit # for docker-ce bash-completion -c-ares # for pgbouncer ca-certificates cifs-utils conntrack-tools # for kubelet @@ -78,7 +77,6 @@ perl-Pod-Perldoc # for vim perl-Pod-Simple # for vim perl-Pod-Usage # for vim pgaudit15_13-1.5.0 -pgbouncer-1.16.0 policycoreutils-python # for container-selinux pyldb # for cifs-utils python-cffi # for python2-cryptography diff --git a/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-18.04/requirements.x86_64.txt b/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-18.04/requirements.x86_64.txt index 6956413288..08e7de9744 100644 --- a/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-18.04/requirements.x86_64.txt +++ b/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-18.04/requirements.x86_64.txt @@ -143,7 +143,6 @@ libsmbclient # postgres related packages # if version is not specified, it's not related to postgres version and the latest is used -pgbouncer 1.16.0 pgdg-keyring postgresql-13-pgaudit 1.5.0 postgresql-10-repmgr 5.2.1 diff --git a/docs/changelogs/CHANGELOG-1.3.md b/docs/changelogs/CHANGELOG-1.3.md index 4ed526b2a9..e998c80f8e 100644 --- a/docs/changelogs/CHANGELOG-1.3.md +++ b/docs/changelogs/CHANGELOG-1.3.md @@ -43,8 +43,14 @@ - [#2180](https://github.com/epiphany-platform/epiphany/issues/2180) - [documentation] Missing clear information about supported CNI plugins - [#2755](https://github.com/epiphany-platform/epiphany/issues/2755) - Upgrade Python dependencies to the latest +### Removed + +- [#2680](https://github.com/epiphany-platform/epiphany/issues/2680) - Remove PgBouncer standalone installation + ### Deprecated ### Breaking changes +- PgBouncer available only as Kubernetes service + ### Known issues diff --git a/docs/home/ARM.md b/docs/home/ARM.md index a00d5d4bd3..2510fdd533 100644 --- a/docs/home/ARM.md +++ b/docs/home/ARM.md @@ -200,10 +200,6 @@ specification: extensions: pgaudit: enabled: yes - pgbouncer: - enabled: no - replication: - enabled: no title: Postgresql --- kind: configuration/rabbitmq diff --git a/docs/home/COMPONENTS.md b/docs/home/COMPONENTS.md index 015283ad99..354bd9c587 100644 --- a/docs/home/COMPONENTS.md +++ b/docs/home/COMPONENTS.md @@ -36,7 +36,6 @@ Note that versions are default versions and can be changed in certain cases thro | PostgreSQL | 13 | https://www.postgresql.org/ | [PostgreSQL license](http://www.postgresql.org/about/licence/) | | HAProxy | 2.2.2 | https://www.haproxy.org/ | [GNU General Public License 2.0](https://www.gnu.org/licenses/old-licenses/gpl-2.0.html) | | PgAudit | 1.5.0 | https://github.com/pgaudit/pgaudit | [PostgreSQL license](http://www.postgresql.org/about/licence/) | -| PgBouncer | 1.16.0 | https://github.com/pgbouncer/pgbouncer | [ISC License](https://opensource.org/licenses/isc) | | repmgr | 5.2.1 | https://github.com/EnterpriseDB/repmgr | [GNU General Public License 3.0](https://github.com/EnterpriseDB/repmgr/blob/master/LICENSE) | | Pgpool | 4.2.4 | https://www.pgpool.net/ | [License](https://www.pgpool.net/mediawiki/index.php/pgpool-II_License) | | Alertmanager | 0.17.0 | https://github.com/prometheus/alertmanager | [Apache License 2.0](https://github.com/prometheus/alertmanager/blob/master/LICENSE) | diff --git a/docs/home/RESOURCES.md b/docs/home/RESOURCES.md index 1e7e5414af..03dac4c716 100644 --- a/docs/home/RESOURCES.md +++ b/docs/home/RESOURCES.md @@ -50,7 +50,7 @@ Here are some materials concerning Epiphany tooling and cluster components - bot 1. [HaProxy](http://www.haproxy.org/) 7. Databases 1. [PostgreSQL](https://www.postgresql.org/docs/) - - [Repmng](https://repmgr.org/) + - [repmgr](https://repmgr.org/) - [PGBouncer](https://www.pgbouncer.org/) - [PGPool](https://www.pgpool.net/mediawiki/index.php/Main_Page) - [PGAudit](https://www.pgaudit.org/) diff --git a/docs/home/SECURITY.md b/docs/home/SECURITY.md index bdde649ff1..85f85a8cd8 100644 --- a/docs/home/SECURITY.md +++ b/docs/home/SECURITY.md @@ -96,7 +96,6 @@ The list does not include ports that are bound to the loopback interface (localh 11. PostgreSQL: - 5432 - PostgreSQL server - - 6432 - PgBouncer 12. Kubernetes: diff --git a/docs/home/howto/DATABASES.md b/docs/home/howto/DATABASES.md index 7930043e92..f3ed8e9f5e 100644 --- a/docs/home/howto/DATABASES.md +++ b/docs/home/howto/DATABASES.md @@ -16,31 +16,8 @@ For this reason, MD5 password encryption is set up and this is not configurable ## How to set up PostgreSQL connection pooling -PostgreSQL connection pooling in Epiphany is served by PgBouncer application. It is available as Kubernetes `ClusterIP` or standalone package. -The [Kubernetes based installation](#how-to-set-up-pgbouncer-pgpool-and-postgresql-parameters) works together with PgPool so it supports PostgreSQL HA setup. -The standalone installation (described below) is deprecated and **will be removed** in the next release. - ---- -**NOTE** - -PgBouncer extension is not supported on ARM. - ---- - -PgBouncer is installed only on PostgreSQL primary node. This needs to be enabled in configuration yaml file: - -```yaml -kind: configuration/postgresql -specification: - extensions: - ... - pgbouncer: - enabled: yes - ... -``` - -PgBouncer listens on standard port 6432. Basic configuration is just template, with very limited access to database. -This is because of security reasons. [Configuration needs to be tailored according component documentation and stick to security rules and best practices](http://www.pgbouncer.org/). +PostgreSQL connection pooling in Epiphany is served by [PgBouncer K8s application](#how-to-set-up-pgbouncer-pgpool-and-postgresql-parameters). +It is available as `ClusterIP` service and works together with PgPool so it supports PostgreSQL HA setup. ## How to set up PostgreSQL HA replication with repmgr cluster diff --git a/docs/home/howto/KUBERNETES.md b/docs/home/howto/KUBERNETES.md index 015aeb0d18..82333c67a9 100644 --- a/docs/home/howto/KUBERNETES.md +++ b/docs/home/howto/KUBERNETES.md @@ -385,7 +385,7 @@ To set specific database host IP address for Keycloak you have to provide additi Note: If `database address` is not specified, epicli assumes that database instance doesn't exist and will create it. -By default, if `database address` is not specified and if Postgres is HA mode, Keycloak uses PGBouncer ClusterIP service +By default, if `database address` is not specified and if Postgres is HA mode, Keycloak uses PGBouncer `ClusterIP` service name as database address. If Postgres is in standalone mode, and `database address` is not specified, then it uses first Postgres host address from `inventory`. diff --git a/docs/home/howto/MODULES.md b/docs/home/howto/MODULES.md index f68b39971e..dab15dcb22 100644 --- a/docs/home/howto/MODULES.md +++ b/docs/home/howto/MODULES.md @@ -231,10 +231,7 @@ AWS: pgaudit.log_relation: 'on # separate log entry for each relation' pgaudit.log_statement_once: 'off' pgaudit.log_parameter: 'on' - pgbouncer: - enabled: false replication: - enabled: false replication_user_name: epi_repmgr replication_user_password: PASSWORD_TO_CHANGE privileged_user_name: epi_repmgr_admin diff --git a/schema/common/defaults/configuration/firewall.yml b/schema/common/defaults/configuration/firewall.yml index 2706f0a5f7..19957f663e 100644 --- a/schema/common/defaults/configuration/firewall.yml +++ b/schema/common/defaults/configuration/firewall.yml @@ -92,7 +92,6 @@ specification: enabled: true ports: - 5432/tcp - - 6432/tcp #PGBouncer prometheus: enabled: true ports: diff --git a/schema/common/defaults/configuration/postgresql.yml b/schema/common/defaults/configuration/postgresql.yml index e6d637261e..6cffb6280f 100644 --- a/schema/common/defaults/configuration/postgresql.yml +++ b/schema/common/defaults/configuration/postgresql.yml @@ -87,8 +87,6 @@ specification: pgaudit.log_relation: 'on # separate log entry for each relation' # default is 'off' pgaudit.log_statement_once: 'off' # same as default pgaudit.log_parameter: 'on' # default is 'off' - pgbouncer: - enabled: false replication: replication_user_name: epi_repmgr replication_user_password: PASSWORD_TO_CHANGE @@ -98,9 +96,6 @@ specification: shared_preload_libraries: - repmgr logrotate: - pgbouncer: - period: weekly - rotations: 5 # Configuration partly based on /etc/logrotate.d/postgresql-common provided by 'postgresql-common' package from Ubuntu repo. # PostgreSQL from Ubuntu repo: # By default 'logging_collector' is disabled, so 'log_directory' parameter is ignored. diff --git a/schema/common/validation/configuration/postgresql.yml b/schema/common/validation/configuration/postgresql.yml index 914753144d..0f3f0344c7 100644 --- a/schema/common/validation/configuration/postgresql.yml +++ b/schema/common/validation/configuration/postgresql.yml @@ -68,11 +68,6 @@ properties: type: string pgaudit.log_parameter: type: string - pgbouncer: - type: object - properties: - enabled: - type: boolean replication: type: object properties: @@ -93,12 +88,5 @@ properties: logrotate: type: object properties: - pgbouncer: - type: object - properties: - period: - type: string - rotations: - type: integer postgresql: type: string diff --git a/tests/spec/spec/postgresql/postgresql_spec.rb b/tests/spec/spec/postgresql/postgresql_spec.rb index 41f2c65185..45dda0a33f 100644 --- a/tests/spec/spec/postgresql/postgresql_spec.rb +++ b/tests/spec/spec/postgresql/postgresql_spec.rb @@ -5,7 +5,6 @@ postgresql_host = '127.0.0.1' postgresql_default_port = 5432 -pgbouncer_default_port = 6432 # Here we set the needed variables from ENV variables which are set in the Rakefile # where we order the postgres hosts: @@ -39,7 +38,6 @@ replication_user = config_docs[:postgresql]["specification"]["extensions"]["replication"]["replication_user_name"] replication_password = config_docs[:postgresql]["specification"]["extensions"]["replication"]["replication_user_password"] max_wal_senders = config_docs[:postgresql]["specification"]["config_file"]["parameter_groups"].detect {|i| i["name"] == 'REPLICATION'}["subgroups"].detect {|i| i["name"] == "Sending Server(s)"}["parameters"].detect {|i| i["name"] == "max_wal_senders"}["value"] -pgbouncer_enabled = config_docs[:postgresql]["specification"]["extensions"]["pgbouncer"]["enabled"] pgaudit_enabled = config_docs[:postgresql]["specification"]["extensions"]["pgaudit"]["enabled"] if upgradeRun? @@ -450,106 +448,9 @@ def queryForDropping end end -### Tests for PGBouncer - -if pgbouncer_enabled - - if primary_node_host.include? host_inventory['hostname'] - - describe 'Check if PGBouncer service is running' do - describe service('pgbouncer') do - it { should be_enabled } - it { should be_running } - end - end - - describe 'Create a test user' do - let(:disable_sudo) { false } - describe command("su - postgres -c \"psql -t -c \\\"CREATE USER #{pg_user} WITH PASSWORD '#{pg_pass}';\\\"\" 2>&1") do - its(:stdout) { should match /^CREATE ROLE$/ } - its(:exit_status) { should eq 0 } - end - end - - describe 'Add user to userlist.txt' do - let(:disable_sudo) { false } - describe command("echo \\\"#{pg_user}\\\" \\\"#{pg_pass}\\\" >> /etc/pgbouncer/userlist.txt && systemctl restart pgbouncer") do - its(:exit_status) { should eq 0 } - end - end - - describe 'Grant privileges on schema to user' do - let(:disable_sudo) { false } - describe command("su - postgres -c \"psql -t -c 'GRANT ALL ON SCHEMA serverspec_test to #{pg_user}; GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA serverspec_test to #{pg_user};'\" 2>&1") do - its(:stdout) { should match /^GRANT$/ } - its(:exit_status) { should eq 0 } - end - end - - describe 'Create a test table' do - let(:disable_sudo) { false } - describe command("psql -h #{postgresql_host} -p #{pgbouncer_default_port} -U #{pg_user} postgres -c 'CREATE TABLE serverspec_test.pgbtest (col varchar(20));' 2>&1") do - its(:stdout) { should match /^CREATE TABLE$/ } - its(:exit_status) { should eq 0 } - end - end - - describe 'Insert values into the test table' do - let(:disable_sudo) { false } - describe command("psql -h #{postgresql_host} -p #{pgbouncer_default_port} -U #{pg_user} postgres -c \"INSERT INTO serverspec_test.pgbtest (col) values ('PGBSUCCESS');\" 2>&1") do - its(:stdout) { should match /^INSERT 0 1$/ } - its(:exit_status) { should eq 0 } - end - end - - describe 'Select values from the test table' do - let(:disable_sudo) { false } - describe command("psql -h #{postgresql_host} -p #{pgbouncer_default_port} -U #{pg_user} postgres -c 'SELECT col from serverspec_test.pgbtest;' 2>&1") do - its(:stdout) { should match /\bPGBSUCCESS\b/ } - its(:exit_status) { should eq 0 } - end - end - - end - - if replicated || (primary_node_host.include? host_inventory['hostname']) - - describe 'Select values from test tables' do - let(:disable_sudo) { false } - describe command("PGPASSWORD=#{pg_pass} psql -h #{postgresql_host} -p #{postgresql_default_port} -U #{pg_user} postgres -c 'SELECT * from serverspec_test.test;' 2>&1") do - its(:stdout) { should match /\bSUCCESS\b/ } - its(:exit_status) { should eq 0 } - end - describe command("PGPASSWORD=#{pg_pass} psql -h #{postgresql_host} -p #{postgresql_default_port} -U #{pg_user} postgres -c 'SELECT col from serverspec_test.pgbtest;' 2>&1") do - its(:stdout) { should match /\bPGBSUCCESS\b/ } - its(:exit_status) { should eq 0 } - end - end - - end - -end - ### Cleaning up -if !replicated - queryForDropping - - if pgbouncer_enabled - describe 'Drop test user' do - let(:disable_sudo) { false } - describe command("su - postgres -c \"psql -t -c 'DROP USER #{pg_user};'\" 2>&1") do - its(:stdout) { should match /^DROP ROLE$/ } - its(:exit_status) { should eq 0 } - end - describe command("su - -c \"sed -i '/#{pg_pass}/d' /etc/pgbouncer/userlist.txt && cat /etc/pgbouncer/userlist.txt\" 2>&1") do - its(:stdout) { should_not match /#{pg_pass}/ } - its(:exit_status) { should eq 0 } - end - end - end - -end +queryForDropping unless replicated if replicated && (last_node_host.include? host_inventory['hostname']) ssh_options = Specinfra.backend.get_config(:ssh_options) @@ -566,18 +467,6 @@ def queryForDropping expect(result).to match 'DROP SCHEMA' end end - it "Delegate drop user query to master node", :if => pgbouncer_enabled do - Net::SSH.start(primary_node_ip, ENV['user'], ssh_options) do|ssh| - result = ssh.exec!("sudo su - postgres -c \"psql -t -c 'DROP USER #{pg_user};'\" 2>&1") - expect(result).to match 'DROP ROLE' - end - end - it "Remove test user from userlist.txt", :if => pgbouncer_enabled do - Net::SSH.start(primary_node_ip, ENV['user'], ssh_options) do|ssh| - result = ssh.exec!("sudo su - -c \"sed -i '/#{pg_pass}/d' /etc/pgbouncer/userlist.txt && cat /etc/pgbouncer/userlist.txt\" 2>&1") - expect(result).not_to match "#{pg_pass}" - end - end end end @@ -633,20 +522,6 @@ def get_query_command_with_retries(json_query:, min_doc_hits:, retries: 600, ela end end - describe 'Check if Elasticsearch logs contain queries executed with PGBouncer', :if => pgbouncer_enabled do - query = get_elasticsearch_query(message_pattern: "#{pg_user} AND NOT MISC,SET") - command = get_query_command_with_retries(json_query: query, min_doc_hits: 6) - describe command(command.squish) do - its(:stdout) { should match /GRANT ALL ON SCHEMA serverspec_test to #{pg_user}/ } - its(:stdout) { should match /GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA serverspec_test to #{pg_user}/ } - its(:stdout) { should match /CREATE TABLE serverspec_test\.pgbtest/ } - its(:stdout) { should match /INSERT INTO serverspec_test\.pgbtest/ } - its(:stdout) { should match /CREATE USER #{pg_user} WITH PASSWORD/ } - its(:stdout) { should match /DROP USER #{pg_user}/ } - its(:exit_status) { should eq 0 } - end - end - describe 'Check support for multiline messages' do query = get_elasticsearch_query(message_pattern: "\"ADD COLUMN city text\"") describe command("curl -k -u admin:#{ELASTICSEARCH[:admin_password]} 'https://#{ELASTICSEARCH[:host]}:#{ELASTICSEARCH[:api_port]}/_search?pretty=true' -H 'Content-Type: application/json' -d '#{query.squish}'") do