From e389ad5c56495ffd03c40c2f720099223cd5722a Mon Sep 17 00:00:00 2001 From: Louis Pieterse Date: Thu, 2 May 2024 13:56:16 +0100 Subject: [PATCH 1/2] PG sources prefer SSL --- dev-project/docker-compose.yml | 13 +++- dev-project/pg/Dockerfile | 3 + dev-project/pg/pg_hba.conf | 108 +++++++++++++++++++++++++++++++++ dev-project/pg/server.crt | 77 +++++++++++++++++++++++ dev-project/pg/server.key | 28 +++++++++ dev-project/pg/server.req | 62 +++++++++++++++++++ 6 files changed, 289 insertions(+), 2 deletions(-) create mode 100644 dev-project/pg/pg_hba.conf create mode 100644 dev-project/pg/server.crt create mode 100644 dev-project/pg/server.key create mode 100644 dev-project/pg/server.req diff --git a/dev-project/docker-compose.yml b/dev-project/docker-compose.yml index 1bc2eae3c..cf4b60324 100644 --- a/dev-project/docker-compose.yml +++ b/dev-project/docker-compose.yml @@ -34,8 +34,17 @@ services: context: ./pg dockerfile: Dockerfile container_name: pipelinewise_dev_postgres_source - # Making some logical decoding adjustments - command: -c "wal_level=logical" -c "max_replication_slots=5" -c "max_wal_senders=5" + volumes: + - ./pg/pg_hba.conf:/var/lib/postgresql/pg_hba.conf + # Making some logical decoding and ssl adjustments + command: > + -c "wal_level=logical" + -c "max_replication_slots=5" + -c "max_wal_senders=5" + -c "ssl=on" + -c "ssl_cert_file=/var/lib/postgresql/server.crt" + -c "ssl_key_file=/var/lib/postgresql/server.key" + -c "hba_file=/var/lib/postgresql/pg_hba.conf" ports: - ${TAP_POSTGRES_PORT_ON_HOST}:${TAP_POSTGRES_PORT} environment: diff --git a/dev-project/pg/Dockerfile b/dev-project/pg/Dockerfile index bee14de01..5a629feb4 100644 --- a/dev-project/pg/Dockerfile +++ b/dev-project/pg/Dockerfile @@ -1,5 +1,8 @@ FROM debezium/postgres:12-alpine +COPY --chown=postgres:postgres server.crt /var/lib/postgresql/server.crt +COPY --chown=postgres:postgres --chmod=600 server.key /var/lib/postgresql/server.key + RUN apk add --no-cache --virtual .debezium-build-deps gcc clang15 llvm15 git make musl-dev pkgconf \ && git clone --depth 1 --branch wal2json_2_3 https://github.com/eulerto/wal2json.git \ && cd /wal2json \ diff --git a/dev-project/pg/pg_hba.conf b/dev-project/pg/pg_hba.conf new file mode 100644 index 000000000..c74dcf3c3 --- /dev/null +++ b/dev-project/pg/pg_hba.conf @@ -0,0 +1,108 @@ +# PostgreSQL Client Authentication Configuration File +# =================================================== +# +# Refer to the "Client Authentication" section in the PostgreSQL +# documentation for a complete description of this file. A short +# synopsis follows. +# +# This file controls: which hosts are allowed to connect, how clients +# are authenticated, which PostgreSQL user names they can use, which +# databases they can access. Records take one of these forms: +# +# local DATABASE USER METHOD [OPTIONS] +# host DATABASE USER ADDRESS METHOD [OPTIONS] +# hostssl DATABASE USER ADDRESS METHOD [OPTIONS] +# hostnossl DATABASE USER ADDRESS METHOD [OPTIONS] +# hostgssenc DATABASE USER ADDRESS METHOD [OPTIONS] +# hostnogssenc DATABASE USER ADDRESS METHOD [OPTIONS] +# +# (The uppercase items must be replaced by actual values.) +# +# The first field is the connection type: "local" is a Unix-domain +# socket, "host" is either a plain or SSL-encrypted TCP/IP socket, +# "hostssl" is an SSL-encrypted TCP/IP socket, and "hostnossl" is a +# non-SSL TCP/IP socket. Similarly, "hostgssenc" uses a +# GSSAPI-encrypted TCP/IP socket, while "hostnogssenc" uses a +# non-GSSAPI socket. +# +# DATABASE can be "all", "sameuser", "samerole", "replication", a +# database name, or a comma-separated list thereof. The "all" +# keyword does not match "replication". Access to replication +# must be enabled in a separate record (see example below). +# +# USER can be "all", a user name, a group name prefixed with "+", or a +# comma-separated list thereof. In both the DATABASE and USER fields +# you can also write a file name prefixed with "@" to include names +# from a separate file. +# +# ADDRESS specifies the set of hosts the record matches. It can be a +# host name, or it is made up of an IP address and a CIDR mask that is +# an integer (between 0 and 32 (IPv4) or 128 (IPv6) inclusive) that +# specifies the number of significant bits in the mask. A host name +# that starts with a dot (.) matches a suffix of the actual host name. +# Alternatively, you can write an IP address and netmask in separate +# columns to specify the set of hosts. Instead of a CIDR-address, you +# can write "samehost" to match any of the server's own IP addresses, +# or "samenet" to match any address in any subnet that the server is +# directly connected to. +# +# METHOD can be "trust", "reject", "md5", "password", "scram-sha-256", +# "gss", "sspi", "ident", "peer", "pam", "ldap", "radius" or "cert". +# Note that "password" sends passwords in clear text; "md5" or +# "scram-sha-256" are preferred since they send encrypted passwords. +# +# OPTIONS are a set of options for the authentication in the format +# NAME=VALUE. The available options depend on the different +# authentication methods -- refer to the "Client Authentication" +# section in the documentation for a list of which options are +# available for which authentication methods. +# +# Database and user names containing spaces, commas, quotes and other +# special characters must be quoted. Quoting one of the keywords +# ---------------------------------- +# +# If you want to allow non-local connections, you need to add more +# "host" records. In that case you will also need to make PostgreSQL +# listen on a non-local interface via the listen_addresses +# configuration parameter, or via the -i or -h command line switches. + +# CAUTION: Configuring the system for local "trust" authentication +# allows any local user to connect as any PostgreSQL user, including +# the database superuser. If you do not trust all your local users, +# use another authentication method. + + +# # TYPE DATABASE USER ADDRESS METHOD + +# # "local" is for Unix domain socket connections only +# local all all trust +# # IPv4 local connections: +# host all all 127.0.0.1/32 trust +# # IPv6 local connections: +# host all all ::1/128 trust +# # Allow replication connections from localhost, by a user with the +# # replication privilege. +# local replication all trust +# host replication all 127.0.0.1/32 trust +# host replication all ::1/128 trust + +# host all all all md5 +# host replication pipelinewise 0.0.0.0/0 trust + + +# TYPE DATABASE USER ADDRESS METHOD + +# "local" is for Unix domain socket connections only +local all all trust +# IPv4 local connections: +hostssl all all 127.0.0.1/32 trust +# IPv6 local connections: +hostssl all all ::1/128 trust +# Allow replication connections from localhost, by a user with the +# replication privilege. +local replication all trust +hostssl replication all 127.0.0.1/32 trust +hostssl replication all ::1/128 trust + +hostssl all all all md5 +hostssl replication pipelinewise 0.0.0.0/0 trust \ No newline at end of file diff --git a/dev-project/pg/server.crt b/dev-project/pg/server.crt new file mode 100644 index 000000000..fd67542cc --- /dev/null +++ b/dev-project/pg/server.crt @@ -0,0 +1,77 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + 5c:d5:36:4d:7b:de:c9:34:78:37:6a:5d:91:b9:be:4a:ed:f4:4a:d9 + Signature Algorithm: sha256WithRSAEncryption + Issuer: CN=localhost + Validity + Not Before: Apr 30 13:18:28 2024 GMT + Not After : May 30 13:18:28 2024 GMT + Subject: CN=localhost + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:a3:56:ee:3b:99:9d:0e:84:c9:e3:22:0d:a8:e1: + 80:5c:a3:ea:29:ce:97:51:de:db:39:4b:ec:bc:9b: + f8:83:9c:a1:21:94:e7:5b:fb:36:30:bc:55:84:ff: + 7f:11:63:68:19:4b:91:7e:ae:40:ca:7a:af:35:dc: + 4b:f6:9f:b6:d7:c9:d8:f6:2e:dd:b9:64:19:16:90: + dc:fa:c0:d8:9b:9e:71:99:2d:c7:a4:56:e9:7d:55: + a3:4a:48:8d:6e:2c:a2:31:db:9c:df:a0:d3:28:74: + d0:ce:37:a2:8a:f7:45:dd:5c:9c:63:d1:7a:22:15: + f5:64:89:2b:ba:1e:fe:0e:60:4f:8c:0e:4b:15:67: + 4d:cc:03:e4:48:3c:3d:85:c4:76:6c:ce:e9:b4:25: + b8:fc:e5:d5:05:cf:b0:60:46:3b:7e:4c:37:ed:9e: + 61:44:7e:3f:c1:91:34:ba:43:4d:39:15:bc:1f:d0: + 2c:bb:52:e9:c0:51:cb:44:d5:c8:13:ec:12:06:f9: + 01:84:a1:ed:d8:9d:dc:69:73:04:fc:ed:63:fe:3b: + 45:f7:37:b8:eb:ba:3e:ad:b3:78:ea:5d:26:c3:b2: + d9:14:35:dc:f6:90:48:a3:38:92:2c:72:1f:0e:a7: + 55:cb:bb:0a:d2:d1:85:bc:65:76:94:0e:46:73:ab: + 73:3f + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + 71:C2:B5:98:56:2F:8A:0F:F2:6A:97:2E:D0:17:DC:1B:28:E0:5D:72 + X509v3 Authority Key Identifier: + 71:C2:B5:98:56:2F:8A:0F:F2:6A:97:2E:D0:17:DC:1B:28:E0:5D:72 + X509v3 Basic Constraints: critical + CA:TRUE + Signature Algorithm: sha256WithRSAEncryption + Signature Value: + 95:c6:d4:7d:f0:6b:45:93:be:5a:64:e8:28:35:48:66:3b:d9: + 21:cc:21:65:ca:26:8a:4b:74:ea:29:b4:9f:7d:6a:ae:cc:14: + d3:01:99:96:4b:1a:07:6e:5a:98:9f:79:c4:d4:ad:05:5e:88: + 86:97:c2:1e:2c:f8:5f:c6:98:55:ed:d0:51:53:ee:ad:e1:7d: + cd:94:02:e7:32:17:54:1c:c7:ed:01:cb:8c:06:73:dc:63:81: + 5e:1d:86:7d:c4:6e:59:a5:0f:01:f6:ae:ea:67:a4:e5:58:8f: + 74:8e:46:e7:ff:b9:aa:d6:a1:cd:1b:44:4b:74:8c:43:c6:f7: + a4:78:f6:6c:60:c0:5d:b4:0c:cf:2e:e0:8e:27:84:81:a1:b6: + 5f:70:77:19:5f:ce:c7:7e:57:99:c1:44:a8:90:c1:ac:69:96: + 97:65:c6:6b:a3:17:17:89:2b:59:9d:65:af:42:9f:5e:09:17: + e3:c1:9c:59:f5:f2:8f:89:33:ec:c6:59:31:59:44:c9:24:3f: + c0:00:f2:b0:0f:4a:c8:23:e1:72:a1:e3:dd:81:11:12:0f:cd: + 9c:de:00:70:b0:ca:70:cd:c8:54:20:fd:49:00:d4:46:8b:ac: + 0d:e6:a9:c8:5c:26:fa:a4:66:ae:ba:c0:a0:d1:20:f9:78:b1: + ed:55:cf:6e +-----BEGIN CERTIFICATE----- +MIIDCTCCAfGgAwIBAgIUXNU2TXveyTR4N2pdkbm+Su30StkwDQYJKoZIhvcNAQEL +BQAwFDESMBAGA1UEAwwJbG9jYWxob3N0MB4XDTI0MDQzMDEzMTgyOFoXDTI0MDUz +MDEzMTgyOFowFDESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEF +AAOCAQ8AMIIBCgKCAQEAo1buO5mdDoTJ4yINqOGAXKPqKc6XUd7bOUvsvJv4g5yh +IZTnW/s2MLxVhP9/EWNoGUuRfq5AynqvNdxL9p+218nY9i7duWQZFpDc+sDYm55x +mS3HpFbpfVWjSkiNbiyiMduc36DTKHTQzjeiivdF3VycY9F6IhX1ZIkruh7+DmBP +jA5LFWdNzAPkSDw9hcR2bM7ptCW4/OXVBc+wYEY7fkw37Z5hRH4/wZE0ukNNORW8 +H9Asu1LpwFHLRNXIE+wSBvkBhKHt2J3caXME/O1j/jtF9ze467o+rbN46l0mw7LZ +FDXc9pBIoziSLHIfDqdVy7sK0tGFvGV2lA5Gc6tzPwIDAQABo1MwUTAdBgNVHQ4E +FgQUccK1mFYvig/yapcu0BfcGyjgXXIwHwYDVR0jBBgwFoAUccK1mFYvig/yapcu +0BfcGyjgXXIwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAlcbU +ffBrRZO+WmToKDVIZjvZIcwhZcomikt06im0n31qrswU0wGZlksaB25amJ95xNSt +BV6IhpfCHiz4X8aYVe3QUVPureF9zZQC5zIXVBzH7QHLjAZz3GOBXh2GfcRuWaUP +Afau6mek5ViPdI5G5/+5qtahzRtES3SMQ8b3pHj2bGDAXbQMzy7gjieEgaG2X3B3 +GV/Ox35XmcFEqJDBrGmWl2XGa6MXF4krWZ1lr0KfXgkX48GcWfXyj4kz7MZZMVlE +ySQ/wADysA9KyCPhcqHj3YEREg/NnN4AcLDKcM3IVCD9SQDURousDeapyFwm+qRm +rrrAoNEg+Xix7VXPbg== +-----END CERTIFICATE----- diff --git a/dev-project/pg/server.key b/dev-project/pg/server.key new file mode 100644 index 000000000..f7d02fa86 --- /dev/null +++ b/dev-project/pg/server.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCjVu47mZ0OhMnj +Ig2o4YBco+opzpdR3ts5S+y8m/iDnKEhlOdb+zYwvFWE/38RY2gZS5F+rkDKeq81 +3Ev2n7bXydj2Lt25ZBkWkNz6wNibnnGZLcekVul9VaNKSI1uLKIx25zfoNModNDO +N6KK90XdXJxj0XoiFfVkiSu6Hv4OYE+MDksVZ03MA+RIPD2FxHZszum0Jbj85dUF +z7BgRjt+TDftnmFEfj/BkTS6Q005Fbwf0Cy7UunAUctE1cgT7BIG+QGEoe3Yndxp +cwT87WP+O0X3N7jruj6ts3jqXSbDstkUNdz2kEijOJIsch8Op1XLuwrS0YW8ZXaU +DkZzq3M/AgMBAAECggEAOk3h/QlQzClvLTHvIUzZNnhWMjUpw5Iiu7mT2CFunIAw +7h1KVoV96viXKEHROEI9GuZkGY2W2A/h8f5gaTjKmg1Vck75W19VKz5reuCSkT4F ++7EhNdLGRrRmdLIgNecdwGFNhm/QX/0OucWXMjagzCruMh1IlNk0vn3OS4IbmsKJ +NiOAeAWBFJ5Ogv3CGFor4KSIbyG0TKktl0xdZ8OOZyVjbABSXL4N44TvJpz5iN7o +0h4dQrHLqxIl2Bk5bXaeOffFRJv4F1YnM7gGK4x2Glistd3QGx0ThJItrcpbrmWD +7sApIpwM1BU124hayXIwSEFuEq//3YR1T0Plp8+6UQKBgQDMp7uK8DRYbec6cENy +y94Z+IBpmMP6p4bAMXfIudqz23hNiCfgWFjzekMK8mNBwb9Qnx3DhaPWtUJNoS3D +9E77EVjZndpzwjleTTCcyC2eBAQ8C4TT9fcPe7N1IvTQ+SWBIIh8JCuIobzbc/tW +cuZ5Cg3RjGSaFfcr4YkX4HyZTwKBgQDMUaTB//tDCoudgH0mKptFlF8I5piBCsLG +18p9MAlsD/pIIbHEnzXahecFhn/CY5uKZox/4tlAQxNJ/BDV2oG6xd6VlMSoJkpV +G4pHHlM1pkWltpi8JIPrk1qrylcF9Sx8sNcwIIgR5dnp3zeREX3cKo3XKZbdgX5z +ilnWNMsrEQKBgQDDFk2Pt0Se2z1llqilILO63AlFIAwJv3KTeVRvMsPJWbgHaHhi +wrUge/a207JMd62w480Smi1BOg5XRAMFXbmAWdvxrikU2hunyjtB6QybKccDmhNn +nnuB9fBvBACNyEK5IdMZS714BH2sUJZRpTwQO8ZKb+NigvfYi6kM6sC7vwKBgF7h +oyM8HpCZexBfNL7ellCPgDp/614xgxYX676K6jU2jkGk9Aqc7MNX31qfhQDehMKW +sLRKwhyniXOYDLFi0U/Mx3qrJU/4yBgRu/seyAFn/3ve+bqVMnXZbgfM5PCtVEVr +UeLVmJrHJJUIxaMmfk4ZrNi4RYkDs5cwLlvdFGQhAoGBAKoO/3h9QUc/0+IORP6Q +BhWIU1UHiYI3XONDNUNcpomrR3YHYIDyr/nhZWTQmif96Kj/Il8+rZIhovMztF9c +JcmlqYgkz+o492E9eTeTV8a1F7dfVcjl4wyNQsFQwWE+1yT/oKocWD46GIY7navQ +eLTmVeK6BTR7ttZgdl3w6+XL +-----END PRIVATE KEY----- diff --git a/dev-project/pg/server.req b/dev-project/pg/server.req new file mode 100644 index 000000000..6bb7e2fb8 --- /dev/null +++ b/dev-project/pg/server.req @@ -0,0 +1,62 @@ +Certificate Request: + Data: + Version: 1 (0x0) + Subject: CN=localhost + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:a3:56:ee:3b:99:9d:0e:84:c9:e3:22:0d:a8:e1: + 80:5c:a3:ea:29:ce:97:51:de:db:39:4b:ec:bc:9b: + f8:83:9c:a1:21:94:e7:5b:fb:36:30:bc:55:84:ff: + 7f:11:63:68:19:4b:91:7e:ae:40:ca:7a:af:35:dc: + 4b:f6:9f:b6:d7:c9:d8:f6:2e:dd:b9:64:19:16:90: + dc:fa:c0:d8:9b:9e:71:99:2d:c7:a4:56:e9:7d:55: + a3:4a:48:8d:6e:2c:a2:31:db:9c:df:a0:d3:28:74: + d0:ce:37:a2:8a:f7:45:dd:5c:9c:63:d1:7a:22:15: + f5:64:89:2b:ba:1e:fe:0e:60:4f:8c:0e:4b:15:67: + 4d:cc:03:e4:48:3c:3d:85:c4:76:6c:ce:e9:b4:25: + b8:fc:e5:d5:05:cf:b0:60:46:3b:7e:4c:37:ed:9e: + 61:44:7e:3f:c1:91:34:ba:43:4d:39:15:bc:1f:d0: + 2c:bb:52:e9:c0:51:cb:44:d5:c8:13:ec:12:06:f9: + 01:84:a1:ed:d8:9d:dc:69:73:04:fc:ed:63:fe:3b: + 45:f7:37:b8:eb:ba:3e:ad:b3:78:ea:5d:26:c3:b2: + d9:14:35:dc:f6:90:48:a3:38:92:2c:72:1f:0e:a7: + 55:cb:bb:0a:d2:d1:85:bc:65:76:94:0e:46:73:ab: + 73:3f + Exponent: 65537 (0x10001) + Attributes: + (none) + Requested Extensions: + Signature Algorithm: sha256WithRSAEncryption + Signature Value: + 4e:b2:dd:6a:29:3e:78:9f:7c:95:e4:82:34:cd:b1:df:91:1f: + 05:75:22:4d:9c:02:f4:69:06:33:6e:c3:c6:a3:c3:6a:ac:12: + 52:0c:aa:a7:be:62:9e:f6:28:cb:ae:0c:56:49:58:d1:16:30: + be:d7:ab:86:0a:66:13:97:90:ed:a8:a5:32:22:9d:45:48:0f: + 58:9e:fd:e8:6d:94:93:56:4d:e9:be:ab:2f:10:37:e3:b1:a3: + f4:74:53:5f:25:e4:1d:fc:e3:ad:2a:fb:5c:24:40:ec:dd:e3: + 64:fe:4d:0f:a5:17:c2:28:f9:b8:a2:0a:99:0a:21:8f:8c:96: + 16:f4:bf:67:74:fb:31:64:1e:3a:af:b6:f2:40:f5:e9:2f:4e: + e5:0a:53:48:e8:33:58:3f:45:e5:b6:c8:d1:29:6d:25:63:b6: + 77:f5:09:be:a3:9a:29:f5:8d:57:31:ec:68:96:d3:f9:9a:1e: + f8:97:1b:d8:b0:3d:e3:cf:5b:a6:21:52:fe:d3:4e:e6:b7:86: + 68:96:c5:46:9a:07:35:2c:a7:45:d9:7d:41:43:13:16:e0:4f: + 28:36:ba:da:f3:11:47:07:7a:c0:e8:10:61:0d:0f:9e:19:d9: + 53:50:f3:15:2d:c2:ba:9f:f1:1a:62:d0:0c:74:08:b1:32:a4: + 0e:c6:92:bd +-----BEGIN CERTIFICATE REQUEST----- +MIICWTCCAUECAQAwFDESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0B +AQEFAAOCAQ8AMIIBCgKCAQEAo1buO5mdDoTJ4yINqOGAXKPqKc6XUd7bOUvsvJv4 +g5yhIZTnW/s2MLxVhP9/EWNoGUuRfq5AynqvNdxL9p+218nY9i7duWQZFpDc+sDY +m55xmS3HpFbpfVWjSkiNbiyiMduc36DTKHTQzjeiivdF3VycY9F6IhX1ZIkruh7+ +DmBPjA5LFWdNzAPkSDw9hcR2bM7ptCW4/OXVBc+wYEY7fkw37Z5hRH4/wZE0ukNN +ORW8H9Asu1LpwFHLRNXIE+wSBvkBhKHt2J3caXME/O1j/jtF9ze467o+rbN46l0m +w7LZFDXc9pBIoziSLHIfDqdVy7sK0tGFvGV2lA5Gc6tzPwIDAQABoAAwDQYJKoZI +hvcNAQELBQADggEBAE6y3WopPniffJXkgjTNsd+RHwV1Ik2cAvRpBjNuw8ajw2qs +ElIMqqe+Yp72KMuuDFZJWNEWML7Xq4YKZhOXkO2opTIinUVID1ie/ehtlJNWTem+ +qy8QN+Oxo/R0U18l5B38460q+1wkQOzd42T+TQ+lF8Io+biiCpkKIY+Mlhb0v2d0 ++zFkHjqvtvJA9ekvTuUKU0joM1g/ReW2yNEpbSVjtnf1Cb6jmin1jVcx7GiW0/ma +HviXG9iwPePPW6YhUv7TTua3hmiWxUaaBzUsp0XZfUFDExbgTyg2utrzEUcHesDo +EGEND54Z2VNQ8xUtwrqf8Rpi0Ax0CLEypA7Gkr0= +-----END CERTIFICATE REQUEST----- From e6744eb0568ed945e98e6c0f8f66718ce9ac273f Mon Sep 17 00:00:00 2001 From: Louis Pieterse Date: Wed, 8 May 2024 17:10:35 +0100 Subject: [PATCH 2/2] MariaDB sources prefer SSL --- .github/CODEOWNERS | 1 + CHANGELOG.md | 5 ++++ dev-project/docker-compose.yml | 21 ++++++++++++--- dev-project/my/ca-cert.pem | 19 +++++++++++++ dev-project/my/server-cert.pem | 18 +++++++++++++ dev-project/my/server-key.pem | 28 ++++++++++++++++++++ pipelinewise/fastsync/commons/tap_mysql.py | 16 ++++++++--- setup.py | 2 +- singer-connectors/tap-mysql/requirements.txt | 2 +- tests/db/tap_mysql_data.sql | 2 +- tests/db/tap_mysql_db.sh | 10 ++++--- tests/end_to_end/helpers/db.py | 1 + 12 files changed, 111 insertions(+), 14 deletions(-) create mode 100644 .github/CODEOWNERS create mode 100644 dev-project/my/ca-cert.pem create mode 100644 dev-project/my/server-cert.pem create mode 100644 dev-project/my/server-key.pem diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 000000000..8382fd1f4 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1 @@ +* @transferwise/analytics-platform diff --git a/CHANGELOG.md b/CHANGELOG.md index d8c9f45b1..be957fc4a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +0.60.0 (2024-05-10) +------------------- +- Bump `pipelinewise-tap-mysql`from `1.5.6` to `1.6.0` +- Attempt SSL for MariaDB and PG sources as Preferred connection method + 0.58.3 (2023-11-28) ------------------- - Bump `pipelinewise-tap-kafka`from `8.2.0` to `8.2.1` diff --git a/dev-project/docker-compose.yml b/dev-project/docker-compose.yml index cf4b60324..b27003e17 100644 --- a/dev-project/docker-compose.yml +++ b/dev-project/docker-compose.yml @@ -56,11 +56,26 @@ services: # MySQL service container used as test source database db_mysql_source: - image: mariadb:10.2.26 + image: mariadb:10.6.17 container_name: pipelinewise_dev_mysql_source ports: - ${TAP_MYSQL_PORT_ON_HOST}:${TAP_MYSQL_PORT} - command: --default-authentication-plugin=mysql_native_password --server-id=1 --log-bin=mysql-bin --binlog-format=ROW --binlog-do-db=${TAP_MYSQL_DB} --binlog-do-db=${TAP_MYSQL_REPLICA_DB} + volumes: + - ./my/server-cert.pem:/etc/my.cnf.d/certificates/server-cert.pem + - ./my/server-key.pem:/etc/my.cnf.d/certificates/server-key.pem + - ./my/ca-cert.pem:/etc/my.cnf.d/certificates/ca-cert.pem + command: > + --default-authentication-plugin=mysql_native_password + --server-id=1 + --log-bin=mysql-bin + --binlog-format=ROW + --binlog-do-db=${TAP_MYSQL_DB} + --binlog-do-db=${TAP_MYSQL_REPLICA_DB} + --tls_version=TLSv1.2 + --require_secure_transport=ON + --ssl_cert=/etc/my.cnf.d/certificates/server-cert.pem + --ssl_key=/etc/my.cnf.d/certificates/server-key.pem + --ssl_ca=/etc/my.cnf.d/certificates/ca-cert.pem environment: MYSQL_ROOT_PASSWORD: ${TAP_MYSQL_ROOT_PASSWORD} MYSQL_USER: ${TAP_MYSQL_USER} @@ -70,7 +85,7 @@ services: - pipelinewise_network db_mysql_source_replica: - image: mariadb:10.2.26 + image: mariadb:10.6.17 container_name: pipelinewise_dev_mysql_source_replica ports: - ${TAP_MYSQL_REPLICA_PORT_ON_HOST}:${TAP_MYSQL_REPLICA_PORT} diff --git a/dev-project/my/ca-cert.pem b/dev-project/my/ca-cert.pem new file mode 100644 index 000000000..8aff547ea --- /dev/null +++ b/dev-project/my/ca-cert.pem @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDCzCCAfOgAwIBAgIUM6GLBTv83a3YJvi92Z2Uo+BQneQwDQYJKoZIhvcNAQEL +BQAwFDESMBAGA1UEAwwJbG9jYWxob3N0MCAXDTI0MDUwNzExMzkzN1oYDzMwMjMw +OTA4MTEzOTM3WjAUMRIwEAYDVQQDDAlsb2NhbGhvc3QwggEiMA0GCSqGSIb3DQEB +AQUAA4IBDwAwggEKAoIBAQDB50ieR/FeYkAAUPVadABvo3BHtGmgTETk+E6ui0ip +pds+MW0tBCHV2jsstQFjf57y3dpN0DTBVSegPlZ2950J8yAVKvC53c8Szt2T+SrY +Akrlc09tTx8uOnHL9g9gP3UddREABrUaS6F1/UjAweS82JnlwCobFVqi/nCwm2Sf +Zp+CrzjbUIaHjYzO+gl95+g4RpqxENSCGx3U6iTA8hoCH+a2wPfttX7IcN1iBRQz +szLkWqA+gEsm42+eI/COMu/M3xsUnOAGusJ2oOuCrs3blCH/1ZOAB6rfe41Zkx9P +FS8FadpK5p8k1Sz0xsjV9ekfgDftXGXAReQDokLjzNg9AgMBAAGjUzBRMB0GA1Ud +DgQWBBTqN3S7ajqoO9AgljB5x/MXtppBlTAfBgNVHSMEGDAWgBTqN3S7ajqoO9Ag +ljB5x/MXtppBlTAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQBD +2sgXX7uP4L3LPff6UWnoLg8wS0pK8ifYsBQlF33kVmPA5RnDjabz5w6K/fag9lkw +qZlLiG976Ov7dg7Ic32sExo6QmTnmWzYHwZO0cI+QXMO1XUM63e+A5Y3/jGHxocY +6Ral9dy4tAbEGqPBCMYwVCl7LXcTgDHK9zSmzOkkpGgPp609Sd/bf/02kNDsuwOQ +rTT4JcTtmaXdZQo1T+Q8x6kc8MTvXa0jfuZ+vIfIHUkdsBTw7nfeYMXGzGbVlfvU +hxTKZeSxVrnfrL1CMMAxb21XP6TaeD4VIMgbW1yCV1BQX65w8apef58HlpShfKU0 +ClBg7sMUAlF8ViavUH9t +-----END CERTIFICATE----- diff --git a/dev-project/my/server-cert.pem b/dev-project/my/server-cert.pem new file mode 100644 index 000000000..2e0b1903a --- /dev/null +++ b/dev-project/my/server-cert.pem @@ -0,0 +1,18 @@ +-----BEGIN CERTIFICATE----- +MIIC5TCCAc2gAwIBAgIBATANBgkqhkiG9w0BAQsFADAUMRIwEAYDVQQDDAlsb2Nh +bGhvc3QwHhcNMjQwNTA3MTE0MDMwWhcNMjUwNTA3MTE0MDMwWjAUMRIwEAYDVQQD +DAlsb2NhbGhvc3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDRZTiB +ElhU/6zKo3WulHFsKjQwkSwwmDE1jfs8YT7Paxtl3izowWW1lf6/bLhFLQ3WcTI3 +BCAxyprWIAJPw1S0ie8zLOousx0tP9pt5L58UF6Y2cvGTi0lXMBkD8IpdVBZaDNV +jguo4qo9LK34Vs+PyTagQV+xBxxpUIVCWUVJBHoVzHfGSCJo/ZoUG1KtpQAUkWCN +sgQC7AhlsR/V0IEitvLnjx8P9dnVNRcHq7wI/156uMbhtn++K6Vh7xL47rnX85et +YEbjzb2S/66dc0c8IZvyEG84jDi94PYmVvM4OorOORkGfa2zbpRzHPYrpAe5CxUo +5R5S5jDyKjwz0JZnAgMBAAGjQjBAMB0GA1UdDgQWBBS4ha9z1pxAhTn9iVBkyjv+ +x4lPGjAfBgNVHSMEGDAWgBTqN3S7ajqoO9AgljB5x/MXtppBlTANBgkqhkiG9w0B +AQsFAAOCAQEACIWEkkS/YE0ozt7lXbLOtvN2IIAqkJN6tgWgT1P1QoNzBflYY3hW +Hb/Z48mnF8pHHllOcyILyM81FT5YNwurKHhoHj88Mquur2LuU1qwdKIPaQCgXyiU +F06gB3F6qoLyc+lzcXcH632/TQSaCeXwGPQYV1/CbR7ZssxrCG8drsFhmkPgQdfo +wbywXOLASkp60EtkqqEisKRm2KhaWoDyLGy4u9e5SDAlpYLgFSOQ/Swac5CTuM7c +xNaImXY8XB5za/OuG7vmRuvXjeCjmvJDBQbEvC6W0w5ZU+/dC6UMW180UMAwh8oR +j2jRFGGoDtFrYXTI2vHxydJRNpuOc+xwfw== +-----END CERTIFICATE----- diff --git a/dev-project/my/server-key.pem b/dev-project/my/server-key.pem new file mode 100644 index 000000000..356ea92e1 --- /dev/null +++ b/dev-project/my/server-key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDRZTiBElhU/6zK +o3WulHFsKjQwkSwwmDE1jfs8YT7Paxtl3izowWW1lf6/bLhFLQ3WcTI3BCAxyprW +IAJPw1S0ie8zLOousx0tP9pt5L58UF6Y2cvGTi0lXMBkD8IpdVBZaDNVjguo4qo9 +LK34Vs+PyTagQV+xBxxpUIVCWUVJBHoVzHfGSCJo/ZoUG1KtpQAUkWCNsgQC7Ahl +sR/V0IEitvLnjx8P9dnVNRcHq7wI/156uMbhtn++K6Vh7xL47rnX85etYEbjzb2S +/66dc0c8IZvyEG84jDi94PYmVvM4OorOORkGfa2zbpRzHPYrpAe5CxUo5R5S5jDy +Kjwz0JZnAgMBAAECggEACoeD/V2OSovmxPPORZ6aS32dzTlnlvP4OYCvuvwK3AQ/ +RvKSoIulESn/OuH9LCFVWkyr4T9YXRiai5zlcNxaJI/xEso15m5ET9OvqKoBeB/U +eewtxoWE2KWe2Qk/0rTe+vf16N3xMSdgtx3Ji7zVyB2unQVcifrZD2Sopsr1ZHP5 +ZANM07D++IWnKaQHLs92RpnyZEvpfanUhZCoB5zLnpq0BRjXBMXeFc3hDNF6kml+ +YOYLijKVOm+WPqCFjDFqSK8YP9v4lqaOXFzAMYt8q1kHjVWe5/OzD9fOkiLbVWmw +29sY7ZZ3DIR5YTVRlKRXTd8btnBO+VDPNCyeFE9dEQKBgQDrDHr4aZ5d1KH4iaG0 +WhJZijDi6rKZxFIW6mjT8CE3w9b5fjb625tr2JSXhjAzqMkMGaHl0c8beDRNfrcv +YiPt0P551/rfT61VJSC6OhC0I30XaFeVJLhBkNDsOUvY4YkTPPyBagW3Y2MpVpFd +z8nJcVpb84iToJ5t0s8dp4mAtwKBgQDkD1zZ+nLLpavcokUKqoa4Pz3OchLQKDQ+ +WxHLn7UEW7Qm8tAcnyyltci1rGZBp7aNO0Oq3DxriPsivJb/Jjv7IifPG1TS2Prs +9eIEy4Dak4UGjIFUYbIYYbPCdpWclwMDAJjHQSl+N9hP1MCDRXKK25BwsF5ptfyu +a+zUmcKH0QKBgBC4ZI8DRGa8V4FMmRoxRhjkgbHCrE3P71798dMRw+spnUNGPTb/ +JEE5uOhGfWtPWG3p7OBiWm9Dz8SrfPB79L5YHaIhQRFdo6Do8TkOJF4MIDvoCkLr +x8eyO4pyUPjuokeiEPxLxHhVM5qfM9wOifYwgBMGcfK+fEantmAPqPJlAoGAHWwZ +94lHHBzQklNhAuJDJSwLhkHTSYWWT+Eh55pAGULXeBrlXHvwn6RXM5D9VEUV2Ryn +OzjUlq8Cb1XTSTGelWSvrBmDlPwnS0DhmurCQfC8Pb8MxF0iQ+D+9vyjiqM04UNH +cpVmjAEaaBozh/wCrgVrmg427/5pjfsgf8EihrECgYA62TsMmTcWvQOE8OSoInwT +rlaMn68+weFgQDHkHmxwSA4fnycbN4xfUBrnDffBSR/yrrM76laaYl05EegRhFSy +iwDL+vnkT7TvS6zcIk4VRWMrGNqXS7/xPeNxYr9Dsk2GlwqcH4DTtS3bLXIkYV4J +gTwIrfpIQVL/vwRcyP3Thg== +-----END PRIVATE KEY----- diff --git a/pipelinewise/fastsync/commons/tap_mysql.py b/pipelinewise/fastsync/commons/tap_mysql.py index 89b8c4389..cada792ac 100644 --- a/pipelinewise/fastsync/commons/tap_mysql.py +++ b/pipelinewise/fastsync/commons/tap_mysql.py @@ -122,10 +122,12 @@ def open_connections(self): self.conn: Connection = pymysql.connect( **conn_params, cursorclass=pymysql.cursors.DictCursor, + ssl={'': True} ) self.conn_unbuffered: Connection = pymysql.connect( **conn_params, cursorclass=pymysql.cursors.SSCursor, + ssl={'': True} ) # Set session variables by running a list of SQLs which is defined @@ -521,8 +523,11 @@ def __get_primary_server_uuid(self) -> str: Returns: server uuid """ - conn = pymysql.connect(**self.get_connection_parameters(prioritize_primary=True)[0], - cursorclass=pymysql.cursors.DictCursor) if self.is_replica else None + conn = pymysql.connect( + **self.get_connection_parameters(prioritize_primary=True)[0], + cursorclass=pymysql.cursors.DictCursor, + ssl={'': True} + ) if self.is_replica else None result = self.query('select @@server_uuid as server_uuid;', conn) @@ -537,8 +542,11 @@ def __get_primary_server_id(self) -> int: Returns: server uuid """ - conn = pymysql.connect(**self.get_connection_parameters(prioritize_primary=True)[0], - cursorclass=pymysql.cursors.DictCursor) if self.is_replica else None + conn = pymysql.connect( + **self.get_connection_parameters(prioritize_primary=True)[0], + cursorclass=pymysql.cursors.DictCursor, + ssl={'': True} + ) if self.is_replica else None result = self.query('select @@server_id as server_id;', conn) diff --git a/setup.py b/setup.py index 03bfc15c7..517ddcbd7 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ setup(name='pipelinewise', python_requires='>=3.7,<3.10', - version='0.58.3', + version='0.60.0', description='PipelineWise', long_description=LONG_DESCRIPTION, long_description_content_type='text/markdown', diff --git a/singer-connectors/tap-mysql/requirements.txt b/singer-connectors/tap-mysql/requirements.txt index f09a912c2..d9e4ff84a 100644 --- a/singer-connectors/tap-mysql/requirements.txt +++ b/singer-connectors/tap-mysql/requirements.txt @@ -1 +1 @@ -pipelinewise-tap-mysql==1.5.6 \ No newline at end of file +pipelinewise-tap-mysql==1.6.0 diff --git a/tests/db/tap_mysql_data.sql b/tests/db/tap_mysql_data.sql index 008547193..bd0213af4 100644 --- a/tests/db/tap_mysql_data.sql +++ b/tests/db/tap_mysql_data.sql @@ -52,7 +52,7 @@ Maatjies', 20, 'A', null, '09:16:10'), (8,'Liewe Maatjies', 10, null, '[{"key": "Value''s One", "actions": []},{"key": "Value\U00000027s Two", "actions": ' || '[]}]', '00:30:00'), (9, 'dolor sit amet', 19, 'C', '[]', '23:50:19'), - (10, 'sit amet', 100, 'E', '', '19:30:19') + (10, 'sit amet', 100, 'E', null, '19:30:19') ; /*!40000 ALTER TABLE `edgydata` ENABLE KEYS */; diff --git a/tests/db/tap_mysql_db.sh b/tests/db/tap_mysql_db.sh index d00d5a5e3..0fc1dfd5f 100755 --- a/tests/db/tap_mysql_db.sh +++ b/tests/db/tap_mysql_db.sh @@ -1,6 +1,6 @@ #!/bin/bash -e # -# Building a test MySQL database for integration testing of tap-mysql +# Building a test MySQL database for integration testing of tap-mysql # The sample database available at https://github.com/ikostan/RESTAURANT-DATABASE PWD="$(dirname "$0")" @@ -26,6 +26,7 @@ fi echo "SETTING UP MYSQL PRIMARY SERVER FOR REPLICATION" mysql --protocol TCP \ +--ssl \ --host ${TAP_MYSQL_HOST} \ --port ${TAP_MYSQL_PORT} \ --user root \ @@ -43,7 +44,7 @@ mysql --protocol TCP \ echo "GETTING MYSQL PRIMARY SERVER LOG INFO" -MASTER_LOG_STATUS=`mysql --protocol TCP --host ${TAP_MYSQL_HOST} --port ${TAP_MYSQL_PORT} --user root --password=${TAP_MYSQL_ROOT_PASSWORD} -e "SHOW MASTER STATUS;"` +MASTER_LOG_STATUS=`mysql --protocol TCP --ssl --host ${TAP_MYSQL_HOST} --port ${TAP_MYSQL_PORT} --user root --password=${TAP_MYSQL_ROOT_PASSWORD} -e "SHOW MASTER STATUS;"` CURRENT_LOG=`echo $MASTER_LOG_STATUS | awk '{print $5}'` CURRENT_POS=`echo $MASTER_LOG_STATUS | awk '{print $6}'` @@ -54,13 +55,13 @@ mysql --protocol TCP \ --port ${TAP_MYSQL_REPLICA_PORT} \ --user ${TAP_MYSQL_REPLICA_USER} \ --password=${TAP_MYSQL_REPLICA_PASSWORD} \ --e "STOP SLAVE; CHANGE MASTER TO MASTER_HOST='${TAP_MYSQL_HOST}',MASTER_USER='${TAP_MYSQL_USER}',MASTER_PASSWORD='${TAP_MYSQL_PASSWORD}',MASTER_LOG_FILE='${CURRENT_LOG}',MASTER_LOG_POS=${CURRENT_POS}; START SLAVE;" +-e "STOP SLAVE; CHANGE MASTER TO MASTER_SSL=1,MASTER_SSL_VERIFY_SERVER_CERT=0,MASTER_HOST='${TAP_MYSQL_HOST}',MASTER_USER='${TAP_MYSQL_USER}',MASTER_PASSWORD='${TAP_MYSQL_PASSWORD}',MASTER_LOG_FILE='${CURRENT_LOG}',MASTER_LOG_POS=${CURRENT_POS}; START SLAVE;" # Download the sample database and build it - echo "DUMPING DATA INTO PRIMARY MYSQL DATABASE" mysql --protocol TCP \ +--ssl \ --host ${TAP_MYSQL_HOST} \ --port ${TAP_MYSQL_PORT} \ --user ${TAP_MYSQL_USER} \ @@ -70,6 +71,7 @@ ${TAP_MYSQL_DB} < ${TEST_DB_SQL} echo "DUMPING DATA INTO PRIMARY MYSQL DATABASE2" mysql --protocol TCP \ +--ssl \ --host ${TAP_MYSQL_HOST} \ --port ${TAP_MYSQL_PORT} \ --user ${TAP_MYSQL_USER} \ diff --git a/tests/end_to_end/helpers/db.py b/tests/end_to_end/helpers/db.py index 700b58305..405f6e514 100644 --- a/tests/end_to_end/helpers/db.py +++ b/tests/end_to_end/helpers/db.py @@ -38,6 +38,7 @@ def run_query_mysql(query, host, port, user, password, database): database=database, charset='utf8mb4', cursorclass=pymysql.cursors.Cursor, + ssl={'': True} ) as cur: cur.execute(query) if cur.rowcount > 0: