From cfebfe7115a8dd5641d07fc3924347dd13c6c8d2 Mon Sep 17 00:00:00 2001 From: Itay Grudev Date: Wed, 13 Nov 2024 00:46:01 +0200 Subject: [PATCH] ParadeDB Enterprise (#58) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Philippe Noël --- .github/workflows/tests-cluster-chainsaw.yaml | 4 ++ charts/paradedb/README.md | 21 +++++++- charts/paradedb/README.md.gotmpl | 18 ++++++- charts/paradedb/templates/_bootstrap.tpl | 6 +-- charts/paradedb/templates/_helpers.tpl | 6 +++ charts/paradedb/templates/cluster.yaml | 2 +- ...01-paradedb-NCC-1701-D_cluster-assert.yaml | 6 +++ .../01-paradedb-NCC-1701-D_cluster.yaml | 12 +++++ .../02-paradedb_test-assert.yaml | 6 +++ .../paradedb-enterprise/02-paradedb_test.yaml | 46 ++++++++++++++++ .../paradedb-enterprise/chainsaw-test.yaml | 54 +++++++++++++++++++ charts/paradedb/values.schema.json | 3 ++ charts/paradedb/values.yaml | 1 + 13 files changed, 178 insertions(+), 7 deletions(-) create mode 100644 charts/paradedb/test/paradedb-enterprise/01-paradedb-NCC-1701-D_cluster-assert.yaml create mode 100644 charts/paradedb/test/paradedb-enterprise/01-paradedb-NCC-1701-D_cluster.yaml create mode 100644 charts/paradedb/test/paradedb-enterprise/02-paradedb_test-assert.yaml create mode 100644 charts/paradedb/test/paradedb-enterprise/02-paradedb_test.yaml create mode 100644 charts/paradedb/test/paradedb-enterprise/chainsaw-test.yaml diff --git a/.github/workflows/tests-cluster-chainsaw.yaml b/.github/workflows/tests-cluster-chainsaw.yaml index 809ad5bdb..f313eaf23 100644 --- a/.github/workflows/tests-cluster-chainsaw.yaml +++ b/.github/workflows/tests-cluster-chainsaw.yaml @@ -73,5 +73,9 @@ jobs: --values ./.github/minio.yaml \ tenant minio-operator/tenant + # The Docker Hub tokens are required for the ParadeDB Enterprise tests - name: Run Kyverno/Chainsaw run: chainsaw test + env: + PARADEDB_ENTERPRISE_DOCKER_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + PARADEDB_ENTERPRISE_DOCKER_PAT: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }} diff --git a/charts/paradedb/README.md b/charts/paradedb/README.md index 01a71a23c..b51ed14eb 100644 --- a/charts/paradedb/README.md +++ b/charts/paradedb/README.md @@ -91,7 +91,23 @@ helm upgrade --atomic --install paradedb --namespace paradedb --create-namespace ### Database Types -To use the ParadeDB Helm Chart, specify `paradedb` via the `type` parameter. +To create a ParadeDB cluster, you must specify either `paradedb` or `paradedb-enterprise` via the `type` parameter. + +> [!IMPORTANT] +> When using `paradedb-enterprise` you must also specify the `cluster.imagePullSecrets` containing the Docker registry credentials. You can create one with: +> +> ```bash +> kubectl -n NAMESPACE create secret docker-registry paradedb-enterprise-registry-cred --docker-server="https://index.docker.io/v1/" --docker-username="USERNAME" --docker-password="ACCESS_TOKEN" +> ``` +> +> You then need to set the name of the secret in the `values.yaml` file with: +> +> ```yaml +> type: paradedb-enterprise +> cluster: +> imagePullSecrets: +> - name: paradedb-enterprise-registry-cred +> ``` ### Modes of Operation @@ -210,6 +226,7 @@ refer to the [CloudNativePG Documentation](https://cloudnative-pg.io/documentat | cluster.priorityClassName | string | `""` | | | cluster.resources | object | `{}` | Resources requirements of every generated Pod. Please refer to https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ for more information. We strongly advise you use the same setting for limits and requests so that your cluster pods are given a Guaranteed QoS. See: https://kubernetes.io/docs/concepts/workloads/pods/pod-qos/ | | cluster.roles | list | `[]` | This feature enables declarative management of existing roles, as well as the creation of new roles if they are not already present in the database. See: https://cloudnative-pg.io/documentation/current/declarative_role_management/ | +| cluster.serviceAccountTemplate | object | `{}` | Configure the generation of the service account | | cluster.storage.size | string | `"8Gi"` | | | cluster.storage.storageClass | string | `""` | | | cluster.superuserSecret | string | `""` | | @@ -288,7 +305,7 @@ refer to the [CloudNativePG Documentation](https://cloudnative-pg.io/documentat | recovery.s3.secretKey | string | `""` | | | recovery.secret.create | bool | `true` | Whether to create a secret for the backup credentials | | recovery.secret.name | string | `""` | Name of the backup credentials secret | -| type | string | `"paradedb"` | Type of the CNPG database. Available types: * `paradedb` | +| type | string | `"paradedb"` | Type of the CNPG database. Available types: * `paradedb` * `paradedb-enterprise` | | version.paradedb | string | `"0.12.0"` | We default to v0.12.0 for testing and local development | | version.postgresql | string | `"16"` | PostgreSQL major version to use | | poolers[].name | string | `` | Name of the pooler resource | diff --git a/charts/paradedb/README.md.gotmpl b/charts/paradedb/README.md.gotmpl index 5b2b50ef7..5719c1429 100644 --- a/charts/paradedb/README.md.gotmpl +++ b/charts/paradedb/README.md.gotmpl @@ -91,7 +91,23 @@ helm upgrade --atomic --install paradedb --namespace paradedb --create-namespace ### Database Types -To use the ParadeDB Helm Chart, specify `paradedb` via the `type` parameter. +To create a ParadeDB cluster, you must specify either `paradedb` or `paradedb-enterprise` via the `type` parameter. + +> [!IMPORTANT] +> When using `paradedb-enterprise` you must also specify the `cluster.imagePullSecrets` containing the Docker registry credentials. You can create one with: +> +> ```bash +> kubectl -n NAMESPACE create secret docker-registry paradedb-enterprise-registry-cred --docker-server="https://index.docker.io/v1/" --docker-username="USERNAME" --docker-password="ACCESS_TOKEN" +> ``` +> +> You then need to set the name of the secret in the `values.yaml` file with: +> +> ```yaml +> type: paradedb-enterprise +> cluster: +> imagePullSecrets: +> - name: paradedb-enterprise-registry-cred +> ``` ### Modes of Operation diff --git a/charts/paradedb/templates/_bootstrap.tpl b/charts/paradedb/templates/_bootstrap.tpl index f40643c66..bb3886a3c 100644 --- a/charts/paradedb/templates/_bootstrap.tpl +++ b/charts/paradedb/templates/_bootstrap.tpl @@ -11,7 +11,7 @@ bootstrap: owner: {{ tpl .Values.cluster.initdb.owner . }} {{- end }} postInitSQL: - {{- if eq .Values.type "paradedb" }} + {{- if or (eq .Values.type "paradedb") (eq .Values.type "paradedb-enterprise") }} - CREATE EXTENSION IF NOT EXISTS pg_cron; {{- end }} {{- with .Values.cluster.initdb }} @@ -20,7 +20,7 @@ bootstrap: {{- end -}} {{- end }} postInitApplicationSQL: - {{- if eq .Values.type "paradedb" }} + {{- if or (eq .Values.type "paradedb") (eq .Values.type "paradedb-enterprise") }} - CREATE EXTENSION IF NOT EXISTS pg_search; - CREATE EXTENSION IF NOT EXISTS pg_analytics; - CREATE EXTENSION IF NOT EXISTS pg_ivm; @@ -37,7 +37,7 @@ bootstrap: {{- end -}} {{- end }} postInitTemplateSQL: - {{- if eq .Values.type "paradedb" }} + {{- if or (eq .Values.type "paradedb") (eq .Values.type "paradedb-enterprise") }} - CREATE EXTENSION IF NOT EXISTS pg_search; - CREATE EXTENSION IF NOT EXISTS pg_analytics; - CREATE EXTENSION IF NOT EXISTS pg_ivm; diff --git a/charts/paradedb/templates/_helpers.tpl b/charts/paradedb/templates/_helpers.tpl index 041e90353..3a225f8d5 100644 --- a/charts/paradedb/templates/_helpers.tpl +++ b/charts/paradedb/templates/_helpers.tpl @@ -69,6 +69,8 @@ If a custom imageName is available, use it, otherwise use the defaults based on {{- printf "ghcr.io/cloudnative-pg/postgresql:%s" .Values.version.postgresql -}} {{- else if eq .Values.type "paradedb" -}} {{- printf "paradedb/paradedb:%s-v%s" .Values.version.postgresql .Values.version.paradedb -}} + {{- else if eq .Values.type "paradedb-enterprise" -}} + {{- printf "paradedb/paradedb-enterprise:%s-v%s" .Values.version.postgresql .Values.version.paradedb -}} {{- else -}} {{ fail "Invalid cluster type!" }} {{- end }} @@ -103,6 +105,8 @@ Postgres UID {{- .Values.cluster.postgresUID }} {{- else if eq .Values.type "paradedb" -}} {{- 999 -}} + {{- else if eq .Values.type "paradedb-enterprise" -}} + {{- 999 -}} {{- else -}} {{- 26 -}} {{- end -}} @@ -116,6 +120,8 @@ Postgres GID {{- .Values.cluster.postgresGID }} {{- else if eq .Values.type "paradedb" -}} {{- 999 -}} + {{- else if eq .Values.type "paradedb-enterprise" -}} + {{- 999 -}} {{- else -}} {{- 26 -}} {{- end -}} diff --git a/charts/paradedb/templates/cluster.yaml b/charts/paradedb/templates/cluster.yaml index c6d13b930..8359c134c 100644 --- a/charts/paradedb/templates/cluster.yaml +++ b/charts/paradedb/templates/cluster.yaml @@ -53,7 +53,7 @@ spec: {{ end }} postgresql: shared_preload_libraries: - {{- if eq .Values.type "paradedb" }} + {{- if or (eq .Values.type "paradedb") (eq .Values.type "paradedb-enterprise") }} - pg_search - pg_analytics - pg_cron diff --git a/charts/paradedb/test/paradedb-enterprise/01-paradedb-NCC-1701-D_cluster-assert.yaml b/charts/paradedb/test/paradedb-enterprise/01-paradedb-NCC-1701-D_cluster-assert.yaml new file mode 100644 index 000000000..75c7d8771 --- /dev/null +++ b/charts/paradedb/test/paradedb-enterprise/01-paradedb-NCC-1701-D_cluster-assert.yaml @@ -0,0 +1,6 @@ +apiVersion: postgresql.cnpg.io/v1 +kind: Cluster +metadata: + name: paradedb-ncc-1701-d +status: + readyInstances: 1 diff --git a/charts/paradedb/test/paradedb-enterprise/01-paradedb-NCC-1701-D_cluster.yaml b/charts/paradedb/test/paradedb-enterprise/01-paradedb-NCC-1701-D_cluster.yaml new file mode 100644 index 000000000..1a023d5b4 --- /dev/null +++ b/charts/paradedb/test/paradedb-enterprise/01-paradedb-NCC-1701-D_cluster.yaml @@ -0,0 +1,12 @@ +type: paradedb-enterprise +mode: standalone + +cluster: + instances: 1 + storage: + size: 256Mi + imagePullSecrets: + - name: paradedb-enterprise-registry-cred + +backups: + enabled: false diff --git a/charts/paradedb/test/paradedb-enterprise/02-paradedb_test-assert.yaml b/charts/paradedb/test/paradedb-enterprise/02-paradedb_test-assert.yaml new file mode 100644 index 000000000..34cf73f0c --- /dev/null +++ b/charts/paradedb/test/paradedb-enterprise/02-paradedb_test-assert.yaml @@ -0,0 +1,6 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: paradedb-enterprise-test +status: + succeeded: 1 diff --git a/charts/paradedb/test/paradedb-enterprise/02-paradedb_test.yaml b/charts/paradedb/test/paradedb-enterprise/02-paradedb_test.yaml new file mode 100644 index 000000000..601918280 --- /dev/null +++ b/charts/paradedb/test/paradedb-enterprise/02-paradedb_test.yaml @@ -0,0 +1,46 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: paradedb-enterprise-test +spec: + template: + spec: + restartPolicy: OnFailure + containers: + - name: data-test + env: + - name: DB_URI + valueFrom: + secretKeyRef: + name: paradedb-ncc-1701-d-app + key: uri + image: alpine:3.19 + command: ['sh', '-c'] + args: + - | + apk --no-cache add postgresql-client + psql "$DB_URI" <<-EOSQL + CALL paradedb.create_bm25_test_table( + schema_name => 'public', + table_name => 'mock_items' + ); + CALL paradedb.create_bm25( + index_name => 'search_idx', + table_name => 'mock_items', + key_field => 'id', + text_fields => paradedb.field('description') || paradedb.field('category'), + numeric_fields => paradedb.field('rating'), + boolean_fields => paradedb.field('in_stock'), + datetime_fields => paradedb.field('created_at'), + json_fields => paradedb.field('metadata'), + range_fields => paradedb.field('weight_range') + ); + EOSQL + RESULT=$(psql "$DB_URI" -t) <<-EOSQL + SELECT description + FROM mock_items + WHERE description @@@ '"bluetooth speaker"~1' + LIMIT 1; + EOSQL + echo -$RESULT- + test "$RESULT" = " Bluetooth-enabled speaker" diff --git a/charts/paradedb/test/paradedb-enterprise/chainsaw-test.yaml b/charts/paradedb/test/paradedb-enterprise/chainsaw-test.yaml new file mode 100644 index 000000000..4c982523f --- /dev/null +++ b/charts/paradedb/test/paradedb-enterprise/chainsaw-test.yaml @@ -0,0 +1,54 @@ +## +# This test sets up a ParadeDB Enterprise Cluster and ensures that ParadeDB extensions are available. +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: paradedb-enterprise +spec: + timeouts: + apply: 1s + assert: 2m + cleanup: 1m + steps: + - name: Install a standalone ParadeDB Enterprise CNPG Cluster + try: + - script: + content: | + kubectl -n $NAMESPACE create secret docker-registry paradedb-enterprise-registry-cred --docker-server="https://index.docker.io/v1/" --docker-username="$PARADEDB_ENTERPRISE_DOCKER_USERNAME" --docker-password="$PARADEDB_ENTERPRISE_DOCKER_PAT" + helm upgrade \ + --install \ + --namespace $NAMESPACE \ + --values ./01-paradedb-NCC-1701-D_cluster.yaml \ + --wait \ + paradedb-ncc-1701-d ../../ + - assert: + file: ./01-paradedb-NCC-1701-D_cluster-assert.yaml + catch: + - describe: + apiVersion: v1 + kind: Pod + - describe: + apiVersion: batch/v1 + kind: Job + - podLogs: + selector: cnpg.io/cluster=paradedb-ncc-1701-d + - name: Verify ParadeDB Enterprise extensions are installed + timeouts: + apply: 1s + assert: 30s + try: + - apply: + file: 02-paradedb_test.yaml + - assert: + file: 02-paradedb_test-assert.yaml + catch: + - describe: + apiVersion: batch/v1 + kind: Job + - podLogs: + selector: batch.kubernetes.io/job-name=data-test + - name: Cleanup + try: + - script: + content: | + helm uninstall --namespace $NAMESPACE paradedb-ncc-1701-d diff --git a/charts/paradedb/values.schema.json b/charts/paradedb/values.schema.json index b486722e6..a810a2e1d 100644 --- a/charts/paradedb/values.schema.json +++ b/charts/paradedb/values.schema.json @@ -293,6 +293,9 @@ "roles": { "type": "array" }, + "serviceAccountTemplate": { + "type": "object" + }, "storage": { "type": "object", "properties": { diff --git a/charts/paradedb/values.yaml b/charts/paradedb/values.yaml index 7f545db8f..57fddf2d6 100644 --- a/charts/paradedb/values.yaml +++ b/charts/paradedb/values.yaml @@ -6,6 +6,7 @@ fullnameOverride: "" ### # -- Type of the CNPG database. Available types: # * `paradedb` +# * `paradedb-enterprise` type: paradedb version: