Revert "Revert "proxy-parquet-config""

neondatabase · Jan 11, 2024 · 8b04b12 · 8b04b12
1 parent 6db45f1
commit 8b04b12
Show file tree

Hide file tree

Showing 4 changed files with 58 additions and 2 deletions.
diff --git a/charts/neon-proxy/Chart.yaml b/charts/neon-proxy/Chart.yaml
@@ -2,7 +2,7 @@ apiVersion: v2
 name: neon-proxy
 description: Neon Proxy
 type: application
-version: 1.7.12
+version: 1.7.11
 appVersion: "0.1.0"
 kubeVersion: "^1.18.x-x"
 home: https://neon.tech

diff --git a/charts/neon-proxy/README.md b/charts/neon-proxy/README.md
@@ -1,6 +1,6 @@
 # neon-proxy
 
-![Version: 1.7.12](https://img.shields.io/badge/Version-1.7.12-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) [![Lint and Test Charts](https://github.com/neondatabase/helm-charts/actions/workflows/lint-test.yaml/badge.svg)](https://github.com/neondatabase/helm-charts/actions/workflows/lint-test.yaml)
+![Version: 1.7.11](https://img.shields.io/badge/Version-1.7.11-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) [![Lint and Test Charts](https://github.com/neondatabase/helm-charts/actions/workflows/lint-test.yaml/badge.svg)](https://github.com/neondatabase/helm-charts/actions/workflows/lint-test.yaml)
 
 Neon Proxy
 
@@ -63,6 +63,8 @@ Kubernetes: `^1.18.x-x`
 | serviceAccount.name | string | `""` |  |
 | settings.authBackend | string | `"link"` | auth method used (console|link|postgres) |
 | settings.authEndpoint | string | `""` | auth endpoint, e.g. "http://console.neon/authenticate_proxy_request/" |
+| settings.awsAccessKeyId | string | `""` | (string) AWS Access Key ID |
+| settings.awsSecretAccessKey | string | `""` | (string) AWS Secret Access Key |
 | settings.controlplane_token | string | `""` | (string) JWT token to pass to control plane management API |
 | settings.disableDynamicRateLimiter | bool | `true` | (bool) Disable dynamic rate limiter |
 | settings.domain | string | `""` | domain used in TLS cert for client postgres connections |
@@ -73,6 +75,13 @@ Kubernetes: `^1.18.x-x`
 | settings.metricCollectionEndpoint | string | `""` | (url) endpoint used to send metrics to. If null, metrics will not be sent. |
 | settings.metricCollectionInterval | string | `""` | (string) how often metrics should be sent. |
 | settings.otelExporterOtlpEndpoint | string | `""` | OpenTelemetry collector URL (will be converted into `OTEL_EXPORTER_OTLP_ENDPOINT` environment variable) |
+| settings.parquetUploadCompression | string | `"uncompressed"` | (string) What level of compression to use |
+| settings.parquetUploadMaximumDuration | string | `"20m"` | (string) How long to wait before forcing a file upload |
+| settings.parquetUploadPageSize | string | `"1048576"` | (string) How large each column page should be in bytes |
+| settings.parquetUploadRemoteStorage | string | `""` | (string) Storage location to upload the parquet files to. |
+| settings.parquetUploadRowGroupSize | string | `"8192"` | (string) How many rows to include in a row group |
+| settings.parquetUploadSize | string | `"100000000"` | (string) How large the total parquet file should be in bytes |
+| settings.region | string | `""` | (string) Region this proxy service is deployed into |
 | settings.sentryEnvironment | string | `"development"` | "development" or "production". It will be visible in sentry in order to filter issues |
 | settings.sentryUrl | string | `""` | url (will be converted into `SENTRY_DSN` environment variable) used by sentry to collect error/panic events in neon-proxy |
 | settings.uri | string | `""` |  |

diff --git a/charts/neon-proxy/templates/deployment.yaml b/charts/neon-proxy/templates/deployment.yaml
@@ -108,6 +108,27 @@ spec:
             {{- range .Values.settings.endpointRpsLimits }}
             - --endpoint-rps-limit={{ . }}
             {{- end }}
+            {{- with .Values.settings.region }}
+            - --region={{ . }}
+            {{ end }}
+            {{- with .Values.settings.parquetUploadRemoteStorage }}
+            - --parquet-upload-remote-storage={{ . }}
+            {{ end }}
+            {{- with .Values.settings.parquetUploadRowGroupSize }}
+            - --parquet-upload-row-group-size={{ . }}
+            {{ end }}
+            {{- with .Values.settings.parquetUploadPageSize }}
+            - --parquet-upload-page-size={{ . }}
+            {{ end }}
+            {{- with .Values.settings.parquetUploadSize }}
+            - --parquet-upload-size={{ . }}
+            {{ end }}
+            {{- with .Values.settings.parquetUploadMaximumDuration }}
+            - --parquet-upload-maximum-duration={{ . }}
+            {{ end }}
+            {{- with .Values.settings.parquetUploadCompression }}
+            - --parquet-upload-compression={{ . }}
+            {{ end }}
             {{- with .Values.settings.wakeComputeLock }}
             - --wake-compute-lock
             - {{ . }}
@@ -130,6 +151,14 @@ spec:
             - name: NEON_PROXY_TO_CONTROLPLANE_TOKEN
               value: {{ . }}
             {{- end }}
+            {{- with .Values.settings.awsAccessKeyId }}
+            - name: AWS_ACCESS_KEY_ID
+              value: {{ . }}
+            {{- end }}
+            {{- with .Values.settings.awsSecretAccessKey }}
+            - name: AWS_SECRET_ACCESS_KEY
+              value: {{ . }}
+            {{- end }}
           {{- end }}
           {{- if .Values.settings.domain }}
           volumeMounts:

diff --git a/charts/neon-proxy/values.yaml b/charts/neon-proxy/values.yaml
@@ -65,6 +65,24 @@ settings:
   endpointRpsLimits:
   - 300@1s
   - 250@10m
+  # settings.parquetUploadRemoteStorage -- (string) Storage location to upload the parquet files to.
+  parquetUploadRemoteStorage: ""
+  # settings.parquetUploadRowGroupSize -- (string) How many rows to include in a row group
+  parquetUploadRowGroupSize: "8192"
+  # settings.parquetUploadPageSize -- (string) How large each column page should be in bytes
+  parquetUploadPageSize: "1048576"
+  # settings.parquetUploadSize -- (string) How large the total parquet file should be in bytes
+  parquetUploadSize: "100000000"
+  # settings.parquetUploadMaximumDuration -- (string) How long to wait before forcing a file upload
+  parquetUploadMaximumDuration: "20m"
+  # settings.parquetUploadCompression -- (string) What level of compression to use
+  parquetUploadCompression: uncompressed
+  # settings.region -- (string) Region this proxy service is deployed into
+  region: ""
+  # settings.awsAccessKeyId -- (string) AWS Access Key ID
+  awsAccessKeyId: ""
+  # settings.awsSecretAccessKey -- (string) AWS Secret Access Key
+  awsSecretAccessKey: ""
 
 serviceAccount:
   # serviceAccount.create - Specifies whether a service account should be created