diff --git a/.github/workflows/linter.yaml b/.github/workflows/linter.yaml index b0ac0ad..86919af 100644 --- a/.github/workflows/linter.yaml +++ b/.github/workflows/linter.yaml @@ -30,3 +30,5 @@ jobs: VALIDATE_YAML: false DEFAULT_BRANCH: main GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # conflict with python flask8 format + VALIDATE_PYTHON_BLACK: false diff --git a/charts/neon-storage-controller/Chart.yaml b/charts/neon-storage-controller/Chart.yaml index 72548ea..e979d73 100644 --- a/charts/neon-storage-controller/Chart.yaml +++ b/charts/neon-storage-controller/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: neon-storage-controller description: Neon storage controller type: application -version: 1.0.1 +version: 1.0.2 appVersion: "0.1.0" kubeVersion: "^1.18.x-x" home: https://neon.tech diff --git a/charts/neon-storage-controller/README.md b/charts/neon-storage-controller/README.md index 14d014b..9101a1c 100644 --- a/charts/neon-storage-controller/README.md +++ b/charts/neon-storage-controller/README.md @@ -1,6 +1,6 @@ # neon-storage-controller -![Version: 1.0.1](https://img.shields.io/badge/Version-1.0.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) [![Lint and Test Charts](https://github.com/neondatabase/helm-charts/actions/workflows/lint-test.yaml/badge.svg)](https://github.com/neondatabase/helm-charts/actions/workflows/lint-test.yaml) +![Version: 1.0.2](https://img.shields.io/badge/Version-1.0.2-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) [![Lint and Test Charts](https://github.com/neondatabase/helm-charts/actions/workflows/lint-test.yaml/badge.svg)](https://github.com/neondatabase/helm-charts/actions/workflows/lint-test.yaml) Neon storage controller @@ -45,6 +45,11 @@ Kubernetes: `^1.18.x-x` | podAnnotations | object | `{}` | Annotations for neon-storage-controller pods | | podLabels | object | `{}` | Additional labels for neon-storage-controller pods | | podSecurityContext | object | `{}` | neon-storage-controller's pods Security Context | +| registerControlPlane.enable | bool | `false` | | +| registerControlPlane.resources.limits.cpu | string | `"100m"` | | +| registerControlPlane.resources.limits.memory | string | `"128M"` | | +| registerControlPlane.resources.requests.cpu | string | `"100m"` | | +| registerControlPlane.resources.requests.memory | string | `"128M"` | | | resources.limits.memory | string | `"4Gi"` | | | resources.requests.cpu | string | `"200m"` | | | resources.requests.memory | string | `"1Gi"` | | @@ -55,6 +60,7 @@ Kubernetes: `^1.18.x-x` | serviceAccount.annotations | object | `{}` | Annotations to add to the service account | | serviceAccount.create | bool | `true` | | | serviceAccount.name | string | `""` | | +| settings.apiKey | string | `""` | | | settings.computeHookUrl | string | `""` | | | settings.controlPlaneJwtToken | string | `""` | | | settings.databaseUrl | string | `""` | | diff --git a/charts/neon-storage-controller/scripts/register-storage-controller.py b/charts/neon-storage-controller/scripts/register-storage-controller.py new file mode 100644 index 0000000..ef33c17 --- /dev/null +++ b/charts/neon-storage-controller/scripts/register-storage-controller.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python + +import os +import sys +import json +import logging +import urllib.request +import urllib.error + +# region_id different in console/cplan with prefix aws- +REGION = os.environ["REGION_ID"] +# ZONE env will be autogenerated from init container +ZONE = os.environ["ZONE"] +HOST = os.environ["HOST"] +PORT = os.getenv("PORT", 50051) + +GLOBAL_CPLANE_JWT_TOKEN = os.environ["JWT_TOKEN"] +LOCAL_CPLANE_JWT_TOKEN = os.environ["CONTROL_PLANE_JWT_TOKEN"] +CONSOLE_API_KEY = os.environ["CONSOLE_API_KEY"] + +# To register new pageservers +URL_PATH = "management/api/v2/pageservers" +# To get pageservers +ADMIN_URL_PATH = "api/v1/admin/pageservers" + +GLOBAL_CPLANE_URL = f"{os.environ['GLOBAL_CPLANE_URL'].strip('/')}/{URL_PATH}" +LOCAL_CPLANE_URL = f"{os.environ['LOCAL_CPLANE_URL'].strip('/')}/{URL_PATH}" +CONSOLE_URL = f"{os.environ['CONSOLE_URL']}/{ADMIN_URL_PATH}" + +PAYLOAD = dict( + host=HOST, + region_id=REGION, + port=6400, + disk_size=0, + instance_id=HOST, + http_host=HOST, + http_port=int(PORT), + availability_zone_id=ZONE, + instance_type="", + register_reason="Storage Controller Virtual Pageserver", + active=False, + is_storage_controller=True, +) + + +def get_data(url, token, host=None): + if host is not None: + url = f"{url}/{host}" + headers = { + "Authorization": f"Bearer {token}", + "Accept": "application/json", + "Content-Type": "application/json", + } + # Check if the server is already registered + req = urllib.request.Request(url=url, headers=headers, method="GET") + try: + with urllib.request.urlopen(req) as response: + if response.getcode() == 200: + return json.loads(response.read()) + except urllib.error.URLError: + pass + return {} + + +def get_pageserver_id(url, token): + data = get_data(url, token, HOST) + if "node_id" in data: + return data["node_id"] + + +def get_pageserver_version(): + data = get_data(CONSOLE_URL, CONSOLE_API_KEY) + if "data" not in data: + return -1 + for pageserver in data["data"]: + region_id = pageserver["region_id"] + if region_id == REGION or region_id == f"{REGION}-new": + return pageserver["version"] + return -1 + + +def register(url, token, payload): + headers = { + "Authorization": f"Bearer {token}", + "Accept": "application/json", + "Content-Type": "application/json", + "User-Agent": "Python script 1.0", + } + data = str(json.dumps(payload)).encode() + req = urllib.request.Request( + url=url, + data=data, + headers=headers, + method="POST", + ) + with urllib.request.urlopen(req) as resp: + response = json.loads(resp.read()) + log.info(response) + if "node_id" in response: + return response["node_id"] + + +if __name__ == "__main__": + logging.basicConfig( + style="{", + format="{asctime} {levelname:8} {name}:{lineno} {message}", + level=logging.INFO, + ) + + log = logging.getLogger() + + log.info( + json.dumps( + dict( + GLOBAL_CPLANE_URL=GLOBAL_CPLANE_URL, + LOCAL_CPLANE_URL=LOCAL_CPLANE_URL, + CONSOLE_URL=CONSOLE_URL, + **PAYLOAD, + ), + indent=4, + ) + ) + + log.info("get version from existing deployed pageserver") + version = get_pageserver_version() + + if version == -1: + log.error(f"Unable to find pageserver version from {CONSOLE_URL}") + sys.exit(1) + + log.info(f"found latest version={version} for region={REGION}") + PAYLOAD.update(dict(version=version)) + + log.info("check if pageserver already registered or not in console") + node_id_in_console = get_pageserver_id(GLOBAL_CPLANE_URL, GLOBAL_CPLANE_JWT_TOKEN) + + if node_id_in_console is None: + log.info("Registering storage controller in console") + node_id_in_console = register( + GLOBAL_CPLANE_URL, GLOBAL_CPLANE_JWT_TOKEN, PAYLOAD + ) + log.info( + f"Storage controller registered in console with node_id \ + {node_id_in_console}" + ) + else: + log.info( + f"Storage controller already registered in console with node_id \ + {node_id_in_console}" + ) + + log.info("check if pageserver already registered or not in cplane") + node_id_in_cplane = get_pageserver_id(LOCAL_CPLANE_URL, LOCAL_CPLANE_JWT_TOKEN) + + if node_id_in_cplane is None: + PAYLOAD.update(dict(node_id=str(node_id_in_console))) + log.info("Registering storage controller in cplane") + node_id_in_cplane = register(LOCAL_CPLANE_URL, LOCAL_CPLANE_JWT_TOKEN, PAYLOAD) + log.info( + f"Storage controller registered in cplane with node_id \ + {node_id_in_cplane}" + ) + else: + log.info( + f"Storage controller already registered in cplane with node_id \ + {node_id_in_cplane}" + ) diff --git a/charts/neon-storage-controller/templates/deployment.yaml b/charts/neon-storage-controller/templates/deployment.yaml index a829fe0..eb2419b 100644 --- a/charts/neon-storage-controller/templates/deployment.yaml +++ b/charts/neon-storage-controller/templates/deployment.yaml @@ -47,21 +47,7 @@ spec: - -l # In the container, use the same port as service. - 0.0.0.0:{{ .Values.service.port }} - {{- if .Values.settings.databaseUrl }} - - --database-url - - {{ .Values.settings.databaseUrl }} - {{- end }} - {{- if .Values.settings.jwtToken }} - - --jwt-token - - {{ .Values.settings.jwtToken }} - {{- end }} - {{- if .Values.settings.publicKey }} - --public-key={{ .Values.settings.publicKey | toJson }} - {{- end }} - {{- if .Values.settings.controlPlaneJwtToken }} - - --control-plane-jwt-token - - {{ .Values.settings.controlPlaneJwtToken }} - {{- end }} {{- if .Values.settings.computeHookUrl }} - --compute-hook-url - {{ .Values.settings.computeHookUrl }} @@ -79,6 +65,9 @@ spec: value: {{ . }} {{- end }} {{- end }} + envFrom: + - secretRef: + name: {{ include "neon-storage-controller.fullname" . }}-env-vars ports: - name: controller containerPort: {{ .Values.service.port }} diff --git a/charts/neon-storage-controller/templates/node-describe-rbac.yaml b/charts/neon-storage-controller/templates/node-describe-rbac.yaml new file mode 100644 index 0000000..04cdbcb --- /dev/null +++ b/charts/neon-storage-controller/templates/node-describe-rbac.yaml @@ -0,0 +1,41 @@ +{{- if .Values.registerControlPlane.enable -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "neon-storage-controller.fullname" . }}-node-describe + labels: + {{- include "neon-storage-controller.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "neon-storage-controller.fullname" . }}-node-reader + labels: + {{- include "neon-storage-controller.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "neon-storage-controller.fullname" . }}-node-reader + labels: + {{- include "neon-storage-controller.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "neon-storage-controller.fullname" . }}-node-reader +subjects: +- kind: ServiceAccount + name: {{ include "neon-storage-controller.fullname" . }}-node-describe + namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/charts/neon-storage-controller/templates/post-install-job.yaml b/charts/neon-storage-controller/templates/post-install-job.yaml new file mode 100644 index 0000000..3e5cc14 --- /dev/null +++ b/charts/neon-storage-controller/templates/post-install-job.yaml @@ -0,0 +1,84 @@ +{{- if .Values.registerControlPlane.enable -}} +{{ if (empty .Values.registerControlPlane.region_id ) }} + {{- fail (printf "Value for .Values.registerControlPlane.region_id is empty") }} +{{- end }} +{{ if (empty .Values.registerControlPlane.global_cplane_url) }} + {{- fail (printf "Value for .Values.registerControlPlane.global_cplane_url is empty") }} +{{- end }} +{{ if (empty .Values.registerControlPlane.local_cplane_url) }} + {{- fail (printf "Value for .Values.registerControlPlane.local_cplane_url is empty") }} +{{- end }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "neon-storage-controller.fullname" . }}-register-job + labels: + {{- include "neon-storage-controller.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": post-install + "helm.sh/hook-weight": "-5" + # Delete the previous resource before a new hook is launche + "helm.sh/hook-delete-policy": before-hook-creation +spec: + parallelism: 1 + completions: 1 + template: + spec: + serviceAccountName: "{{ include "neon-storage-controller.fullname" . }}-node-describe" + initContainers: + - name: set-envs + image: public.ecr.aws/bitnami/kubectl:1.26 + resources: + {{- toYaml .Values.registerControlPlane.resources | nindent 10 }} + command: ["sh", "-c"] + args: + - | + ZONE=$(kubectl get node "$NODE_NAME" -o=jsonpath='{.metadata.labels.topology\.kubernetes\.io/zone}') + echo export ZONE=$ZONE > /node/env.sh + volumeMounts: + - name: node-info + mountPath: /node + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + containers: + - name: register-storage-controller + image: public.ecr.aws/docker/library/python:3.12.2-slim-bullseye + imagePullPolicy: IfNotPresent + command: ["/bin/bash", "-c"] + args: ["source /node/env.sh && /usr/local/bin/python /opt/scripts/register-storage-controller.py"] + resources: + {{- toYaml .Values.registerControlPlane.resources | nindent 10 }} + volumeMounts: + - name: config-volume + mountPath: /opt/scripts + - name: node-info + mountPath: /node + env: + - name: HOST + value: {{ index .Values.service.annotations "external-dns.alpha.kubernetes.io/hostname" | quote }} + - name: PORT + value: {{ .Values.service.port | quote }} + - name: GLOBAL_CPLANE_URL + value: {{ .Values.registerControlPlane.global_cplane_url | quote }} + - name: LOCAL_CPLANE_URL + value: {{ .Values.registerControlPlane.local_cplane_url | quote }} + - name: CONSOLE_URL + value: {{ .Values.registerControlPlane.console_url | quote }} + - name: REGION_ID + value: {{ .Values.registerControlPlane.region_id | quote }} + envFrom: + - secretRef: + name: {{ include "neon-storage-controller.fullname" . }}-env-vars + volumes: + - name: config-volume + secret: + secretName: {{ include "neon-storage-controller.fullname" . }}-post-install-script + defaultMode: 0755 + - name: node-info + emptyDir: {} + restartPolicy: Never + terminationGracePeriodSeconds: 0 +{{- end }} diff --git a/charts/neon-storage-controller/templates/post-install-secrets.yaml b/charts/neon-storage-controller/templates/post-install-secrets.yaml new file mode 100644 index 0000000..b64fb64 --- /dev/null +++ b/charts/neon-storage-controller/templates/post-install-secrets.yaml @@ -0,0 +1,12 @@ +{{- if .Values.registerControlPlane.enable -}} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "neon-storage-controller.fullname" . }}-post-install-script + labels: + {{- include "neon-storage-controller.labels" . | nindent 4 }} +type: Opaque +data: + register-storage-controller.py: |- + {{ tpl (.Files.Get "scripts/register-storage-controller.py") . | b64enc }} +{{- end }} diff --git a/charts/neon-storage-controller/templates/secrets.yaml b/charts/neon-storage-controller/templates/secrets.yaml new file mode 100644 index 0000000..2166224 --- /dev/null +++ b/charts/neon-storage-controller/templates/secrets.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "neon-storage-controller.fullname" . }}-env-vars + labels: + {{- include "neon-storage-controller.labels" . | nindent 4 }} +type: Opaque +data: + DATABASE_URL: {{ .Values.settings.databaseUrl | b64enc | quote }} + JWT_TOKEN: {{ .Values.settings.jwtToken| b64enc | quote }} + CONTROL_PLANE_JWT_TOKEN: {{ .Values.settings.controlPlaneJwtToken | b64enc | quote }} + CONSOLE_API_KEY: {{ .Values.settings.apiKey | b64enc | quote }} diff --git a/charts/neon-storage-controller/values.yaml b/charts/neon-storage-controller/values.yaml index b06c9f2..04d033b 100644 --- a/charts/neon-storage-controller/values.yaml +++ b/charts/neon-storage-controller/values.yaml @@ -34,6 +34,30 @@ settings: controlPlaneJwtToken: "" # URL for compute notifications computeHookUrl: "" + # Console API key to list existing pageserver to get version + apiKey: "" + +# Enable auto register to control plane +# This will run postinstall job +registerControlPlane: + enable: false + # region_id: "aws-us-east-2" + # generate using + # curl https://console.stage.neon.tech/regions/console/api/v1/admin/issue_token \ + # -H "Accept: application/json" \ + # -H "Content-Type: application/json" \ + # -H "Authorization: Bearer $NEON_API_KEY" \ + # -X POST -d '{"ttl_seconds": 31536000, "scope": "infra"}' + # global_cplane_url: "http://neon-internal-api.aws.neon.build" + # local_cplane_url: "https://control-plane.zeta.us-east-2.internal.aws.neon.build" + # console_url: "" + resources: + limits: + cpu: 100m + memory: 128M + requests: + cpu: 100m + memory: 128M serviceAccount: # serviceAccount.create - Specifies whether a service account should be created